logstash-core 1.5.0.beta2-java

data/lib/logstash/pluginmanager.rb

@@ -0,0 +1,17 @@
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+require "logstash/environment"
+
+ENV["GEM_HOME"] = ENV["GEM_PATH"] = LogStash::Environment.logstash_gem_home
+Gem.use_paths(LogStash::Environment.logstash_gem_home)
+
+require 'logstash/pluginmanager/main'
+
+if __FILE__ == $0
+  begin
+    LogStash::PluginManager::Main.run("bin/plugin", ARGV)
+  rescue LogStash::PluginManager::Error => e
+    $stderr.puts(e.message)
+    exit(1)
+  end
+end

data/lib/logstash/pluginmanager/install.rb

@@ -0,0 +1,112 @@
+require 'clamp'
+require 'logstash/namespace'
+require 'logstash/environment'
+require 'logstash/pluginmanager/util'
+require 'jar-dependencies'
+require 'jar_install_post_install_hook'
+require 'file-dependencies/gem'
+
+require "logstash/gemfile"
+require "logstash/bundler"
+
+class LogStash::PluginManager::Install < Clamp::Command
+  parameter "[PLUGIN] ...", "plugin name(s) or file"
+  option "--version", "VERSION", "version of the plugin to install"
+  option "--[no-]verify", :flag, "verify plugin validity before installation", :default => true
+  option "--development", :flag, "install all development dependencies of currently installed plugins", :default => false
+
+  # the install logic below support installing multiple plugins with each a version specification
+  # but the argument parsing does not support it for now so currently if specifying --version only
+  # one plugin name can be also specified.
+  #
+  # TODO: find right syntax to allow specifying list of plugins with optional version specification for each
+
+  def execute
+    if development?
+      raise(LogStash::PluginManager::Error, "Cannot specify plugin(s) with --development, it will add the development dependencies of the currently installed plugins") unless plugin_list.empty?
+    else
+      raise(LogStash::PluginManager::Error, "No plugin specified") if plugin_list.empty? && verify?
+
+      # temporary until we fullfil TODO ^^
+      raise(LogStash::PluginManager::Error, "Only 1 plugin name can be specified with --version") if version && plugin_list.size > 1
+    end
+    raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
+
+    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
+    # keep a copy of the gemset to revert on error
+    original_gemset = gemfile.gemset.copy
+
+    # force Rubygems sources to our Gemfile sources
+    Gem.sources = gemfile.gemset.sources
+
+    # install_list will be an array of [plugin name, version] tuples, version can be nil
+    install_list = []
+
+    if development?
+      specs = LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile)
+      install_list = specs.inject([]) do |result, spec|
+        result = result + spec.dependencies.select{|dep| dep.type == :development}.map{|dep| [dep.name] + dep.requirement.as_list + [{:group => :development}]}
+      end
+    else
+      # at this point we know that plugin_list is not empty and if the --version is specified there is only one plugin in plugin_list
+
+      install_list = version ? [plugin_list << version] : plugin_list.map{|plugin| [plugin, nil]}
+
+      install_list.each do |plugin, version|
+        puts("Validating #{[plugin, version].compact.join("-")}")
+        raise(LogStash::PluginManager::Error, "Installation aborted") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
+      end if verify?
+
+      # at this point we know that we either have a valid gem name & version or a valid .gem file path
+
+      # if LogStash::PluginManager.plugin_file?(plugin)
+      #   raise(LogStash::PluginManager::Error) unless cache_gem_file(plugin)
+      #   spec = LogStash::PluginManager.plugin_file_spec(plugin)
+      #   gemfile.update(spec.name, spec.version.to_s)
+      # else
+      #   plugins.each{|tuple| gemfile.update(*tuple)}
+      # end
+    end
+
+
+    install_list = LogStash::PluginManager.merge_duplicates(install_list)
+    install_list.each{|plugin, version| gemfile.update(plugin, version)}
+    gemfile.save
+
+    puts("Installing" + (install_list.empty? ? "..." : " " + install_list.map{|plugin, version| plugin}.join(", ")))
+
+    bundler_options = {:install => true}
+    bundler_options[:without] = [] if development?
+
+    # any errors will be logged to $stderr by invoke_bundler!
+    output, exception = LogStash::Bundler.invoke_bundler!(bundler_options)
+
+    if ENV["DEBUG"]
+      $stderr.puts(output)
+      $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
+    end
+
+    if exception
+      # revert to original Gemfile content
+      gemfile.gemset = original_gemset
+      gemfile.save
+      raise(LogStash::PluginManager::Error, "Installation aborted")
+    end
+
+    puts("Installation successful")
+  end
+
+  # copy .gem file into bundler cache directory, log any error to $stderr
+  # @param path [String] the source .gem file to copy
+  # @return [Boolean] true if successful
+  def cache_gem_file(path)
+    dest = ::File.join(LogStash::Environment.logstash_gem_home, "cache")
+    begin
+      FileUtils.cp(path, dest)
+    rescue => e
+      $stderr.puts("Error copying #{plugin} to #{dest}, caused by #{e.class}")
+      return false
+    end
+    true
+  end
+end # class Logstash::PluginManager

data/lib/logstash/pluginmanager/list.rb

@@ -0,0 +1,38 @@
+require 'clamp'
+require 'logstash/namespace'
+require 'logstash/pluginmanager/util'
+require 'rubygems/spec_fetcher'
+
+class LogStash::PluginManager::List < Clamp::Command
+
+  parameter "[PLUGIN]", "Part of plugin name to search for, leave empty for all plugins"
+
+  option "--installed", :flag, "List only explicitly installed plugins using bin/plugin install ...", :default => false
+  option "--verbose", :flag, "Also show plugin version number", :default => false
+  option "--group", "NAME", "Filter plugins per group: input, output, filter or codec" do |arg|
+    raise(ArgumentError, "should be one of: input, output, filter or codec") unless ['input', 'output', 'filter', 'codec'].include?(arg)
+    arg
+  end
+
+  def execute
+    Gem.configuration.verbose = false
+
+    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
+
+    # start with all locally installed plugin gems regardless of the Gemfile content
+    specs = LogStash::PluginManager.find_plugins_gem_specs
+
+    # apply filters
+    specs = specs.select{|spec| gemfile.find(spec.name)} if installed?
+    specs = specs.select{|spec| spec.name =~ /#{plugin}/i} if plugin
+    specs = specs.select{|spec| spec.metadata['logstash_group'] == group} if group
+
+    raise(LogStash::PluginManager::Error, "No plugins found") if specs.empty?
+
+    specs.sort_by{|spec| spec.name}.each do |spec|
+      line = "#{spec.name}"
+      line += " (#{spec.version})" if verbose?
+      puts(line)
+    end
+  end
+end # class Logstash::PluginManager

data/lib/logstash/pluginmanager/main.rb

@@ -0,0 +1,22 @@
+require "logstash/namespace"
+require "logstash/errors"
+require "logstash/pluginmanager/install"
+require "logstash/pluginmanager/uninstall"
+require "logstash/pluginmanager/list"
+require "logstash/pluginmanager/update"
+require "logstash/pluginmanager/util"
+require "logstash/pluginmanager/maven_tools_patch"
+require "clamp"
+
+module LogStash
+  module PluginManager
+    class Error < StandardError; end
+
+    class Main < Clamp::Command
+      subcommand "install", "Install a plugin", LogStash::PluginManager::Install
+      subcommand "uninstall", "Uninstall a plugin", LogStash::PluginManager::Uninstall
+      subcommand "update", "Install a plugin", LogStash::PluginManager::Update
+      subcommand "list", "List all installed plugins", LogStash::PluginManager::List
+    end
+  end
+end

data/lib/logstash/pluginmanager/maven_tools_patch.rb

@@ -0,0 +1,12 @@
+# This adds the "repo" element to the jar-dependencies DSL
+# allowing a gemspec to require a jar that exists in a custom
+# maven repository
+# Example:
+#   gemspec.requirements << "repo http://localhosty/repo"
+require 'maven/tools/dsl/project_gemspec'
+class Maven::Tools::DSL::ProjectGemspec
+  def repo(url)
+    @parent.repository(:id => url, :url => url)
+  end
+end
+

data/lib/logstash/pluginmanager/uninstall.rb

@@ -0,0 +1,49 @@
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/errors"
+require "logstash/environment"
+require "logstash/pluginmanager/util"
+require "clamp"
+
+require "logstash/gemfile"
+require "logstash/bundler"
+
+class LogStash::PluginManager::Uninstall < Clamp::Command
+  parameter "PLUGIN", "plugin name"
+
+
+  def execute
+    raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
+
+    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
+    # keep a copy of the gemset to revert on error
+    original_gemset = gemfile.gemset.copy
+
+    # make sure this is an installed plugin and present in Gemfile.
+    # it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
+    raise(LogStash::PluginManager::Error, "This plugin has not been previously installed, aborting") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
+
+    # since we previously did a gemfile.find(plugin) there is no reason why
+    # remove would not work (return nil) here
+    if gemfile.remove(plugin)
+      gemfile.save
+
+      puts("Uninstalling #{plugin}")
+
+      # any errors will be logged to $stderr by invoke_bundler!
+      output, exception = LogStash::Bundler.invoke_bundler!(:clean => true)
+
+      if ENV["DEBUG"]
+        $stderr.puts(output)
+        $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
+      end
+
+      if exception
+        # revert to original Gemfile content
+        gemfile.gemset = original_gemset
+        gemfile.save
+        raise(LogStash::PluginManager::Error, "Uninstall aborted")
+      end
+    end
+  end
+end

data/lib/logstash/pluginmanager/update.rb

@@ -0,0 +1,50 @@
+require 'clamp'
+require 'logstash/namespace'
+require 'logstash/pluginmanager/util'
+require 'jar-dependencies'
+require 'jar_install_post_install_hook'
+require 'file-dependencies/gem'
+
+require "logstash/gemfile"
+require "logstash/bundler"
+
+class LogStash::PluginManager::Update < Clamp::Command
+  parameter "[PLUGIN] ...", "Plugin name(s) to upgrade to latest version"
+
+  def execute
+    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
+    # keep a copy of the gemset to revert on error
+    original_gemset = gemfile.gemset.copy
+
+    # create list of plugins to update
+    plugins = unless plugin_list.empty?
+      not_installed = plugin_list.find{|plugin| !LogStash::PluginManager.installed_plugin?(plugin, gemfile)}
+      raise(LogStash::PluginManager::Error, "Plugin #{not_installed} has not been previously installed, aborting") if not_installed
+      plugin_list
+    else
+      LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile).map{|spec| spec.name}
+    end
+
+    # remove any version constrain from the Gemfile so the plugin(s) can be updated to latest version
+    # calling update without requiremend will remove any previous requirements
+    plugins.each{|plugin| gemfile.update(plugin)}
+    gemfile.save
+
+    puts("Updating " + plugins.join(", "))
+
+    # any errors will be logged to $stderr by invoke_bundler!
+    output, exception = LogStash::Bundler.invoke_bundler!(:update => plugins)
+
+    if ENV["DEBUG"]
+      $stderr.puts(output)
+      $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
+    end
+
+    if exception
+      # revert to original Gemfile content
+      gemfile.gemset = original_gemset
+      gemfile.save
+      raise(LogStash::PluginManager::Error, "Update aborted")
+    end
+  end
+end

data/lib/logstash/pluginmanager/util.rb

@@ -0,0 +1,88 @@
+module LogStash::PluginManager
+
+  # check for valid logstash plugin gem name & version or .gem file, logs errors to $stdout
+  # uses Rubygems API and will remotely validated agains the current Gem.sources
+  # @param plugin [String] plugin name or .gem file path
+  # @param version [String] gem version requirement string
+  # @return [Boolean] true if valid logstash plugin gem name & version or a .gem file
+  def self.logstash_plugin?(plugin, version = nil)
+    if plugin_file?(plugin)
+      begin
+        return logstash_plugin_gem_spec?(plugin_file_spec(plugin))
+      rescue => e
+        $stderr.puts("Error reading plugin file #{plugin}, caused by #{e.class}")
+        $stderr.puts(e.message) if ENV["DEBUG"]
+        return false
+      end
+    else
+      dep = Gem::Dependency.new(plugin, version || Gem::Requirement.default)
+      specs, error = Gem::SpecFetcher.fetcher.spec_for_dependency(dep)
+
+      # depending on version requirements, multiple specs can be returned in which case
+      # we will grab the one with the highest version number
+      if latest = specs.map(&:first).max_by(&:version)
+        unless valid = logstash_plugin_gem_spec?(latest)
+          $stderr.puts("#{plugin} is not a Logstash plugin")
+        end
+        return valid
+      else
+        $stderr.puts("Plugin #{plugin}" + (version ? " version #{version}" : "") + " does not exist")
+        return false
+      end
+    end
+  end
+
+  # @param spec [Gem::Specification] plugin gem specification
+  # @return [Boolean] true if this spec is for an installable logstash plugin
+  def self.logstash_plugin_gem_spec?(spec)
+    spec.metadata && spec.metadata["logstash_plugin"] == "true"
+  end
+
+  # @param path [String] path to .gem file
+  # @return [Gem::Specification] .get file gem specification
+  # @raise [Exception] Gem::Package::FormatError will be raised on invalid .gem file format, might be other exceptions too
+  def self.plugin_file_spec(path)
+    Gem::Package.new(path).spec
+  end
+
+  # @param plugin [String] the plugin name or the local path to a .gem file
+  # @return [Boolean] true if the plugin is a local .gem file
+  def self.plugin_file?(plugin)
+    (plugin =~ /\.gem$/ && File.file?(plugin))
+  end
+
+  # retrieve gem specs for all or specified name valid logstash plugins locally installed
+  # @param name [String] specific plugin name to find or nil for all plungins
+  # @return [Array<Gem::Specification>] all local logstash plugin gem specs
+  def self.find_plugins_gem_specs(name = nil)
+    specs = name ? Gem::Specification.find_all_by_name(name) : Gem::Specification.find_all
+    specs.select{|spec| logstash_plugin_gem_spec?(spec)}
+  end
+
+  # list of all locally installed plugins specs specified in the Gemfile.
+  # note that an installed plugin dependecies like codecs will not be listed, only those
+  # specifically listed in the Gemfile.
+  # @param gemfile [LogStash::Gemfile] the gemfile to validate against
+  # @return [Array<Gem::Specification>] list of plugin names
+  def self.all_installed_plugins_gem_specs(gemfile)
+    # we start form the installed gemspecs so we can verify the metadata for valid logstash plugin
+    # then filter out those not included in the Gemfile
+    find_plugins_gem_specs.select{|spec| !!gemfile.find(spec.name)}
+  end
+
+  # @param plugin [String] plugin name
+  # @param gemfile [LogStash::Gemfile] the gemfile to validate against
+  # @return [Boolean] true if the plugin is an installed logstash plugin and spefificed in the Gemfile
+  def self.installed_plugin?(plugin, gemfile)
+    !!gemfile.find(plugin) && find_plugins_gem_specs(plugin).any?
+  end
+
+  # @param plugin_list [Array] array of [plugin name, version] tuples
+  # @return [Array] array of [plugin name, version, ...] tuples when duplciate names have been merged and non duplicate version requirements added
+  def self.merge_duplicates(plugin_list)
+
+    # quick & dirty naive dedup for now
+    # TODO: properly merge versions requirements
+    plugin_list.uniq(&:first)
+  end
+end

data/lib/logstash/program.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+
+module LogStash::Program
+  public
+  def exit(value)
+    if RUBY_ENGINE == "jruby"
+      # Kernel::exit() in jruby just tosses an exception? Let's actually exit.
+      Java::java.lang.System.exit(value)
+    else
+      Kernel::exit(value)
+    end
+  end # def exit
+end # module LogStash::Program

data/lib/logstash/runner.rb

@@ -0,0 +1,167 @@
+# encoding: utf-8
+
+Encoding.default_external = Encoding::UTF_8
+$START = Time.now
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+require "logstash/environment"
+LogStash::Environment.bundler_setup!
+LogStash::Environment.load_locale!
+
+Thread.abort_on_exception = true
+
+require "logstash/namespace"
+require "logstash/program"
+
+class LogStash::RSpecsRunner
+  def initialize(args)
+    @args = args
+  end
+
+  def run
+    @result = RSpec::Core::Runner.run(@args)
+  end
+
+  def wait
+    return @result
+  end
+end
+
+class LogStash::Runner
+  include LogStash::Program
+
+  def main(args)
+    require "logstash/util"
+    require "stud/trap"
+    require "stud/task"
+    @startup_interruption_trap = Stud::trap("INT") { puts "Interrupted"; exit 0 }
+
+    LogStash::Util::set_thread_name(self.class.name)
+    #$LOAD_PATH << File.join(File.dirname(__FILE__), "..")
+
+    if RUBY_VERSION < "1.9.2"
+      $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      return 1
+    end
+
+    Stud::untrap("INT", @startup_interruption_trap)
+
+    task = run(args)
+    exit(task.wait)
+  end # def self.main
+
+  def run(args)
+    command = args.shift
+    commands = {
+      "version" => lambda do
+        require "logstash/agent"
+        agent_args = ["--version"]
+        if args.include?("--verbose")
+          agent_args << "--verbose"
+        end
+        return LogStash::Agent.run($0, agent_args)
+      end,
+      "rspec" => lambda do
+        require "rspec/core/runner"
+        require "rspec"
+        spec_path = File.expand_path(File.join(File.dirname(__FILE__), "/../../spec"))
+        $LOAD_PATH << spec_path
+        all_specs = Dir.glob(File.join(spec_path, "/**/*_spec.rb"))
+        rspec = LogStash::RSpecsRunner.new(args.empty? ? all_specs : args)
+        return rspec.run
+      end,
+      "irb" => lambda do
+        require "irb"
+        return IRB.start(__FILE__)
+      end,
+      "pry" => lambda do
+        require "pry"
+        return binding.pry
+      end,
+      "docgen" => lambda do
+        require 'docs/asciidocgen'
+        opts = OptionParser.new
+        settings = {}
+        opts.on("-o DIR", "--output DIR",
+          "Directory to output to; optional. If not specified,"\
+          "we write to stdout.") do |val|
+          settings[:output] = val
+        end
+        args = opts.parse(ARGV)
+        docs = LogStashConfigAsciiDocGenerator.new
+        args.each do |arg|
+          docs.generate(arg, settings)
+        end
+        return 0
+      end,
+      "plugin" => lambda do
+        require 'logstash/pluginmanager'
+        plugin_manager = LogStash::PluginManager::Main.new($0)
+        begin
+          plugin_manager.parse(args)
+          return plugin_manager.execute
+        rescue Clamp::HelpWanted => e
+          show_help(e.command)
+          return 0
+        end
+      end,
+      "agent" => lambda do
+        require "logstash/agent"
+        # Hack up a runner
+        agent = LogStash::Agent.new($0)
+        begin
+          agent.parse(args)
+        rescue Clamp::HelpWanted => e
+          show_help(e.command)
+          return 0
+        rescue Clamp::UsageError => e
+          # If 'too many arguments' then give the arguments to
+          # the next command. Otherwise it's a real error.
+          raise if e.message != "too many arguments"
+          remaining = agent.remaining_arguments
+        end
+
+        return agent.execute
+      end
+    } # commands
+
+    if commands.include?(command)
+      return Stud::Task.new { commands[command].call }
+    else
+      if command.nil?
+        $stderr.puts "No command given"
+      else
+        if !%w(--help -h help).include?(command)
+          # Emit 'no such command' if it's not someone asking for help.
+          $stderr.puts "No such command #{command.inspect}"
+        end
+      end
+      $stderr.puts %q[
+Usage: logstash <command> [command args]
+Run a command with the --help flag to see the arguments.
+For example: logstash agent --help
+
+Available commands:
+  agent - runs the logstash agent
+  version - emits version info about this logstash
+  rspec - runs tests
+      ]
+      #$stderr.puts commands.keys.map { |s| "  #{s}" }.join("\n")
+      return Stud::Task.new { 1 }
+    end
+  end # def run
+
+  # @return true if this file is the main file being run and not via rspec
+  def self.autorun?
+    # caller is the current execution stack
+    $0 == __FILE__ && caller.none?{|entry| entry =~ /rspec/}
+  end
+
+  private
+
+  def show_help(command)
+    puts command.help
+  end
+end # class LogStash::Runner
+
+LogStash::Runner.new.main(ARGV) if LogStash::Runner.autorun?