logstash-core 1.5.0.rc2.snapshot-java → 1.5.0.rc3-java

data/lib/logstash/pluginmanager/command.rb

@@ -0,0 +1,38 @@
+class LogStash::PluginManager::Command < Clamp::Command
+  def gemfile
+    @gemfile ||= LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, 'r+')).load
+  end
+
+  # If set in debug mode we will raise an exception and display the stacktrace
+  def report_exception(readable_message, exception)
+    if ENV["DEBUG"]
+      raise exception
+    else
+      signal_error("#{readable_message}, message: #{exception.message}")
+    end
+  end
+
+  def display_bundler_output(output)
+    if ENV['DEBUG'] && output
+      # Display what bundler did in the last run
+      $stderr.puts("Bundler output")
+      $stderr.puts(output)
+    end
+  end
+
+
+  # Each plugin install for a gemfile create a path with a unique id.
+  # we must clear what is not currently used in the 
+  def remove_unused_locally_installed_gems!
+    used_path = gemfile.locally_installed_gems.collect { |gem| gem.options[:path] }
+
+    Dir.glob(File.join(LogStash::Environment::LOCAL_GEM_PATH, '*')) do |path|
+      FileUtils.rm_rf(relative_path(path)) if used_path.none? { |p| p.start_with?(relative_path(path)) }
+    end
+  end
+
+  def relative_path(path)
+    require "pathname"
+    ::Pathname.new(path).relative_path_from(::Pathname.new(LogStash::Environment::LOGSTASH_HOME)).to_s
+  end
+end

data/lib/logstash/pluginmanager/install.rb

@@ -1,16 +1,17 @@
-require 'clamp'
-require 'logstash/namespace'
-require 'logstash/environment'
-require 'logstash/pluginmanager/util'
-require 'jar-dependencies'
-require 'jar_install_post_install_hook'
-require 'file-dependencies/gem'
-
+require "clamp"
+require "logstash/namespace"
+require "logstash/environment"
+require "logstash/pluginmanager/util"
+require "logstash/pluginmanager/command"
+require "jar-dependencies"
+require "jar_install_post_install_hook"
+require "file-dependencies/gem"
 require "logstash/gemfile"
 require "logstash/bundler"
+require "fileutils"
 
-class LogStash::PluginManager::Install < Clamp::Command
-  parameter "[PLUGIN] ...", "plugin name(s) or file"
+class LogStash::PluginManager::Install < LogStash::PluginManager::Command
+  parameter "[PLUGIN] ...", "plugin name(s) or file", :attribute_name => :plugins_arg
   option "--version", "VERSION", "version of the plugin to install"
   option "--[no-]verify", :flag, "verify plugin validity before installation", :default => true
   option "--development", :flag, "install all development dependencies of currently installed plugins", :default => false
@@ -18,95 +19,123 @@ class LogStash::PluginManager::Install < Clamp::Command
   # the install logic below support installing multiple plugins with each a version specification
   # but the argument parsing does not support it for now so currently if specifying --version only
   # one plugin name can be also specified.
-  #
-  # TODO: find right syntax to allow specifying list of plugins with optional version specification for each
-
   def execute
-    if development?
-      raise(LogStash::PluginManager::Error, "Cannot specify plugin(s) with --development, it will add the development dependencies of the currently installed plugins") unless plugin_list.empty?
+    validate_cli_options!
+    
+    if local_gems?
+      gems = extract_local_gems_plugins
+    elsif development?
+      gems = plugins_development_gems
     else
-      raise(LogStash::PluginManager::Error, "No plugin specified") if plugin_list.empty? && verify?
-
-      # temporary until we fullfil TODO ^^
-      raise(LogStash::PluginManager::Error, "Only 1 plugin name can be specified with --version") if version && plugin_list.size > 1
+      gems = plugins_gems
+      verify_remote!(gems) if verify?
     end
-    raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
-
-    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
-    # keep a copy of the gemset to revert on error
-    original_gemset = gemfile.gemset.copy
 
-    # force Rubygems sources to our Gemfile sources
-    Gem.sources = gemfile.gemset.sources
-
-    # install_list will be an array of [plugin name, version] tuples, version can be nil
-    install_list = []
+    install_gems_list!(gems)
+    remove_unused_locally_installed_gems! 
+  end
 
+  private
+  def validate_cli_options!
     if development?
-      specs = LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile)
-      install_list = specs.inject([]) do |result, spec|
-        result = result + spec.dependencies.select{|dep| dep.type == :development}.map{|dep| [dep.name] + dep.requirement.as_list + [{:group => :development}]}
-      end
+      signal_usage_error("Cannot specify plugin(s) with --development, it will add the development dependencies of the currently installed plugins") unless plugins_arg.empty?
     else
-      # at this point we know that plugin_list is not empty and if the --version is specified there is only one plugin in plugin_list
+      signal_usage_error("No plugin specified") if plugins_arg.empty? && verify?
+      # TODO: find right syntax to allow specifying list of plugins with optional version specification for each
+      signal_usage_error("Only 1 plugin name can be specified with --version") if version && plugins_arg.size > 1
+    end
+    signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless ::File.writable?(LogStash::Environment::GEMFILE_PATH)
+  end
 
-      install_list = version ? [plugin_list << version] : plugin_list.map{|plugin| [plugin, nil]}
+  # Check if the specified gems contains
+  # the logstash `metadata`
+  def verify_remote!(gems)
+    gems.each do |plugin, version|
+      puts("Validating #{[plugin, version].compact.join("-")}")
+      signal_error("Installation aborted, verification failed for #{plugin} #{version}") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
+    end 
+  end
 
-      install_list.each do |plugin, version|
-        puts("Validating #{[plugin, version].compact.join("-")}")
-        raise(LogStash::PluginManager::Error, "Installation aborted") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
-      end if verify?
+  def plugins_development_gems
+    # Get currently defined gems and their dev dependencies
+    specs = []
 
-      # at this point we know that we either have a valid gem name & version or a valid .gem file path
+    specs = LogStash::PluginManager.all_installed_plugins_gem_specs(gemfile)
 
-      # if LogStash::PluginManager.plugin_file?(plugin)
-      #   raise(LogStash::PluginManager::Error) unless cache_gem_file(plugin)
-      #   spec = LogStash::PluginManager.plugin_file_spec(plugin)
-      #   gemfile.update(spec.name, spec.version.to_s)
-      # else
-      #   plugins.each{|tuple| gemfile.update(*tuple)}
-      # end
+    # Construct the list of dependencies to add to the current gemfile
+    specs.each_with_object([]) do |spec, install_list|
+      dependencies = spec.dependencies 
+        .select { |dep| dep.type == :development }
+        .map { |dep| [dep.name] + dep.requirement.as_list }
+
+      install_list.concat(dependencies)
     end
+  end
 
+  def plugins_gems
+    version ? [plugins_arg << version] : plugins_arg.map { |plugin| [plugin, nil] }
+  end
 
+  # install_list will be an array of [plugin name, version, options] tuples, version it
+  # can be nil at this point we know that plugins_arg is not empty and if the
+  # --version is specified there is only one plugin in plugins_arg
+  #
+  def install_gems_list!(install_list)
+    # If something goes wrong during the installation `LogStash::Gemfile` will restore a backup version.
     install_list = LogStash::PluginManager.merge_duplicates(install_list)
-    install_list.each{|plugin, version| gemfile.update(plugin, version)}
-    gemfile.save
 
-    puts("Installing" + (install_list.empty? ? "..." : " " + install_list.map{|plugin, version| plugin}.join(", ")))
+    # Add plugins/gems to the current gemfile
+    puts("Installing" + (install_list.empty? ? "..." : " " + install_list.collect(&:first).join(", ")))
+    install_list.each { |plugin, version, options| gemfile.update(plugin, version, options) }
+
+    # Sync gemfiles changes to disk to make them available to the `bundler install`'s API
+    gemfile.save
 
     bundler_options = {:install => true}
     bundler_options[:without] = [] if development?
+    bundler_options[:rubygems_source] = gemfile.gemset.sources
 
-    # any errors will be logged to $stderr by invoke_bundler!
-    output, exception = LogStash::Bundler.invoke_bundler!(bundler_options)
+    output = LogStash::Bundler.invoke_bundler!(bundler_options)
 
-    if ENV["DEBUG"]
-      $stderr.puts(output)
-      $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
-    end
+    puts("Installation successful")
+  rescue => exception
+    gemfile.restore!
+    report_exception("Installation Aborted", exception)
+  ensure
+    display_bundler_output(output)
+  end
 
-    if exception
-      # revert to original Gemfile content
-      gemfile.gemset = original_gemset
-      gemfile.save
-      raise(LogStash::PluginManager::Error, "Installation aborted")
-    end
+  # Extract the specified local gems in a predefined local path 
+  # Update the gemfile to use a relative path to this plugin and run
+  # Bundler, this will mark the gem not updatable by `bin/plugin update`
+  # This is the most reliable way to make it work in bundler without 
+  # hacking with `how bundler works`
+  #
+  # Bundler 2.0, will have support for plugins source we could create a .gem source
+  # to support it.
+  def extract_local_gems_plugins
+    plugins_arg.collect do |plugin| 
+      # We do the verify before extracting the gem so we dont have to deal with unused path
+      if verify?
+        puts("Validating #{plugin}")
+        signal_error("Installation aborted, verification failed for #{plugin}") unless LogStash::PluginManager.logstash_plugin?(plugin, version)
+      end
 
-    puts("Installation successful")
+      package, path = LogStash::Bundler.unpack(plugin, LogStash::Environment::LOCAL_GEM_PATH)
+      [package.spec.name, package.spec.version, { :path => relative_path(path) }]
+    end
   end
 
-  # copy .gem file into bundler cache directory, log any error to $stderr
-  # @param path [String] the source .gem file to copy
-  # @return [Boolean] true if successful
-  def cache_gem_file(path)
-    dest = ::File.join(LogStash::Environment.logstash_gem_home, "cache")
-    begin
-      FileUtils.cp(path, dest)
-    rescue => e
-      $stderr.puts("Error copying #{plugin} to #{dest}, caused by #{e.class}")
-      return false
+  # We cannot install both .gem and normal plugin in one call of `plugin install`
+  def local_gems?
+    return false if plugins_arg.empty?
+
+    local_gem = plugins_arg.collect { |plugin| ::File.extname(plugin) == ".gem" }.uniq
+
+    if local_gem.size == 1
+      return local_gem.first
+    else
+      signal_usage_error("Mixed source of plugins, you can't mix local `.gem` and remote gems")
     end
-    true
   end
 end # class Logstash::PluginManager

data/lib/logstash/pluginmanager/list.rb

@@ -1,9 +1,11 @@
 require 'clamp'
 require 'logstash/namespace'
 require 'logstash/pluginmanager/util'
+require 'logstash/pluginmanager/command'
+require "logstash/bundler"
 require 'rubygems/spec_fetcher'
 
-class LogStash::PluginManager::List < Clamp::Command
+class LogStash::PluginManager::List < LogStash::PluginManager::Command
 
   parameter "[PLUGIN]", "Part of plugin name to search for, leave empty for all plugins"
 
@@ -15,27 +17,28 @@ class LogStash::PluginManager::List < Clamp::Command
   end
 
   def execute
-    require 'logstash/environment'
-    LogStash::Environment.bundler_setup!
+    LogStash::Bundler.setup!
 
-    Gem.configuration.verbose = false
+    signal_error("No plugins found") if filtered_specs.empty?
 
-    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
-
-    # start with all locally installed plugin gems regardless of the Gemfile content
-    specs = LogStash::PluginManager.find_plugins_gem_specs
-
-    # apply filters
-    specs = specs.select{|spec| gemfile.find(spec.name)} if installed?
-    specs = specs.select{|spec| spec.name =~ /#{plugin}/i} if plugin
-    specs = specs.select{|spec| spec.metadata['logstash_group'] == group} if group
-
-    raise(LogStash::PluginManager::Error, "No plugins found") if specs.empty?
-
-    specs.sort_by{|spec| spec.name}.each do |spec|
+    filtered_specs.sort_by{|spec| spec.name}.each do |spec|
       line = "#{spec.name}"
       line += " (#{spec.version})" if verbose?
       puts(line)
     end
   end
+
+  def filtered_specs
+    @filtered_specs ||= begin
+                          # start with all locally installed plugin gems regardless of the Gemfile content
+                          specs = LogStash::PluginManager.find_plugins_gem_specs
+
+                          # apply filters
+                          specs = specs.select{|spec| gemfile.find(spec.name)} if installed?
+                          specs = specs.select{|spec| spec.name =~ /#{plugin}/i} if plugin
+                          specs = specs.select{|spec| spec.metadata['logstash_group'] == group} if group
+
+                          specs
+                        end
+  end
 end # class Logstash::PluginManager

data/lib/logstash/pluginmanager/main.rb

@@ -5,7 +5,6 @@ require "logstash/pluginmanager/uninstall"
 require "logstash/pluginmanager/list"
 require "logstash/pluginmanager/update"
 require "logstash/pluginmanager/util"
-require "logstash/pluginmanager/maven_tools_patch"
 require "clamp"
 
 module LogStash

data/lib/logstash/pluginmanager/uninstall.rb

@@ -3,25 +3,21 @@ require "logstash/logging"
 require "logstash/errors"
 require "logstash/environment"
 require "logstash/pluginmanager/util"
+require "logstash/pluginmanager/command"
 require "clamp"
 
 require "logstash/gemfile"
 require "logstash/bundler"
 
-class LogStash::PluginManager::Uninstall < Clamp::Command
+class LogStash::PluginManager::Uninstall < LogStash::PluginManager::Command
   parameter "PLUGIN", "plugin name"
 
-
   def execute
-    raise(LogStash::PluginManager::Error, "File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
-
-    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
-    # keep a copy of the gemset to revert on error
-    original_gemset = gemfile.gemset.copy
+    signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
 
     # make sure this is an installed plugin and present in Gemfile.
     # it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
-    raise(LogStash::PluginManager::Error, "This plugin has not been previously installed, aborting") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
+    signal_error("This plugin has not been previously installed, aborting") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
 
     # since we previously did a gemfile.find(plugin) there is no reason why
     # remove would not work (return nil) here
@@ -31,19 +27,15 @@ class LogStash::PluginManager::Uninstall < Clamp::Command
       puts("Uninstalling #{plugin}")
 
       # any errors will be logged to $stderr by invoke_bundler!
-      output, exception = LogStash::Bundler.invoke_bundler!(:install => true, :clean => true)
-
-      if ENV["DEBUG"]
-        $stderr.puts(output)
-        $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
-      end
-
-      if exception
-        # revert to original Gemfile content
-        gemfile.gemset = original_gemset
-        gemfile.save
-        raise(LogStash::PluginManager::Error, "Uninstall aborted")
-      end
+      # output, exception = LogStash::Bundler.invoke_bundler!(:install => true, :clean => true)
+      output = LogStash::Bundler.invoke_bundler!(:install => true, :clean => true)
+
+      remove_unused_locally_installed_gems!
     end
+  rescue => exception
+    gemfile.restore!
+    report_exception("Uninstall Aborted", exception)
+  ensure
+    display_bundler_output(output)
   end
 end

data/lib/logstash/pluginmanager/update.rb

@@ -1,51 +1,80 @@
-require 'clamp'
-require 'logstash/namespace'
-require 'logstash/pluginmanager/util'
-require 'jar-dependencies'
-require 'jar_install_post_install_hook'
-require 'file-dependencies/gem'
-
+require "clamp"
+require "logstash/namespace"
+require "logstash/pluginmanager/util"
+require "logstash/pluginmanager/command"
+require "jar-dependencies"
+require "jar_install_post_install_hook"
+require "file-dependencies/gem"
 require "logstash/gemfile"
 require "logstash/bundler"
 
-class LogStash::PluginManager::Update < Clamp::Command
-  parameter "[PLUGIN] ...", "Plugin name(s) to upgrade to latest version"
+class LogStash::PluginManager::Update < LogStash::PluginManager::Command
+  parameter "[PLUGIN] ...", "Plugin name(s) to upgrade to latest version", :attribute_name => :plugins_arg
 
   def execute
-    gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
-    # keep a copy of the gemset to revert on error
-    original_gemset = gemfile.gemset.copy
-
-    previous_gem_specs_map = find_latest_gem_specs
+    local_gems = gemfile.locally_installed_gems
 
-    # create list of plugins to update
-    plugins = unless plugin_list.empty?
-      not_installed = plugin_list.select{|plugin| !previous_gem_specs_map.has_key?(plugin.downcase)}
-      raise(LogStash::PluginManager::Error, "Plugin #{not_installed.join(', ')} is not installed so it cannot be updated, aborting") unless not_installed.empty?
-      plugin_list
+    if update_all? && !local_gems.empty?
+      error_plugin_that_use_path!(local_gems)
     else
-      previous_gem_specs_map.values.map{|spec| spec.name}
+      plugins_with_path = plugins_arg & local_gems
+      error_plugin_that_use_path!(plugins_with_path) if plugins_with_path.size > 0
     end
 
+    update_gems!
+  end
+
+  private
+  def error_plugin_that_use_path!(plugins)
+    signal_error("Update is not supported for manually defined plugins or local .gem plugin installations: #{plugins.collect(&:name).join(",")}")
+  end
+
+  def update_all?
+    plugins_arg.size == 0
+  end
+
+  def update_gems!
+    # If any error is raise inside the block the Gemfile will restore a backup of the Gemfile
+    previous_gem_specs_map = find_latest_gem_specs
+
     # remove any version constrain from the Gemfile so the plugin(s) can be updated to latest version
     # calling update without requiremend will remove any previous requirements
-    plugins.select{|plugin| gemfile.find(plugin)}.each{|plugin| gemfile.update(plugin)}
+    plugins = plugins_to_update(previous_gem_specs_map)
+    plugins
+      .select { |plugin| gemfile.find(plugin) }
+      .each { |plugin| gemfile.update(plugin) }
+
+    # force a disk sync before running bundler
     gemfile.save
 
     puts("Updating " + plugins.join(", "))
 
     # any errors will be logged to $stderr by invoke_bundler!
-    output, exception = LogStash::Bundler.invoke_bundler!(:update => plugins)
-    output, exception = LogStash::Bundler.invoke_bundler!(:clean => true) unless exception
+    # Bundler cannot update and clean gems in one operation so we have to call the CLI twice.
+    output = LogStash::Bundler.invoke_bundler!(:update => plugins)
+    output = LogStash::Bundler.invoke_bundler!(:clean => true)
 
-    if exception
-      # revert to original Gemfile content
-      gemfile.gemset = original_gemset
-      gemfile.save
+    display_updated_plugins(previous_gem_specs_map)
+  rescue => exception
+    gemfile.restore!
+    report_exception("Updated Aborted", exception)
+  ensure
+    display_bundler_output(output)
+  end
 
-      report_exception(output, exception)
+  # create list of plugins to update
+  def plugins_to_update(previous_gem_specs_map)
+    if update_all?
+      previous_gem_specs_map.values.map{|spec| spec.name}
+    else
+      not_installed = plugins_arg.select{|plugin| !previous_gem_specs_map.has_key?(plugin.downcase)}
+      signal_error("Plugin #{not_installed.join(', ')} is not installed so it cannot be updated, aborting") unless not_installed.empty?
+      plugins_arg
     end
+  end
 
+  # We compare the before the update and after the update
+  def display_updated_plugins(previous_gem_specs_map)
     update_count = 0
     find_latest_gem_specs.values.each do |spec|
       name = spec.name.downcase
@@ -59,11 +88,10 @@ class LogStash::PluginManager::Update < Clamp::Command
         update_count += 1
       end
     end
+
     puts("No plugin updated") if update_count.zero?
   end
 
-  private
-
   # retrieve only the latest spec for all locally installed plugins
   # @return [Hash] result hash {plugin_name.downcase => plugin_spec}
   def find_latest_gem_specs
@@ -73,13 +101,4 @@ class LogStash::PluginManager::Update < Clamp::Command
       result
     end
   end
-
-  def report_exception(output, exception)
-    if ENV["DEBUG"]
-      $stderr.puts(output)
-      $stderr.puts("Error: #{exception.class}, #{exception.message}") if exception
-    end
-
-    raise(LogStash::PluginManager::Error, "Update aborted")
-  end
 end