logstash-core 5.0.2-java → 5.1.1.1-java

data/lib/logstash/plugin.rb

@@ -5,7 +5,6 @@ require "logstash/config/mixin"
 require "logstash/instrument/null_metric"
 require "concurrent"
 require "securerandom"
-require "logstash/plugins/registry"
 
 class LogStash::Plugin
   include LogStash::Util::Loggable
@@ -39,13 +38,18 @@ class LogStash::Plugin
     self.class.name.hash
   end
 
-
   def eql?(other)
     self.class.name == other.class.name && @params == other.params
   end
 
   def initialize(params=nil)
     @logger = self.logger
+    # need to access settings statically because plugins are initialized in config_ast with no context.
+    settings = LogStash::SETTINGS
+    @slow_logger = self.slow_logger(settings.get("slowlog.threshold.warn"),
+                                    settings.get("slowlog.threshold.info"),
+                                    settings.get("slowlog.threshold.debug"),
+                                    settings.get("slowlog.threshold.trace"))
     @params = LogStash::Util.deep_clone(params)
     # The id should always be defined normally, but in tests that might not be the case
     # In the future we may make this more strict in the Plugin API
@@ -103,9 +107,9 @@ class LogStash::Plugin
     # we will use the NullMetric in this case.
     @metric_plugin ||= if @enable_metric
                          # Fallback when testing plugin and no metric collector are correctly configured.
-                         @metric.nil? ? LogStash::Instrument::NullMetric.new : @metric
+                         @metric.nil? ? LogStash::Instrument::NamespacedNullMetric.new : @metric
                        else
-                         LogStash::Instrument::NullMetric.new
+                         LogStash::Instrument::NamespacedNullMetric.new(@metric, :null)
                        end
   end
   # return the configured name of this plugin
@@ -114,50 +118,12 @@ class LogStash::Plugin
     self.class.config_name
   end
 
-
-  # Look up a plugin by type and name.
+  # This is keep for backward compatibility, the logic was moved into the registry class
+  # but some plugins use this method to return a specific instance on lookup
+  #
+  # Should I remove this now and make sure the pipeline invoke the Registry or I should wait for 6.0
+  # Its not really part of the public api but its used by the tests a lot to mock the plugins.
   def self.lookup(type, name)
-    path = "logstash/#{type}s/#{name}"
-    LogStash::Registry.instance.lookup(type ,name) do |plugin_klass, plugin_name|
-      is_a_plugin?(plugin_klass, plugin_name)
-    end
-    
-  rescue LoadError, NameError => e
-    logger.debug("Problems loading the plugin with", :type => type, :name => name, :path => path)
-    raise(LogStash::PluginLoadingError, I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :path => path, :error => e.to_s))
-  end
-
-  public
-  def self.declare_plugin(type, name)
-    path = "logstash/#{type}s/#{name}"
-    registry = LogStash::Registry.instance
-    registry.register(path, self)
-  end
-
-  private
-  # lookup a plugin by type and name in the existing LogStash module namespace
-  # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
-  # @param type [String] plugin type, "input", "ouput", "filter"
-  # @param name [String] plugin name, ex.: "grok"
-  # @return [Class] the plugin class or raises NameError
-  # @raise NameError if plugin class does not exist or is invalid
-  def self.namespace_lookup(type, name)
-    type_const = "#{type.capitalize}s"
-    namespace = LogStash.const_get(type_const)
-    # the namespace can contain constants which are not for plugins classes (do not respond to :config_name)
-    # namespace.constants is the shallow collection of all constants symbols in namespace
-    # note that below namespace.const_get(c) should never result in a NameError since c is from the constants collection
-    klass_sym = namespace.constants.find { |c| is_a_plugin?(namespace.const_get(c), name) }
-    klass = klass_sym && namespace.const_get(klass_sym)
-    raise(NameError) unless klass
-    klass
-  end
-
-  # check if klass is a valid plugin for name
-  # @param klass [Class] plugin class
-  # @param name [String] plugin name
-  # @return [Boolean] true if klass is a valid plugin for name
-  def self.is_a_plugin?(klass, name)
-    klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
+    LogStash::PLUGIN_REGISTRY.lookup_pipeline_plugin(type, name)
   end
 end # class LogStash::Plugin

data/lib/logstash/plugins/hooks_registry.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+module LogStash module Plugins
+  # This calls allow logstash to expose the endpoints for listeners
+  class HooksRegistry
+    java_import "java.util.concurrent.ConcurrentHashMap"
+    java_import "java.util.concurrent.CopyOnWriteArrayList"
+
+    def initialize
+      @registered_emmitters = ConcurrentHashMap.new
+      @registered_hooks = ConcurrentHashMap.new
+    end
+
+    def register_emitter(emitter_scope, dispatcher)
+      @registered_emmitters.put(emitter_scope, dispatcher)
+      sync_hooks
+    end
+
+    def remove_emitter(emitter_scope)
+      @registered_emmitters.remove(emitter_scope)
+    end
+
+    def register_hooks(emitter_scope, callback)
+      callbacks = @registered_hooks.computeIfAbsent(emitter_scope) do
+        CopyOnWriteArrayList.new
+      end
+
+      callbacks.add(callback)
+      sync_hooks
+    end
+
+    def emmitters_count
+      @registered_emmitters.size
+    end
+
+    def hooks_count(emitter_scope = nil)
+      if emitter_scope.nil?
+        @registered_hooks.elements().collect(&:size).reduce(0, :+)
+      else
+        callbacks = @registered_hooks.get(emitter_scope)
+        callbacks.nil? ? 0 : @registered_hooks.get(emitter_scope).size
+      end
+    end
+
+    private
+    def sync_hooks
+      @registered_emmitters.each do |emitter, dispatcher|
+        listeners = @registered_hooks.get(emitter)
+
+        unless listeners.nil?
+          listeners.each do |listener|
+            dispatcher.add_listener(listener)
+          end
+        end
+      end
+    end
+  end
+end end

data/lib/logstash/plugins/registry.rb

@@ -1,85 +1,248 @@
 # encoding: utf-8
-require 'singleton'
 require "rubygems/package"
 require "logstash/util/loggable"
+require "logstash/plugin"
+require "logstash/plugins/hooks_registry"
 
-module LogStash
+module LogStash module Plugins
   class Registry
     include LogStash::Util::Loggable
 
-    ##
-    # Placeholder class for registered plugins
-    ##
-    class Plugin
-      attr_reader :type, :name
+    # Add a bit more sanity with when interacting with the rubygems'
+    # specifications database, most of out code interact directly with really low level
+    # components of bundler/rubygems we need to encapsulate that and this is a start.
+    class GemRegistry
+      LOGSTASH_METADATA_KEY = "logstash_plugin"
 
-      def initialize(type, name)
-        @type  = type
-        @name  = name
+      class << self
+        def installed_gems
+          ::Gem::Specification
+        end
+
+        def logstash_plugins
+          installed_gems
+            .select { |spec| spec.metadata && spec.metadata[LOGSTASH_METADATA_KEY] }
+            .collect { |spec| PluginRawContext.new(spec) }
+        end
+      end
+    end
+
+    class PluginRawContext
+      HOOK_FILE = "logstash_registry.rb"
+      NAME_DELIMITER = "-"
+
+      attr_reader :spec
+
+      def initialize(spec)
+        @spec = spec
+        @destructured_name = spec.name.split(NAME_DELIMITER)
+      end
+
+      def name
+        @destructured_name[2..-1].join(NAME_DELIMITER)
+      end
+
+      def type
+        @destructured_name[1]
       end
 
-      def path
-        "logstash/#{type}s/#{name}"
+      # In the context of the plugin, the hook file available  need to exist in any top level
+      # required paths.
+      #
+      # Example for the logstash-output-elasticsearch we have this line in the gemspec.
+      #
+      # s.require_paths = ["lib"], so the we will expect to have a `logstash_registry.rb` file in the `lib`
+      # directory.
+      def hooks_file
+        @hook_file ||= spec.full_require_paths.collect do |path|
+          f = ::File.join(path, HOOK_FILE)
+          ::File.exist?(f) ? f : nil
+        end.compact.first
       end
 
-      def cannonic_gem_name
-        "logstash-#{type}-#{name}"
+      def has_hooks?
+        !hooks_file.nil?
       end
 
-      def installed?
-        find_plugin_spec(cannonic_gem_name).any?
+      def execute_hooks!
+        require hooks_file
       end
+    end
 
-      private
+    class PluginSpecification
+      attr_reader :type, :name, :klass
 
-      def find_plugin_spec(name)
-        specs = ::Gem::Specification.find_all_by_name(name)
-        specs.select{|spec| logstash_plugin_spec?(spec)}
+      def initialize(type, name, klass)
+        @type  = type.to_sym
+        @name  = name
+        @klass = klass
       end
+    end
 
-      def logstash_plugin_spec?(spec)
-        spec.metadata && spec.metadata["logstash_plugin"] == "true"
+    class UniversalPluginSpecification < PluginSpecification
+      def initialize(type, name, klass)
+        super(type, name, klass)
+        @instance = klass.new
       end
 
+      def register(hooks, settings)
+        @instance.register_hooks(hooks)
+        @instance.additionals_settings(settings)
+      end
     end
 
-    include Singleton
+    attr_reader :hooks
 
     def initialize
       @registry = {}
-      @logger = self.logger
+      @hooks = HooksRegistry.new
+    end
+
+    def setup!
+      load_available_plugins
+      execute_universal_plugins
+    end
+
+    def execute_universal_plugins
+      @registry.values
+        .select { |specification| specification.is_a?(UniversalPluginSpecification) }
+        .each { |specification| specification.register(hooks, LogStash::SETTINGS) }
+    end
+
+    def load_available_plugins
+      GemRegistry.logstash_plugins.each do |plugin_context|
+        # When a plugin has a HOOK_FILE defined, its the responsability of the plugin
+        # to register itself to the registry of available plugins.
+        #
+        # Legacy plugin will lazy register themselves
+        if plugin_context.has_hooks?
+          begin
+            logger.debug("Executing hooks", :name => plugin_context.name, :type => plugin_context.type, :hooks_file => plugin_context.hooks_file)
+            plugin_context.execute_hooks!
+          rescue => e
+            logger.error("error occured when loading plugins hooks file", :name => plugin_context.name, :type => plugin_context.type, :exception => e.message, :stacktrace => e.backtrace )
+          end
+        end
+      end
     end
 
     def lookup(type, plugin_name, &block)
+      plugin = get(type, plugin_name)
+      # Assume that we have a legacy plugin
+      if plugin.nil?
+        plugin = legacy_lookup(type, plugin_name)
+      end
 
-      plugin = Plugin.new(type, plugin_name)
+      if block_given? # if provided pass a block to do validation
+        raise LoadError, "Block validation fails for plugin named #{plugin_name} of type #{type}," unless block.call(plugin.klass, plugin_name)
+      end
+
+      return plugin.klass
+    end
 
-      if plugin.installed?
-        return @registry[plugin.path] if registered?(plugin.path)
-        require plugin.path
-        klass = @registry[plugin.path]
-        if block_given? # if provided pass a block to do validation
-          raise LoadError unless block.call(klass, plugin_name)
+    # The legacy_lookup method uses the 1.5->5.0 file structure to find and match
+    # a plugin and will do a lookup on the namespace of the required class to find a matching
+    # plugin with the appropriate type.
+    def legacy_lookup(type, plugin_name)
+      begin
+        path = "logstash/#{type}s/#{plugin_name}"
+
+        begin
+          require path
+        rescue LoadError
+          # Plugin might be already defined in the current scope
+          # This scenario often happen in test when we write an adhoc class
         end
-        return klass
-      else
-        # The plugin was defined directly in the code, so there is no need to use the
-        # require way of loading classes
-        return @registry[plugin.path] if registered?(plugin.path)
-        raise LoadError
+
+        klass = namespace_lookup(type, plugin_name)
+        plugin = lazy_add(type, plugin_name, klass)
+      rescue => e
+        logger.error("Problems loading a plugin with",
+                    :type => type,
+                    :name => plugin_name,
+                    :path => path,
+                    :error_message => e.message,
+                    :error_class => e.class,
+                    :error_backtrace => e.backtrace)
+
+        raise LoadError, "Problems loading the requested plugin named #{plugin_name} of type #{type}. Error: #{e.class} #{e.message}"
       end
-    rescue => e
-      @logger.debug("Problems loading a plugin with", :type => type, :name => plugin, :path => plugin.path, :error => e) if @logger.debug?
-      raise LoadError, "Problems loading the requested plugin named #{plugin_name} of type #{type}."
+
+      plugin
+    end
+
+    def lookup_pipeline_plugin(type, name)
+      LogStash::PLUGIN_REGISTRY.lookup(type, name) do |plugin_klass, plugin_name|
+        is_a_plugin?(plugin_klass, plugin_name)
+      end
+    rescue LoadError, NameError => e
+      logger.debug("Problems loading the plugin with", :type => type, :name => name)
+      raise(LogStash::PluginLoadingError, I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :error => e.to_s))
+    end
+
+    def lazy_add(type, name, klass)
+      logger.debug("On demand adding plugin to the registry", :name => name, :type => type, :class => klass)
+      add_plugin(type, name, klass)
+    end
+
+    def add(type, name, klass)
+      logger.debug("Adding plugin to the registry", :name => name, :type => type, :class => klass)
+      add_plugin(type, name, klass)
     end
 
-    def register(path, klass)
-      @registry[path] = klass
+    def get(type, plugin_name)
+      @registry[key_for(type, plugin_name)]
+    end
+
+    def exists?(type, name)
+      @registry.include?(key_for(type, name))
+    end
+
+    def size
+      @registry.size
+    end
+
+    private
+    # lookup a plugin by type and name in the existing LogStash module namespace
+    # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
+    # @param type [String] plugin type, "input", "ouput", "filter"
+    # @param name [String] plugin name, ex.: "grok"
+    # @return [Class] the plugin class or raises NameError
+    # @raise NameError if plugin class does not exist or is invalid
+    def namespace_lookup(type, name)
+      type_const = "#{type.capitalize}s"
+      namespace = LogStash.const_get(type_const)
+      # the namespace can contain constants which are not for plugins classes (do not respond to :config_name)
+      # namespace.constants is the shallow collection of all constants symbols in namespace
+      # note that below namespace.const_get(c) should never result in a NameError since c is from the constants collection
+      klass_sym = namespace.constants.find { |c| is_a_plugin?(namespace.const_get(c), name) }
+      klass = klass_sym && namespace.const_get(klass_sym)
+
+      raise(NameError) unless klass
+      klass
+    end
+
+    # check if klass is a valid plugin for name
+    # @param klass [Class] plugin class
+    # @param name [String] plugin name
+    # @return [Boolean] true if klass is a valid plugin for name
+    def is_a_plugin?(klass, name)
+      klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
+    end
+
+    def add_plugin(type, name, klass)
+      if !exists?(type, name)
+        specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
+        @registry[key_for(type, name)] = specification_klass.new(type, name, klass)
+      else
+        logger.debug("Ignoring, plugin already added to the registry", :name => name, :type => type, :klass => klass)
+      end
     end
 
-    def registered?(path)
-      @registry.has_key?(path)
+    def key_for(type, plugin_name)
+      "#{type}-#{plugin_name}"
     end
+  end end
 
-  end
+  PLUGIN_REGISTRY = Plugins::Registry.new
 end

data/lib/logstash/runner.rb

@@ -5,17 +5,20 @@ $DEBUGLIST = (ENV["DEBUG"] || "").split(",")
 
 require "clamp"
 require "net/http"
+
+require "logstash/namespace"
+require "logstash-core/logstash-core"
 require "logstash/environment"
 
 LogStash::Environment.load_locale!
 
-require "logstash/namespace"
 require "logstash/agent"
 require "logstash/config/defaults"
 require "logstash/shutdown_watcher"
 require "logstash/patches/clamp"
 require "logstash/settings"
 require "logstash/version"
+require "logstash/plugins/registry"
 
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
@@ -28,7 +31,7 @@ class LogStash::Runner < Clamp::StrictCommand
 
   # Node Settings
   option ["-n", "--node.name"], "NAME",
-    I18n.t("logstash.runner.flag.node_name"),
+    I18n.t("logstash.runner.flag.name"),
     :attribute_name => "node.name",
     :default => LogStash::SETTINGS.get_default("node.name")
 
@@ -196,6 +199,11 @@ class LogStash::Runner < Clamp::StrictCommand
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
 
+    # We configure the registry and load any plugin that can register hooks
+    # with logstash, this need to be done before any operation.
+    LogStash::PLUGIN_REGISTRY.setup!
+    @settings.validate_all
+
     LogStash::Util::set_thread_name(self.class.name)
 
     if RUBY_VERSION < "1.9.2"
@@ -221,8 +229,6 @@ class LogStash::Runner < Clamp::StrictCommand
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
 
-    @settings.validate_all
-
     @settings.format_settings.each {|line| logger.debug(line) }
 
     if setting("config.string").nil? && setting("path.config").nil?
@@ -335,7 +341,6 @@ class LogStash::Runner < Clamp::StrictCommand
     LogStash::Agent.new(*args)
   end
 
-
   # Emit a failure message and abort.
   def fail(message)
     signal_usage_error(message)