logstash-core 7.1.1-java → 7.2.0-java

data/lib/logstash/outputs/base.rb

@@ -102,6 +102,13 @@ class LogStash::Outputs::Base < LogStash::Plugin
     self.class.concurrency
   end
 
+  def metric=(metric)
+    super
+    # Hack to create a new metric namespace using 'plugins' as the root
+    @codec.metric = metric.root.namespace(metric.namespace_name[0...-2].push(:codecs, codec.id))
+    metric
+  end
+
   def execution_context=(context)
     super
     # There is no easy way to propage an instance variable into the codec, because the codec

data/lib/logstash/pipeline.rb

@@ -394,6 +394,14 @@ module LogStash; class Pipeline < BasePipeline
     filter_queue_client.add_output_metrics(batch.filtered_size)
   end
 
+  def resolve_cluster_uuids
+    outputs.each_with_object(Set.new) do |output, cluster_uuids|
+      if LogStash::PluginMetadata.exists?(output.id)
+        cluster_uuids << LogStash::PluginMetadata.for_plugin(output.id).get(:cluster_uuid)
+      end
+    end.to_a.compact
+  end
+
   def wait_inputs
     @input_threads.each(&:join)
   end
@@ -442,9 +450,14 @@ module LogStash; class Pipeline < BasePipeline
       # Assuming the failure that caused this exception is transient,
       # let's sleep for a bit and execute #run again
       sleep(1)
+      begin
+        plugin.do_close
+      rescue => close_exception
+        @logger.debug("Input plugin raised exception while closing, ignoring",
+                      default_logging_keys(:plugin => plugin.class.config_name, :exception => close_exception.message,
+                                           :backtrace => close_exception.backtrace))
+      end
       retry
-    ensure
-      plugin.do_close
     end
   end # def inputworker
 

data/lib/logstash/pipeline_settings.rb

@@ -15,6 +15,7 @@ module LogStash
       "dead_letter_queue.max_bytes",
       "metric.collect",
       "pipeline.java_execution",
+      "pipeline.plugin_classloaders",
       "path.config",
       "path.dead_letter_queue",
       "path.queue",

data/lib/logstash/plugin.rb

@@ -3,6 +3,8 @@ require "logstash/config/mixin"
 require "concurrent"
 require "securerandom"
 
+require_relative 'plugin_metadata'
+
 class LogStash::Plugin
   include LogStash::Util::Loggable
 
@@ -70,7 +72,11 @@ class LogStash::Plugin
   # main task terminates
   def do_close
     @logger.debug("Closing", :plugin => self.class.name)
-    close
+    begin
+      close
+    ensure
+      LogStash::PluginMetadata.delete_for_plugin(self.id)
+    end
   end
 
   # Subclasses should implement this close method if you need to perform any
@@ -136,4 +142,22 @@ class LogStash::Plugin
     require "logstash/plugins/registry"
     LogStash::PLUGIN_REGISTRY.lookup_pipeline_plugin(type, name)
   end
+
+  ##
+  # Returns this plugin's metadata key/value store.
+  #
+  # @see LogStash::PluginMetadata for restrictions and caveats.
+  # @since 7.1
+  #
+  # @usage:
+  # ~~~
+  # if defined?(plugin_metadata)
+  #   plugin_metadata.set(:foo, 'value')
+  # end
+  # ~~~
+  #
+  # @return [LogStash::PluginMetadata]
+  def plugin_metadata
+    LogStash::PluginMetadata.for_plugin(self.id)
+  end
 end # class LogStash::Plugin

data/lib/logstash/plugin_metadata.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+
+require 'thread_safe/cache'
+
+module LogStash
+  ##
+  # `PluginMetadata` provides a space to store key/value metadata about a plugin, typically metadata about
+  # external resources that can be gleaned during plugin registration.
+  #
+  # Data should not be persisted across pipeline reloads, and should be cleaned up after a pipeline reload
+  #
+  #  - It MUST NOT be used to store processing state
+  #  - It SHOULD NOT be updated frequently.
+  #  - Individual metadata keys MUST be Symbols and SHOULD NOT be dynamically generated
+  #
+  # USAGE FROM PLUGINS
+  # ------------------
+  #
+  # Since we allow plugins to be updated, it is important to introduce bindings to new Logstash features in a way
+  # that doesn't break when installed onto a Logstash that doesn't have those features, e.g.:
+  #
+  # ~~~
+  #
+  # plugin_metadata.set(:foo, bar) if defined?(plugin_metadata?)
+  #
+  # ~~~
+  #
+  # @since 7.1
+  class PluginMetadata
+    include LogStash::Util::Loggable
+
+    Thread.exclusive do
+      @registry = ThreadSafe::Cache.new unless defined?(@registry)
+    end
+
+    class << self
+      ##
+      # Get the PluginMetadata object corresponding to the given plugin id
+      #
+      # @param plugin_id [String]
+      #
+      # @return [PluginMetadata]: the metadata object for the provided `plugin_id`; if no
+      #                           metadata object exists, it will be created.
+      def for_plugin(plugin_id)
+        @registry.compute_if_absent(plugin_id) { PluginMetadata.new }
+      end
+
+      ##
+      # Determine if we have an existing PluginMetadata object for the given plugin id
+      # This allows us to avoid creating a metadata object if we don't already have one.
+      #
+      # @param plugin_id [String]
+      #
+      # @return [Boolean]
+      def exists?(plugin_id)
+        @registry.key?(plugin_id)
+      end
+
+      ##
+      # Deletes, and then clears the contents of an existing PluginMetadata object for the given plugin id if one exists
+      #
+      # @param plugin_id [String]
+      #
+      # @return [Boolean]
+      def delete_for_plugin(plugin_id)
+        logger.debug("Removing metadata for plugin #{plugin_id}")
+        old_registry = @registry.delete(plugin_id)
+        old_registry.clear unless old_registry.nil?
+      end
+
+      ##
+      # @api private
+      def reset!
+        @registry.clear
+      end
+    end
+
+    ##
+    # @see [LogStash::PluginMetadata#for_plugin(String)]
+    # @api private
+    def initialize
+      @datastore = ThreadSafe::Cache.new
+    end
+
+    ##
+    # Set the metadata key for this plugin, returning the previous value (if any)
+    #
+    # @param key [Symbol]
+    # @param value [Object]
+    #
+    # @return [Object]
+    def set(key, value)
+      if value.nil?
+        @datastore.delete(key)
+      else
+        @datastore.get_and_set(key, value)
+      end
+    end
+
+    ##
+    # Get the metadata value for the given key on this plugin
+    #
+    # @param key [Symbol]
+    #
+    # @return [Object]: the value object associated with the given key on this
+    #                   plugin, or nil if no value is associated
+    def get(key)
+      @datastore.get(key)
+    end
+
+    ##
+    # Determine whether specific key/value metadata exists for this plugin
+    #
+    # @param key [Symbol]: the key
+    #
+    # @return [Boolean]: true if the plugin includes metadata for the key
+    def set?(key)
+      @datastore.key?(key)
+    end
+
+    ##
+    # Delete the metadata key for this plugin, returning the previous value (if any)
+    #
+    # @param key [Symbol]
+    #
+    # @return [Object]
+    def delete(key)
+      @datastore.delete(key)
+    end
+
+    ##
+    # Clear all metadata keys for this plugin
+    #
+    # @return [Object]
+    def clear
+      @datastore.clear
+    end
+  end
+end

data/lib/logstash/plugins/registry.rb

@@ -105,6 +105,7 @@ module LogStash module Plugins
       # but it is the case that we can call lookups from multiple threads,
       # when multiple pipelines are in play, and that a lookup may modify the registry.
       @registry = java.util.concurrent.ConcurrentHashMap.new
+      @java_plugins = java.util.concurrent.ConcurrentHashMap.new
       @hooks = HooksRegistry.new
     end
 
@@ -133,6 +134,15 @@ module LogStash module Plugins
       require "logstash/plugins/builtin"
 
       GemRegistry.logstash_plugins.each do |plugin_context|
+        if plugin_context.spec.metadata.key?('java_plugin')
+          jar_files = plugin_context.spec.files.select {|f| f =~ /.*\.jar/}
+          expected_jar_name = plugin_context.spec.name + "-" + plugin_context.spec.version.to_s + ".jar"
+          if (jar_files.length != 1 || !jar_files[0].end_with?(expected_jar_name))
+            raise LoadError, "Java plugin '#{plugin_context.spec.name}' does not contain a single jar file with the plugin's name and version"
+          end
+          @java_plugins[plugin_context.spec.name] = [plugin_context.spec.loaded_from, jar_files[0]]
+        end
+
         # When a plugin has a HOOK_FILE defined, its the responsibility of the plugin
         # to register itself to the registry of available plugins.
         #
@@ -262,13 +272,31 @@ module LogStash module Plugins
     # @param name [String] plugin name
     # @return [Boolean] true if klass is a valid plugin for name
     def is_a_plugin?(klass, name)
+      (klass.class == Java::JavaLang::Class && klass.simple_name.downcase == name.gsub('_','')) ||
       (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
       (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
     end
 
     def add_plugin(type, name, klass)
       if klass.respond_to?("javaClass", true)
-        @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
+        if LogStash::SETTINGS.get_value('pipeline.plugin_classloaders')
+          full_name = 'logstash-' + key_for(type, name)
+          if @java_plugins.key?(full_name)
+            plugin_paths = @java_plugins[full_name]
+          else
+            raise LoadError,  "Could not find metadata for Java plugin: #{full_name}"
+          end
+
+          java_import org.logstash.plugins.PluginClassLoader
+          java_import org.logstash.Logstash
+
+          classloader = PluginClassLoader.create(plugin_paths[0], plugin_paths[1], Logstash.java_class.class_loader)
+          klazz = classloader.load_class(klass.javaClass.name)
+
+          @registry[key_for(type, name)] = PluginSpecification.new(type, name, klazz.ruby_class.java_class)
+        else
+          @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
+        end
       elsif !exists?(type, name)
         specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
         @registry[key_for(type, name)] = specification_klass.new(type, name, klass)

data/lib/logstash/runner.rb

@@ -113,6 +113,11 @@ class LogStash::Runner < Clamp::StrictCommand
          :attribute_name => "pipeline.java_execution",
          :default => LogStash::SETTINGS.get_default("pipeline.java_execution")
 
+  option ["--plugin-classloaders"], :flag,
+         I18n.t("logstash.runner.flag.plugin-classloaders"),
+         :attribute_name => "pipeline.plugin_classloaders",
+         :default => LogStash::SETTINGS.get_default("pipeline.plugin_classloaders")
+
   option ["-b", "--pipeline.batch.size"], "SIZE",
     I18n.t("logstash.runner.flag.pipeline-batch-size"),
     :attribute_name => "pipeline.batch.size",

data/locales/en.yml

@@ -290,6 +290,8 @@ en:
           Sets the number of pipeline workers to run.
         java-execution: |+
           Use Java execution engine.
+        plugin-classloaders: |+
+          (Beta) Load Java plugins in independent classloaders to isolate their dependencies.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+

data/logstash-core.gemspec

@@ -57,7 +57,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "sinatra", '~> 1', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2'
   gem.add_runtime_dependency "jruby-openssl", "~> 0.10" # >= 0.9.13 Required to support TLSv1.2
-  gem.add_runtime_dependency "faraday", "~> 0.9.0" # because of conflicting dependencies between twitter input and octokit
   gem.add_runtime_dependency "chronic_duration", "~> 0.10"
 
   gem.add_runtime_dependency "treetop", "~> 1" #(MIT license)
@@ -73,4 +72,5 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "elasticsearch", "~> 5"
   gem.add_runtime_dependency "manticore", '~> 0.6'
+  gem.add_runtime_dependency "faraday", '~> 0.9.0'
 end

data/spec/logstash/codecs/delegator_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "spec_helper"
+
+class LogStash::Codecs::MockCodec < LogStash::Codecs::Base
+  config_name "my_name"
+
+  def multi_encode(e)
+  end
+
+  def encode(e)
+  end
+
+  def decode(e)
+    for i in e.split('|')
+      yield i
+    end
+  end
+end
+
+describe LogStash::Codecs::Delegator do
+  let(:collector)   { LogStash::Instrument::Collector.new }
+  let(:metric) { LogStash::Instrument::Metric.new(collector) }
+  let(:codec) { LogStash::Codecs::MockCodec.new }
+
+  subject do
+    delegator = described_class.new(codec)
+    delegator.metric = metric.namespace([:stats, :pipelines, :main, :plugins, :codecs, :my_id])
+    delegator
+  end
+
+  let(:snapshot_store) { collector.snapshot_metric.metric_store }
+
+  let(:snapshot_metric) { snapshot_store.get_shallow(:stats) }
+
+  describe "#encode" do
+    it "should delegate call to codec" do
+      expect(codec).to receive(:encode).with("abcdef")
+      subject.encode("abcdef")
+    end
+
+    it "should increment metrics" do
+      subject.encode("test")
+      expect(snapshot_metric[:pipelines][:main][:plugins][:codecs][:my_id][:encode][:writes_in].value).to eq(1)
+    end
+  end
+
+  describe "#multi_encode" do
+    it "should delegate call to codec" do
+      expect(codec).to receive(:multi_encode).with(%w(ay laa))
+      subject.multi_encode(%w(ay laa))
+    end
+
+    it "should increment metrics" do
+      subject.multi_encode(%w(ay test))
+      expect(snapshot_metric[:pipelines][:main][:plugins][:codecs][:my_id][:encode][:writes_in].value).to eq(2)
+    end
+  end
+
+  describe "#decode" do
+    it "should delegate call to codec" do
+      expect(codec).to receive(:decode).with("ayooooo")
+      subject.decode("ayooooo")
+    end
+
+    it "should increment metrics" do
+      subject.decode("bird|law") {}
+      expect(snapshot_metric[:pipelines][:main][:plugins][:codecs][:my_id][:decode][:writes_in].value).to eq(1)
+      expect(snapshot_metric[:pipelines][:main][:plugins][:codecs][:my_id][:decode][:out].value).to eq(2)
+    end
+  end
+
+  describe "#close" do
+    it "should delegate call to codec" do
+      expect(codec).to receive(:close)
+      subject.close
+    end
+  end
+
+  describe "#plugin_type" do
+    it "should delegate call to codec" do
+      expect(codec).to receive(:plugin_type)
+      subject.plugin_type
+    end
+  end
+end

data/spec/logstash/compiler/compiler_spec.rb

@@ -252,6 +252,34 @@ describe LogStash::Compiler do
           expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
         end
 
+        describe "a filter plugin with a repeated hash directive with duplicated keys" do
+          let(:source) { "input { } filter { #{plugin_source} } output { } " }
+          let(:plugin_source) do
+            %q[
+              grok {
+                match => { "message" => "foo" }
+                match => { "message" => "bar" }
+                break_on_match => false
+              }
+          ]
+          end
+          subject(:c_plugin) { compiled[:filter] }
+
+          let(:expected_plugin_args) do
+            {
+                "match" => {
+                    "message" => ["foo", "bar"]
+                },
+                "break_on_match" => "false"
+            }
+          end
+
+          it "should merge the values of the duplicate keys into an array" do
+            expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
+          end
+
+        end
+
         describe "a filter plugin that has nested Hash directives" do
           let(:source) { "input { } filter { #{plugin_source} } output { } " }
           let(:plugin_source) do