logstash-core 6.1.4-java → 6.2.0-java

data/lib/logstash/plugin.rb

@@ -73,7 +73,7 @@ class LogStash::Plugin
   # close is called during shutdown, after the plugin worker
   # main task terminates
   def do_close
-    @logger.debug("closing", :plugin => self.class.name)
+    @logger.debug("Closing", :plugin => self.class.name)
     close
   end
 

data/lib/logstash/plugins/plugin_factory.rb

@@ -32,11 +32,10 @@ module LogStash
     class PluginFactory
       include org.logstash.config.ir.compiler.RubyIntegration::PluginFactory
 
-      def initialize(lir, metric_factory, logger, exec_factory, filter_class)
+      def initialize(lir, metric_factory, exec_factory, filter_class)
         @lir = lir
         @plugins_by_id = {}
         @metric_factory = metric_factory
-        @logger = logger
         @exec_factory = exec_factory
         @filter_class = filter_class
       end
@@ -83,9 +82,9 @@ module LogStash
         execution_context = @exec_factory.create(id, klass.config_name)
 
         if plugin_type == "output"
-          OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
+          OutputDelegator.new(klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
         elsif plugin_type == "filter"
-          @filter_class.new(@logger, klass, type_scoped_metric, execution_context, args)
+          @filter_class.new(klass, type_scoped_metric, execution_context, args)
         else # input or codec plugin
           plugin_instance = klass.new(args)
           scoped_metric = type_scoped_metric.namespace(id.to_sym)

data/lib/logstash/runner.rb

@@ -85,6 +85,11 @@ class LogStash::Runner < Clamp::StrictCommand
     :attribute_name => "cloud.auth"
 
   # Pipeline settings
+  option ["--pipeline.id"], "ID",
+    I18n.t("logstash.runner.flag.pipeline-id"),
+    :attribute_name => "pipeline.id",
+    :default => LogStash::SETTINGS.get_default("pipeline.id")
+
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
     :attribute_name => "pipeline.workers",

data/lib/logstash/settings.rb

@@ -9,6 +9,7 @@ module LogStash
   class Settings
 
     include LogStash::Util::SubstitutionVariables
+    include LogStash::Util::Loggable
     
     def initialize
       @settings = {}
@@ -27,6 +28,10 @@ module LogStash
       end
     end
 
+    def registered?(setting_name)
+       @settings.key?(setting_name)
+    end
+
     def get_setting(setting_name)
       setting = @settings[setting_name]
       raise ArgumentError.new("Setting \"#{setting_name}\" hasn't been registered") if setting.nil?

data/lib/logstash/timestamp.rb

@@ -1,25 +1,2 @@
-# encoding: utf-8
-
-require "logstash/namespace"
-
-module LogStash
-
-  class Timestamp
-    include Comparable
-
-    def eql?(other)
-      self.== other
-    end
-
-    # TODO (colin) implement in Java
-    def +(other)
-      self.time + other
-    end
-
-    # TODO (colin) implement in Java
-    def -(value)
-      self.time - (value.is_a?(Timestamp) ? value.time : value)
-    end
-
-  end
-end
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly require "logstash/timestamp".

data/lib/logstash/util/secretstore.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+# Ruby helper to work with the secret store
+module ::LogStash::Util::SecretStore
+
+  java_import "org.logstash.secret.store.SecretStoreFactory"
+  java_import "org.logstash.secret.SecretIdentifier"
+  java_import "org.logstash.secret.store.SecureConfig"
+  java_import "org.logstash.secret.cli.SecretStoreCli"
+
+  # Return the configuration necessary to work with a secret store
+  def self.get_config
+    secure_config = SecureConfig.new
+    secure_config.add("keystore.file", LogStash::SETTINGS.get_setting("keystore.file").value.chars)
+    pass = ENV["LOGSTASH_KEYSTORE_PASS"]
+    secure_config.add("keystore.pass", pass.chars) unless pass.nil?
+    secure_config.add("keystore.classname", LogStash::SETTINGS.get_setting("keystore.classname").value.chars)
+    secure_config
+  end
+
+  # Check to see if the secret store exists, return true if exists, false otherwise
+  def self.exists?
+    SecretStoreFactory.exists(get_config)
+  end
+
+  # Returns a org.logstash.secret.store.SecretStore if it exists, nil otherwise
+  def self.get_if_exists
+    SecretStoreFactory.load(get_config) if exists?
+  end
+
+  # Returns a org.org.logstash.secret.SecretIdentifier for use with the secret store
+  def self.get_store_id(id)
+    SecretIdentifier.new(id)
+  end
+
+end

data/lib/logstash/util/settings_helper.rb

@@ -32,6 +32,7 @@ module LogStash::Util::SettingsHelper
     settings_path = fetch_settings_path(args)
 
     LogStash::SETTINGS.set("path.settings", settings_path) if settings_path
+    LogStash::SETTINGS.set("keystore.file", ::File.join(settings_path, "logstash.keystore")) if settings_path
 
     begin
       LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))

data/lib/logstash/util/substitution_variables.rb

@@ -1,9 +1,15 @@
 # encoding: utf-8
+require "logstash/logging"
+require "logstash/util/loggable"
+require "logstash/util/secretstore"
+
 module ::LogStash::Util::SubstitutionVariables
 
+  include LogStash::Util::Loggable
+
   SUBSTITUTION_PLACEHOLDER_REGEX = /\${(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?}/
 
-  # Recursive method to replace environment variable references in parameters
+  # Recursive method to replace substitution variable references in parameters
   def deep_replace(value)
     if value.is_a?(Hash)
       value.each do |valueHashKey, valueHashValue|
@@ -22,7 +28,7 @@ module ::LogStash::Util::SubstitutionVariables
 
   # Replace all substitution variable references in the 'value' param and returns the substituted value, or the original value if a substitution can not be made
   # Process following patterns : ${VAR}, ${VAR:defaultValue}
-  # If value matches the pattern, returns the following precedence : Environment entry value, default value as provided in the pattern
+  # If value matches the pattern, returns the following precedence : Secret store value, Environment entry value, default value as provided in the pattern
   # If the value does not match the pattern, the 'value' param returns as-is
   def replace_placeholders(value)
     return value unless value.is_a?(String)
@@ -34,12 +40,19 @@ module ::LogStash::Util::SubstitutionVariables
       # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
       name = Regexp.last_match(:name)
       default = Regexp.last_match(:default)
+      logger.debug("Replacing `#{placeholder}` with actual value")
 
-      replacement = ENV.fetch(name, default)
+      #check the secret store if it exists
+      secret_store = LogStash::Util::SecretStore.get_if_exists
+      replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(LogStash::Util::SecretStore.get_store_id(name))
+      #check the environment
+      replacement = ENV.fetch(name, default) if replacement.nil?
       if replacement.nil?
-        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
+        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Replacement variable `#{name}` is not defined in a Logstash secret store " +
+            "or as an Environment entry and there is no default value given."
       end
-      replacement
+      replacement.to_s
     end
   end # def replace_placeholders
+
 end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -94,7 +94,7 @@ module LogStash; module Util
       # from this queue. We also depend on this to be able to block consumers while we snapshot
       # in-flight buffers
 
-      def initialize(queue, batch_size = 125, wait_for = 250)
+      def initialize(queue, batch_size = 125, wait_for = 50)
         @queue = queue
         @mutex = Mutex.new
         # Note that @inflight_batches as a central mechanism for tracking inflight

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -29,7 +29,7 @@ module LogStash; module Util
       # from this queue. We also depend on this to be able to block consumers while we snapshot
       # in-flight buffers
 
-      def initialize(queue, batch_size = 125, wait_for = 250)
+      def initialize(queue, batch_size = 125, wait_for = 50)
         @queue = queue
         # Note that @inflight_batches as a central mechanism for tracking inflight
         # batches will fail if we have multiple read clients in the pipeline.
@@ -83,11 +83,11 @@ module LogStash; module Util
       # create a new empty batch
       # @return [ReadBatch] a new empty read batch
       def new_batch
-        ReadBatch.new(@queue, 0, 0)
+        LogStash::MemoryReadBatch.new(java.util.LinkedHashSet.new(0))
       end
 
       def read_batch
-        batch = ReadBatch.new(@queue, @batch_size, @wait_for)
+        batch = LogStash::MemoryReadBatch.new(LsQueueUtils.drain(@queue, @batch_size, @wait_for))
         start_metrics(batch)
         batch
       end
@@ -125,38 +125,6 @@ module LogStash; module Util
       end
     end
 
-    class ReadBatch
-      def initialize(queue, size, wait)
-        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
-        # @cancelled = Hash.new
-
-        @originals = LsQueueUtils.drain(queue, size, wait)
-      end
-
-      def merge(event)
-        return if event.nil?
-        @originals.add(event)
-      end
-
-      def to_a
-        events = []
-        @originals.each {|e| events << e unless e.cancelled?}
-        events
-      end
-
-      def each(&blk)
-        # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
-        # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
-        @originals.each {|e| blk.call(e) unless e.cancelled?}
-      end
-
-      def filtered_size
-        @originals.size
-      end
-
-      alias_method(:size, :filtered_size)
-    end
-
     class WriteClient
       def initialize(queue)
         @queue = queue

data/locales/en.yml

@@ -58,9 +58,9 @@ en:
       sighup: >-
         SIGHUP received.
       sigint: >-
-        SIGINT received. Shutting down the agent.
+        SIGINT received. Shutting down.
       sigterm: >-
-        SIGTERM received. Shutting down the agent.
+        SIGTERM received. Shutting down.
       slow_shutdown: |-
         Received shutdown signal, but pipeline is still waiting for in-flight events
         to be processed. Sending another ^C will force quit Logstash, but this may cause
@@ -262,6 +262,8 @@ en:
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host
         http_port: Web API http port
+        pipeline-id: |+
+          Sets the ID of the pipeline.
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         experimental-java-execution: |+
@@ -327,8 +329,6 @@ en:
         name: |+
           Specify the name of this logstash instance, if no value is given
           it will default to the current hostname.
-        agent: |+
-          Specify an alternate agent plugin name.
         config_debug: |+
           Print the compiled config ruby code out as a debug log (you must also have --log.level=debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result

data/logstash-core.gemspec

@@ -70,13 +70,6 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "jrjackson", "~> #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "jar-dependencies"
-  # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version
-  # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
-  # which does not have this problem.
-  gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
   gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
   gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
-
-  eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
 end

data/spec/conditionals_spec.rb

@@ -491,15 +491,14 @@ describe "conditionals in filter" do
     sample_one({"type" => "original"}) do
       expect(subject).to be_an(Array)
       expect(subject.length).to eq(2)
-      subject.sort! {|a, b| a.get("type") <=> b.get("type")}
-
-      expect(subject[1].get("type")).to eq("original")
-      expect(subject[1].get("cond1")).to eq("true")
-      expect(subject[1].get("cond2")).to eq(nil)
-
-      expect(subject[0].get("type")).to eq("clone")
-      # expect(subject[1].get("cond1")).to eq(nil)
-      # expect(subject[1].get("cond2")).to eq("true")
+      original_event = subject[0]
+      expect(original_event.get("type")).to eq("original")
+      expect(original_event.get("cond1")).to eq("true")
+      expect(original_event.get("cond2")).to eq(nil)
+      cloned_event = subject[1]
+      expect(cloned_event.get("cond1")).to eq(nil)
+      expect(cloned_event.get("cond2")).to eq("true")
+      expect(cloned_event.get("type")).to eq("clone")
     end
   end
 
@@ -520,20 +519,18 @@ describe "conditionals in filter" do
     CONFIG
 
     sample_one({"type" => "original"}) do
-      expect(subject.length).to eq(3)
-      subject.sort! {|a, b| a.get("type") <=> b.get("type")}
-
-      expect(subject[0].get("type")).to eq("clone1")
-      expect(subject[0].get("cond1")).to eq("true")
-      expect(subject[0].get("cond2")).to eq(nil)
-
-      expect(subject[1].get("type")).to eq("clone2")
-      expect(subject[1].get("cond1")).to eq(nil)
-      expect(subject[1].get("cond2")).to eq("true")
-
-      expect(subject[2].get("type")).to eq("original")
-      expect(subject[2].get("cond1")).to eq(nil)
-      expect(subject[2].get("cond2")).to eq(nil)
+      clone_event_1 = subject[0]
+      expect(clone_event_1.get("type")).to eq("clone1")
+      expect(clone_event_1.get("cond1")).to eq("true")
+      expect(clone_event_1.get("cond2")).to eq(nil)
+      clone_event_2 = subject[1]
+      expect(clone_event_2.get("type")).to eq("clone2")
+      expect(clone_event_2.get("cond1")).to eq(nil)
+      expect(clone_event_2.get("cond2")).to eq("true")
+      original_event = subject[2]
+      expect(original_event.get("type")).to eq("original")
+      expect(original_event.get("cond1")).to eq(nil)
+      expect(original_event.get("cond2")).to eq(nil)
     end
   end
 
@@ -586,7 +583,7 @@ describe "conditionals in filter" do
       expect(tags[6]).to eq("prev")
       expect(tags[7]).to eq("final")
     end
-    
+
     sample_one("type" => "original") do
       tags = subject.get("tags")
       expect(tags[0]).to eq("prev")

data/spec/logstash/filter_delegator_spec.rb

@@ -14,8 +14,7 @@ describe LogStash::FilterDelegator do
   end
 
   include_context "execution_context"
-  
-  let(:logger) { double(:logger) }
+
   let(:filter_id) { "my-filter" }
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
@@ -43,11 +42,11 @@ describe LogStash::FilterDelegator do
     end
   end
 
-  subject { described_class.new(logger, plugin_klass, metric, execution_context, config) }
+  subject { described_class.new(plugin_klass, metric, execution_context, config) }
 
   it "create a plugin with the passed options" do
     expect(plugin_klass).to receive(:new).with(config).and_return(plugin_klass.new(config))
-    described_class.new(logger, plugin_klass, metric, execution_context, config)
+    described_class.new(plugin_klass, metric, execution_context, config)
   end
 
   context "when the plugin support flush" do

data/spec/logstash/java_filter_delegator_spec.rb

@@ -14,8 +14,7 @@ describe LogStash::JavaFilterDelegator do
   end
 
   include_context "execution_context"
-  
-  let(:logger) { double(:logger) }
+
   let(:filter_id) { "my-filter" }
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
@@ -43,11 +42,11 @@ describe LogStash::JavaFilterDelegator do
     end
   end
 
-  subject { described_class.new(logger, plugin_klass, metric, execution_context, config) }
+  subject { described_class.new(plugin_klass, metric, execution_context, config) }
 
   it "create a plugin with the passed options" do
     expect(plugin_klass).to receive(:new).with(config).and_return(plugin_klass.new(config))
-    described_class.new(logger, plugin_klass, metric, execution_context, config)
+    described_class.new(plugin_klass, metric, execution_context, config)
   end
 
   context "when the plugin support flush" do

data/spec/logstash/java_pipeline_spec.rb

@@ -355,7 +355,7 @@ describe LogStash::JavaPipeline do
       after do
         pipeline.shutdown
       end
-      
+
       it "should call close of output without output-workers" do
         pipeline.run
 
@@ -380,7 +380,7 @@ describe LogStash::JavaPipeline do
       # cause the suite to fail :(
       pipeline.close
     end
-    
+
     it "should use LIR provided IDs" do
       expect(pipeline.inputs.first.id).to eq(pipeline.lir.input_plugin_vertices.first.id)
       expect(pipeline.filters.first.id).to eq(pipeline.lir.filter_plugin_vertices.first.id)
@@ -642,6 +642,101 @@ describe LogStash::JavaPipeline do
     end
   end
 
+  context "Periodic Flush Wrapped in Nested Conditional" do
+    let(:config) do
+      <<-EOS
+      input {
+        dummy_input {}
+      }
+      filter {
+        if [type] == "foo" {
+          if [@bar] {
+             dummy_flushing_filter {}
+          }
+        } else {
+          drop {}
+        }
+      }
+      output {
+        dummy_output {}
+      }
+      EOS
+    end
+    let(:output) { ::LogStash::Outputs::DummyOutput.new }
+
+    before do
+      allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output)
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummy_input").and_return(LogStash::Inputs::DummyBlockingInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummy_flushing_filter").and_return(DummyFlushingFilterPeriodic)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "drop").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+    end
+
+    it "flush periodically" do
+      Thread.abort_on_exception = true
+      pipeline = mock_java_pipeline_from_string(config, pipeline_settings_obj)
+      t = Thread.new { pipeline.run }
+      Timeout.timeout(timeout) do
+        sleep(0.1) until pipeline.ready?
+      end
+      Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(11).for do
+          # give us a bit of time to flush the events
+          output.events.size >= 2
+        end.to be_truthy
+      end
+
+      expect(output.events.any? {|e| e.get("message") == "dummy_flush"}).to eq(true)
+
+      pipeline.shutdown
+
+      t.join
+    end
+  end
+
+  context "with multiple outputs" do
+    let(:config) do
+      <<-EOS
+      input {
+        generator { count => 10 }
+      }
+      filter {
+       clone {
+          add_field => { 
+            'cloned' =>  'cloned' 
+          }
+          clones => ["clone1"]
+        }
+      }
+      output {
+        dummy_output {}
+        dummy_output {}
+        dummy_output {}
+      }
+      EOS
+    end
+    let(:output) { ::LogStash::Outputs::DummyOutput.new }
+
+    before do
+      allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output)
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "clone").and_call_original
+      3.times {
+        allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput)
+        allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+      }
+    end
+
+    it "correctly distributes events" do
+      pipeline = mock_java_pipeline_from_string(config, pipeline_settings_obj)
+      pipeline.run
+      pipeline.shutdown
+      expect(output.events.size).to eq(60)
+      expect(output.events.count {|e| e.get("cloned") == "cloned"}).to eq(30)
+    end
+  end
+
   context "#started_at" do
     # use a run limiting count to shutdown the pipeline automatically
     let(:config) do