logstash-core 6.3.2-java → 6.4.0-java

data/lib/logstash/util/secretstore.rb

@@ -1,38 +1 @@
-# encoding: utf-8
-
-# Ruby helper to work with the secret store
-module ::LogStash::Util::SecretStore
-
-  java_import "org.logstash.secret.store.SecretStoreFactory"
-  java_import "org.logstash.secret.SecretIdentifier"
-  java_import "org.logstash.secret.store.SecureConfig"
-  java_import "org.logstash.secret.cli.SecretStoreCli"
-
-  SECRET_STORE_FACTORY = SecretStoreFactory::fromEnvironment()
-
-  # Return the configuration necessary to work with a secret store
-  def self.get_config
-    secure_config = SecureConfig.new
-    secure_config.add("keystore.file", LogStash::SETTINGS.get_setting("keystore.file").value.chars)
-    pass = ENV["LOGSTASH_KEYSTORE_PASS"]
-    secure_config.add("keystore.pass", pass.chars) unless pass.nil?
-    secure_config.add("keystore.classname", LogStash::SETTINGS.get_setting("keystore.classname").value.chars)
-    secure_config
-  end
-
-  # Check to see if the secret store exists, return true if exists, false otherwise
-  def self.exists?
-    SECRET_STORE_FACTORY.exists(get_config)
-  end
-
-  # Returns a org.logstash.secret.store.SecretStore if it exists, nil otherwise
-  def self.get_if_exists
-    SECRET_STORE_FACTORY.load(get_config) if exists?
-  end
-
-  # Returns a org.org.logstash.secret.SecretIdentifier for use with the secret store
-  def self.get_store_id(id)
-    SecretIdentifier.new(id)
-  end
-
-end
+# Keeping this file for backwards compatibility with plugins that include it directly.

data/lib/logstash/util/substitution_variables.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
-require "logstash/logging"
-require "logstash/util/loggable"
-require "logstash/util/secretstore"
+
+java_import "org.logstash.secret.store.SecretStoreExt"
 
 module ::LogStash::Util::SubstitutionVariables
 
@@ -43,8 +42,8 @@ module ::LogStash::Util::SubstitutionVariables
       logger.debug("Replacing `#{placeholder}` with actual value")
 
       #check the secret store if it exists
-      secret_store = LogStash::Util::SecretStore.get_if_exists
-      replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(LogStash::Util::SecretStore.get_store_id(name))
+      secret_store = SecretStoreExt.getIfExists(LogStash::SETTINGS.get_setting("keystore.file").value, LogStash::SETTINGS.get_setting("keystore.classname").value)
+      replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(SecretStoreExt.getStoreId(name))
       #check the environment
       replacement = ENV.fetch(name, default) if replacement.nil?
       if replacement.nil?

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/util"
 
 # This class exists to format the settings for default worker threads

data/locales/en.yml

@@ -228,6 +228,33 @@ en:
           "%{default_output}"
           If you wish to use both defaults, please use
           the empty string for the '-e' flag.
+        field-reference-parser: |+
+          Use the given MODE when parsing field
+          references.
+
+          The field reference parser is used to expand
+          field references in your pipeline configs,
+          and will be becoming more strict to better
+          handle illegal and ambbiguous inputs in a
+          future release of Logstash.
+
+          Available MODEs are:
+           - `LEGACY`: parse with the legacy parser,
+             which is known to handle ambiguous- and
+             illegal-syntax in surprising ways;
+             warnings will not be emitted.
+           - `COMPAT`: warn once for each distinct
+             ambiguous- or illegal-syntax input, but
+             continue to expand field references with
+             the legacy parser.
+           - `STRICT`: parse in a strict manner; when
+             given ambiguous- or illegal-syntax input,
+             raises a runtime exception that should
+             be handled by the calling plugin.
+
+           The MODE can also be set with
+           `config.field_reference.parser`
+
         modules: |+
           Load Logstash modules.
           Modules can be defined using multiple instances
@@ -348,4 +375,4 @@ en:
           DEPRECATED: use --log.level=info instead.
         quiet: |+
           Set the log level to info.
-          DEPRECATED: use --log.level=quiet instead.
+          DEPRECATED: use --log.level=info instead.

data/spec/logstash/config/mixin_spec.rb

@@ -7,7 +7,7 @@ describe LogStash::Config::Mixin do
     let(:password) { "sekret" }
     let(:double_logger) { double("logger").as_null_object }
 
-    subject do 
+    subject do
       Class.new(LogStash::Filters::Base) do
         include LogStash::Config::Mixin
         config_name "test_deprecated"
@@ -151,7 +151,7 @@ describe LogStash::Config::Mixin do
 
         it "should raise a configuration error" do
           expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
-        end        
+        end
       end
 
       context "with no value specified" do
@@ -160,7 +160,7 @@ describe LogStash::Config::Mixin do
         it "should raise a configuration error" do
           expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
         end
-      end          
+      end
     end
   end
 
@@ -208,7 +208,7 @@ describe LogStash::Config::Mixin do
 
     shared_examples("safe URI") do |options|
       options ||= {}
-      
+
       subject { klass.new("uri" => uri_str) }
 
       it "should be a SafeURI object" do

data/spec/logstash/converge_result_spec.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/converge_result"
 require "logstash/pipeline_action/stop"
 require "spec_helper"
 

data/spec/logstash/event_dispatcher_spec.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 #
-require "logstash/event_dispatcher"
-
 describe LogStash::EventDispatcher do
   class DummyEmitter
     attr_reader :dispatcher

data/spec/logstash/event_spec.rb

@@ -76,6 +76,17 @@ describe LogStash::Event do
         expect(e.get("[foo][-1]")).to eq(list[-1])
       end
     end
+
+    context 'with illegal-syntax field reference' do
+      # NOTE: in true-legacy-mode FieldReference parsing, the input `[` caused Logstash
+      # to crash entirely with a Java ArrayIndexOutOfBounds exception; this spec ensures that
+      # we instead raise a RuntimeException that can be handled normally within the
+      # Ruby runtime.
+      it 'raises a RuntimeError' do
+        e = LogStash::Event.new
+        expect { e.get('[') }.to raise_exception(::RuntimeError)
+      end
+    end
   end
 
   context "#set" do
@@ -145,32 +156,6 @@ describe LogStash::Event do
       expect(e.get("foo").to_f).to be_within(0.1).of(time.to_f)
     end
 
-    it "should set XXJavaProxy Jackson crafted" do
-      proxy = org.logstash.RspecTestUtils.getMapFixtureJackson()
-      # proxy is {"string": "foo", "int": 42, "float": 42.42, "array": ["bar","baz"], "hash": {"string":"quux"} }
-      e = LogStash::Event.new()
-      e.set("[proxy]", proxy)
-      expect(e.get("[proxy][string]")).to eql("foo")
-      expect(e.get("[proxy][int]")).to eql(42)
-      expect(e.get("[proxy][float]")).to eql(42.42)
-      expect(e.get("[proxy][array][0]")).to eql("bar")
-      expect(e.get("[proxy][array][1]")).to eql("baz")
-      expect(e.get("[proxy][hash][string]")).to eql("quux")
-    end
-
-    it "should set XXJavaProxy hand crafted" do
-      proxy = org.logstash.RspecTestUtils.getMapFixtureHandcrafted()
-      # proxy is {"string": "foo", "int": 42, "float": 42.42, "array": ["bar","baz"], "hash": {"string":"quux"} }
-      e = LogStash::Event.new()
-      e.set("[proxy]", proxy)
-      expect(e.get("[proxy][string]")).to eql("foo")
-      expect(e.get("[proxy][int]")).to eql(42)
-      expect(e.get("[proxy][float]")).to eql(42.42)
-      expect(e.get("[proxy][array][0]")).to eql("bar")
-      expect(e.get("[proxy][array][1]")).to eql("baz")
-      expect(e.get("[proxy][hash][string]")).to eql("quux")
-    end
-
     it "should fail on non UTF-8 encoding" do
       # e = LogStash::Event.new
       # s1 = "\xE0 Montr\xE9al".force_encoding("ISO-8859-1")
@@ -181,6 +166,17 @@ describe LogStash::Event do
       # expect(s2.encoding.name).to eq("UTF-8")
       # expect(s2.valid_encoding?).to eq(true)
     end
+
+    context 'with illegal-syntax field reference' do
+      # NOTE: in true-legacy-mode FieldReference parsing, the input `[` caused Logstash
+      # to crash entirely with a Java ArrayIndexOutOfBounds exception; this spec ensures that
+      # we instead raise a RuntimeException that can be handled normally within the
+      # Ruby runtime.
+      it 'raises a RuntimeError' do
+        e = LogStash::Event.new
+        expect { e.set('[', 'value') }.to raise_exception(::RuntimeError)
+      end
+    end
   end
 
   context "timestamp" do

data/spec/logstash/execution_context_spec.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 require "spec_helper"
-require "logstash/util/dead_letter_queue_manager"
-require "logstash/execution_context"
 
 describe LogStash::ExecutionContext do
   let(:pipeline) { double("pipeline") }

data/spec/logstash/filter_delegator_spec.rb

@@ -1,37 +1,24 @@
 # encoding: utf-8
 require "spec_helper"
+require 'logstash/instrument/collector'
 require "logstash/filter_delegator"
-require "logstash/instrument/null_metric"
 require "logstash/event"
-require "logstash/execution_context"
 require "support/shared_contexts"
 
 describe LogStash::FilterDelegator do
 
-  class MockGauge
-    def increment(_)
-    end
-  end
-
   include_context "execution_context"
 
   let(:filter_id) { "my-filter" }
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
   end
-  let(:collector) { [] }
-  let(:counter_in) { MockGauge.new }
-  let(:counter_out) { MockGauge.new }
-  let(:counter_time) { MockGauge.new }
-  let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
+  let(:collector) {LogStash::Instrument::Collector.new}
+  let(:metric) { LogStash::Instrument::Metric.new(collector).namespace(:null) }
   let(:events) { [LogStash::Event.new, LogStash::Event.new] }
 
   before :each do
     allow(pipeline).to receive(:id).and_return(pipeline_id)
-    allow(metric).to receive(:namespace).with(anything).and_return(metric)
-    allow(metric).to receive(:counter).with(:in).and_return(counter_in)
-    allow(metric).to receive(:counter).with(:out).and_return(counter_out)
-    allow(metric).to receive(:counter).with(:duration_in_millis).and_return(counter_time)
   end
 
   let(:plugin_klass) do
@@ -70,8 +57,8 @@ describe LogStash::FilterDelegator do
     context "when the flush return events" do
       it "increments the out" do
         subject.multi_filter([LogStash::Event.new])
-        expect(counter_out).to receive(:increment).with(1)
         subject.flush({})
+        expect(collector.snapshot_metric.metric_store.get_with_path("/null")[:null]["my-filter".to_sym][:events][:out].value).to eq(1)
       end
     end
 
@@ -83,18 +70,15 @@ describe LogStash::FilterDelegator do
     end
 
     context "when the filter buffer events" do
-      before do
-        allow(metric).to receive(:increment).with(anything, anything)
-      end
 
       it "has incremented :in" do
-        expect(counter_in).to receive(:increment).with(events.size)
         subject.multi_filter(events)
+        expect(collector.snapshot_metric.metric_store.get_with_path("/null")[:null]["my-filter".to_sym][:events][:in].value).to eq(events.size)
       end
 
       it "has not incremented :out" do
-        expect(counter_out).not_to receive(:increment).with(anything)
         subject.multi_filter(events)
+        expect(collector.snapshot_metric.metric_store.get_with_path("/null")[:null]["my-filter".to_sym][:events][:out].value).to eq(0)
       end
     end
 
@@ -119,10 +103,10 @@ describe LogStash::FilterDelegator do
       end
 
       it "increments the in/out of the metric" do
-        expect(counter_in).to receive(:increment).with(events.size)
-        expect(counter_out).to receive(:increment).with(events.size * 2)
-
         subject.multi_filter(events)
+        event_metrics = collector.snapshot_metric.metric_store.get_with_path("/null")[:null]["my-filter".to_sym][:events]
+        expect(event_metrics[:in].value).to eq(events.size)
+        expect(event_metrics[:out].value).to eq(2 * events.size)
       end
     end
   end
@@ -148,10 +132,10 @@ describe LogStash::FilterDelegator do
     end
 
     it "increments the in/out of the metric" do
-      expect(counter_in).to receive(:increment).with(events.size)
-      expect(counter_out).to receive(:increment).with(events.size)
-
       subject.multi_filter(events)
+      event_metrics = collector.snapshot_metric.metric_store.get_with_path("/null")[:null]["my-filter".to_sym][:events]
+      expect(event_metrics[:in].value).to eq(events.size)
+      expect(event_metrics[:out].value).to eq(events.size)
     end
   end
 

data/spec/logstash/inputs/base_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "spec_helper"
-require "logstash/execution_context"
 require "logstash/inputs/base"
 require "support/shared_contexts"
 
@@ -90,15 +89,15 @@ describe "LogStash::Inputs::Base#decorate" do
     let(:input) do
       LogStash::Inputs::NOOP.new("add_field" => {"field" => ["value1", "value2"], "field2" => "value"})
     end
-    
+
     let(:cloned) do
       input.clone
     end
-    
+
     it "should clone the codec when cloned" do
       expect(input.codec).not_to eq(cloned.codec)
-    end  
-    
+    end
+
     it "should preserve codec params" do
       expect(input.codec.params).to eq(cloned.codec.params)
     end

data/spec/logstash/instrument/metric_spec.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/metric"
 require "logstash/instrument/collector"
 require_relative "../../support/matchers"
 require "spec_helper"

data/spec/logstash/instrument/namespaced_metric_spec.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/namespaced_metric"
-require "logstash/instrument/metric"
 require_relative "../../support/matchers"
 require_relative "../../support/shared_examples"
 require "spec_helper"

data/spec/logstash/instrument/namespaced_null_metric_spec.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/namespaced_null_metric"
-require "logstash/instrument/null_metric"
 require_relative "../../support/matchers"
 require "spec_helper"
 
@@ -16,7 +14,7 @@ describe LogStash::Instrument::NamespacedNullMetric do
   end
 
   it "returns a TimedException when we call without a block" do
-    expect(subject.time(:duration_ms)).to be(LogStash::Instrument::NullMetric::NullTimedExecution)
+    expect(subject.time(:duration_ms)).to be_kind_of(LogStash::Instrument::NullMetric::NullTimedExecution)
   end
 
   it "returns the value of the block" do

data/spec/logstash/instrument/null_metric_spec.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/null_metric"
-require "logstash/instrument/namespaced_metric"
 require_relative "../../support/shared_examples"
 require_relative "../../support/matchers"
 require "spec_helper"
@@ -8,8 +6,7 @@ require "spec_helper"
 describe LogStash::Instrument::NullMetric do
 
   let(:key) { "test" }
-  let(:collector) { [] }
-  subject { LogStash::Instrument::NullMetric.new(collector) }
+  subject { LogStash::Instrument::NullMetric.new(nil) }
 
   it "defines the same interface as `Metric`" do
     expect(described_class).to implement_interface_of(LogStash::Instrument::Metric)

data/spec/logstash/instrument/periodic_poller/base_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
-require "logstash/instrument/metric"
 require "logstash/instrument/collector"
 
 describe LogStash::Instrument::PeriodicPoller::Base do

data/spec/logstash/instrument/periodic_poller/os_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/os"
-require "logstash/instrument/metric"
 require "logstash/instrument/collector"
 
 describe LogStash::Instrument::PeriodicPoller::Os do

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/metric"
 require "logstash/event"
 require_relative "../../support/mocks_classes"
 require "spec_helper"

data/spec/logstash/java_filter_delegator_spec.rb

@@ -1,9 +1,7 @@
 # encoding: utf-8
 require "spec_helper"
 require "logstash/filter_delegator"
-require "logstash/instrument/null_metric"
 require "logstash/event"
-require "logstash/execution_context"
 require "support/shared_contexts"
 
 java_import org.logstash.RubyUtil
@@ -21,7 +19,6 @@ describe LogStash::JavaFilterDelegator do
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
   end
-  let(:collector) { [] }
   let(:metric) {
     LogStash::Instrument::NamespacedMetric.new(
         LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new), [:filter]