logstash-core 6.0.0.beta1-java → 6.0.0.beta2-java

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -205,19 +205,18 @@ module LogStash; module Util
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = [
-          @event_metric.time(:duration_in_millis),
-          @pipeline_metric.time(:duration_in_millis)
-        ]
+        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
       end
 
       def stop_clock(batch)
         unless @inflight_clocks[Thread.current].nil?
           if batch.size > 0
-            # onl/y stop (which also records) the metrics if the batch is non-empty.
+            # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            @inflight_clocks[Thread.current].each(&:stop)
+            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
+            @event_metric.report_time(:duration_in_millis, time_taken)
+            @pipeline_metric.report_time(:duration_in_millis, time_taken)
           end
           @inflight_clocks.delete(Thread.current)
         end

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -68,11 +68,17 @@ module LogStash; module Util
 
       def set_events_metric(metric)
         @event_metric = metric
+        @event_metric_out = @event_metric.counter(:out)
+        @event_metric_filtered = @event_metric.counter(:filtered)
+        @event_metric_time = @event_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@event_metric)
       end
 
       def set_pipeline_metric(metric)
         @pipeline_metric = metric
+        @pipeline_metric_out = @pipeline_metric.counter(:out)
+        @pipeline_metric_filtered = @pipeline_metric.counter(:filtered)
+        @pipeline_metric_time = @pipeline_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@pipeline_metric)
       end
 
@@ -140,10 +146,7 @@ module LogStash; module Util
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = [
-          @event_metric.time(:duration_in_millis),
-          @pipeline_metric.time(:duration_in_millis)
-        ]
+        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
       end
 
       def stop_clock(batch)
@@ -152,20 +155,22 @@ module LogStash; module Util
             # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            @inflight_clocks[Thread.current].each(&:stop)
+            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
+            @event_metric_time.increment(time_taken)
+            @pipeline_metric_time.increment(time_taken)
           end
           @inflight_clocks.delete(Thread.current)
         end
       end
 
       def add_filtered_metrics(batch)
-        @event_metric.increment(:filtered, batch.filtered_size)
-        @pipeline_metric.increment(:filtered, batch.filtered_size)
+        @event_metric_filtered.increment(batch.filtered_size)
+        @pipeline_metric_filtered.increment(batch.filtered_size)
       end
 
       def add_output_metrics(batch)
-        @event_metric.increment(:out, batch.filtered_size)
-        @pipeline_metric.increment(:out, batch.filtered_size)
+        @event_metric_out.increment(batch.filtered_size)
+        @pipeline_metric_out.increment(batch.filtered_size)
       end
     end
 

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "6.0.0-beta1"
+LOGSTASH_VERSION = "6.0.0-beta2"

data/locales/en.yml

@@ -242,6 +242,22 @@ en:
           '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
           as in
           '-M "example.var.filter.mutate.fieldname=fieldvalue"'
+        modules_setup: |+
+          Load index template into Elasticsearch, and saved searches, 
+          index-pattern, visualizations, and dashboards into Kibana when
+          running modules.
+        cloud_id: |+
+          Sets the elasticsearch and kibana host settings for
+          module connections in Elastic Cloud.
+          Your Elastic Cloud User interface or the Cloud support
+          team should provide this.
+          Add an optional label prefix '<label>:' to help you
+          identify multiple cloud.ids.
+          e.g. 'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
+        cloud_auth: |+
+          Sets the elasticsearch and kibana username and password
+          for module connections in Elastic Cloud
+          e.g. 'username:<password>'
         configtest: |+
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host

data/spec/logstash/agent/converge_spec.rb

@@ -74,19 +74,18 @@ describe LogStash::Agent do
         describe "#running_user_defined_pipelines" do
           it "returns the user defined pipelines" do
             start_agent(subject)
-            subject.with_running_user_defined_pipelines do |pipelines|
-              expect(pipelines).to include(:main)
-              expect(pipelines).not_to include(:system_pipeline)
-            end
-            subject.shutdown
+            wait_for do
+              subject.with_running_user_defined_pipelines {|pipelines| pipelines.keys }
+            end.to eq([:main])
           end
         end
 
         describe "#running_user_defined_pipelines?" do
           it "returns true" do
             start_agent(subject)
-            expect(subject.running_user_defined_pipelines?).to be_truthy
-            subject.shutdown
+            wait_for do
+              subject.running_user_defined_pipelines?
+            end.to be_truthy
           end
         end
       end

data/spec/logstash/config/source/multi_local_spec.rb

@@ -155,5 +155,16 @@ describe LogStash::Config::Source::MultiLocal do
         expect { subject.pipeline_configs }.to raise_error(ArgumentError)
       end
     end
+
+    context 'using dead letter queue settings' do
+      let(:retrieved_pipelines) do [
+          { "pipeline.id" => "main", "path.dead_letter_queue" => "/tmp", "dead_letter_queue.max_bytes" => 10000 },
+      ]
+      end
+      it "should not raise an error" do
+        expect { subject.pipeline_configs }.not_to raise_error(ArgumentError)
+      end
+
+    end
   end
 end

data/spec/logstash/filter_delegator_spec.rb

@@ -7,20 +7,32 @@ require "logstash/execution_context"
 require "support/shared_contexts"
 
 describe LogStash::FilterDelegator do
+
+  class MockGauge
+    def increment(_)
+    end
+  end
+
   include_context "execution_context"
-  
+
   let(:logger) { double(:logger) }
   let(:filter_id) { "my-filter" }
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
   end
   let(:collector) { [] }
+  let(:counter_in) { MockGauge.new }
+  let(:counter_out) { MockGauge.new }
+  let(:counter_time) { MockGauge.new }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
   let(:events) { [LogStash::Event.new, LogStash::Event.new] }
 
   before :each do
     allow(pipeline).to receive(:id).and_return(pipeline_id)
     allow(metric).to receive(:namespace).with(anything).and_return(metric)
+    allow(metric).to receive(:counter).with(:in).and_return(counter_in)
+    allow(metric).to receive(:counter).with(:out).and_return(counter_out)
+    allow(metric).to receive(:counter).with(:duration_in_millis).and_return(counter_time)
   end
 
   let(:plugin_klass) do
@@ -60,7 +72,7 @@ describe LogStash::FilterDelegator do
     context "when the flush return events" do
       it "increments the out" do
         subject.multi_filter([LogStash::Event.new])
-        expect(metric).to receive(:increment).with(:out, 1)
+        expect(counter_out).to receive(:increment).with(1)
         subject.flush({})
       end
     end
@@ -78,12 +90,12 @@ describe LogStash::FilterDelegator do
       end
 
       it "has incremented :in" do
-        expect(metric).to receive(:increment).with(:in, events.size)
+        expect(counter_in).to receive(:increment).with(events.size)
         subject.multi_filter(events)
       end
 
       it "has not incremented :out" do
-        expect(metric).not_to receive(:increment).with(:out, anything)
+        expect(counter_out).not_to receive(:increment).with(anything)
         subject.multi_filter(events)
       end
     end
@@ -109,8 +121,8 @@ describe LogStash::FilterDelegator do
       end
 
       it "increments the in/out of the metric" do
-        expect(metric).to receive(:increment).with(:in, events.size)
-        expect(metric).to receive(:increment).with(:out, events.size * 2)
+        expect(counter_in).to receive(:increment).with(events.size)
+        expect(counter_out).to receive(:increment).with(events.size * 2)
 
         subject.multi_filter(events)
       end
@@ -138,8 +150,8 @@ describe LogStash::FilterDelegator do
     end
 
     it "increments the in/out of the metric" do
-      expect(metric).to receive(:increment).with(:in, events.size)
-      expect(metric).to receive(:increment).with(:out, events.size)
+      expect(counter_in).to receive(:increment).with(events.size)
+      expect(counter_out).to receive(:increment).with(events.size)
 
       subject.multi_filter(events)
     end

data/spec/logstash/legacy_ruby_event_spec.rb

@@ -399,17 +399,17 @@ describe LogStash::Event do
       end
 
       it "should assign current time when no timestamp" do
-        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of (Time.now.to_i)
+        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(2).of (Time.now.to_i)
       end
 
       it "should tag for invalid value" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
 
         event = LogStash::Event.new("@timestamp" => 666)
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq(666)
       end
@@ -421,7 +421,7 @@ describe LogStash::Event do
 
       it "should tag for invalid string format" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
       end

data/spec/logstash/modules/scaffold_spec.rb

@@ -124,18 +124,13 @@ ERB
       expect(resource2).to be_a(LogStash::Modules::KibanaDashboards)
       expect(resource1.import_path).to eq("api/kibana/settings")
       expect(resource1.content).to be_a(Array)
-      expect(resource1.content.size).to eq(2)
+      expect(resource1.content.size).to eq(1)
 
       test_object = resource1.content[0]
       expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
       expect(test_object.name).to eq("defaultIndex")
       expect(test_object.value).to eq("foo-*")
 
-      test_object = resource1.content[1]
-      expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
-      expect(test_object.name).to eq("metrics:max_buckets")
-      expect(test_object.value).to eq(86400)
-
       expect(resource2.import_path).to eq("api/kibana/dashboards/import")
       expect(resource2.content).to be_a(Array)
       expect(resource2.content.size).to eq(5)
@@ -207,7 +202,7 @@ ERB
       test_module.with_settings(module_settings)
       test_module.import(LogStash::Modules::ElasticsearchImporter.new(client), LogStash::Modules::KibanaImporter.new(kbnclient))
       expect(paths).to eq(expected_paths)
-      expect(contents[0]).to eq({"changes"=>{"defaultIndex"=>"tester-*", "metrics:max_buckets"=>"86400"}})
+      expect(contents[0]).to eq({"changes"=>{"defaultIndex"=>"tester-*"}})
       second_kbn_post = contents[1]
       expect(second_kbn_post[:version]).to eq("9.8.7-6")
       expect(second_kbn_post[:objects]).to be_a(Array)

data/spec/logstash/modules/settings_merger_spec.rb

@@ -0,0 +1,111 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/util/cloud_setting_id"
+require "logstash/util/cloud_setting_auth"
+require "logstash/modules/settings_merger"
+require "logstash/util/password"
+
+class SubstituteSettingsForRSpec
+  def initialize(hash = {}) @hash = hash; end
+  def put(key, value) @hash[key] = value; end
+  def get(key) @hash[key]; end
+end
+
+describe LogStash::Modules::SettingsMerger do
+  describe "#merge" do
+    let(:cli) {[{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]}
+    let(:yml) {[{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]}
+    subject(:results) { described_class.merge(cli, yml) }
+    it "merges cli overwriting any common fields in yml" do
+      expect(results).to be_a(Array)
+      expect(results.size).to eq(3)
+      expect(results[0]["name"]).to eq("mod1")
+      expect(results[0]["var.input.tcp.port"]).to eq("3333")
+      expect(results[0]["var.kibana.username"]).to eq("rupert")
+      expect(results[1]["name"]).to eq("mod2")
+      expect(results[2]["name"]).to eq("mod3")
+      expect(results[2]["var.input.tcp.port"]).to eq(4445)
+    end
+  end
+
+  describe "#merge_cloud_settings" do
+    let(:cloud_id) { LogStash::Util::CloudSettingId.new("label:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }
+    let(:cloud_auth) { LogStash::Util::CloudSettingAuth.new("elastix:bigwhoppingfairytail") }
+    let(:mod_settings) { {} }
+
+    context "when both are supplied" do
+      let(:expected_table) do
+        {
+          "var.kibana.scheme" => "https",
+          "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+          "var.elasticsearch.username" => "elastix",
+          "var.kibana.username" => "elastix"
+        }
+      end
+      let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id, "cloud.auth" => cloud_auth}) }
+
+      before do
+        described_class.merge_cloud_settings(mod_settings, ls_settings)
+      end
+
+      it "adds entries to module settings" do
+        expected_table.each do |key, expected|
+          expect(mod_settings[key]).to eq(expected)
+        end
+        expect(mod_settings["var.elasticsearch.password"].value).to eq("bigwhoppingfairytail")
+        expect(mod_settings["var.kibana.password"].value).to eq("bigwhoppingfairytail")
+      end
+    end
+
+    context "when cloud.id is supplied" do
+      let(:expected_table) do
+        {
+          "var.kibana.scheme" => "https",
+          "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+        }
+      end
+      let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id}) }
+
+      before do
+        described_class.merge_cloud_settings(mod_settings, ls_settings)
+      end
+
+      it "adds entries to module settings" do
+        expected_table.each do |key, expected|
+          expect(mod_settings[key]).to eq(expected)
+        end
+      end
+    end
+
+    context "when only cloud.auth is supplied" do
+      let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.auth" => cloud_auth}) }
+      it "should raise an error" do
+        expect{ described_class.merge_cloud_settings(mod_settings, ls_settings) }.to raise_exception(ArgumentError)
+      end
+    end
+
+    context "when neither cloud.id nor cloud.auth is supplied" do
+      let(:ls_settings) { SubstituteSettingsForRSpec.new() }
+      it "should do nothing" do
+        expect(mod_settings).to be_empty
+      end
+    end
+  end
+
+  describe "#format_module_settings" do
+    let(:before_hash) { {"foo" => "red", "bar" => "blue", "qux" => "pink"} }
+    let(:after_hash) { {"foo" => "red", "bar" => "steel-blue", "baz" => LogStash::Util::Password.new("cyan"), "qux" => nil} }
+    subject(:results) { described_class.format_module_settings(before_hash, after_hash) }
+    it "yields an array of formatted lines for ease of logging" do
+      expect(results.size).to eq(after_hash.size + 2)
+      expect(results.first).to eq("-------- Module Settings ---------")
+      expect(results.last).to eq("-------- Module Settings ---------")
+      expect(results[1]).to eq("foo: 'red'")
+      expect(results[2]).to eq("bar: 'steel-blue', was: 'blue'")
+      expect(results[3]).to eq("baz: '<password>', was: ''")
+      expect(results[4]).to eq("qux: '', was: 'pink'")
+    end
+  end
+end

data/spec/logstash/output_delegator_spec.rb

@@ -5,10 +5,19 @@ require "spec_helper"
 require "support/shared_contexts"
 
 describe LogStash::OutputDelegator do
+
+  class MockGauge
+    def increment(_)
+    end
+  end
+
   let(:logger) { double("logger") }
   let(:events) { 7.times.map { LogStash::Event.new }}
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
   let(:collector) { [] }
+  let(:counter_in) { MockGauge.new }
+  let(:counter_out) { MockGauge.new }
+  let(:counter_time) { MockGauge.new }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
 
   include_context "execution_context"
@@ -23,6 +32,9 @@ describe LogStash::OutputDelegator do
     before(:each) do
       # use the same metric instance
       allow(metric).to receive(:namespace).with(any_args).and_return(metric)
+      allow(metric).to receive(:counter).with(:in).and_return(counter_in)
+      allow(metric).to receive(:counter).with(:out).and_return(counter_out)
+      allow(metric).to receive(:counter).with(:duration_in_millis).and_return(counter_time)
 
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:name).and_return("example")
@@ -58,15 +70,13 @@ describe LogStash::OutputDelegator do
       end
 
       it "should increment the number of events received" do
-        expect(subject.metric_events).to receive(:increment).with(:in, events.length)
-        expect(subject.metric_events).to receive(:increment).with(:out, events.length)
+        expect(counter_in).to receive(:increment).with(events.length)
+        expect(counter_out).to receive(:increment).with(events.length)
         subject.multi_receive(events)
       end
 
       it "should record the `duration_in_millis`" do
-        clock = spy("clock")
-        expect(subject.metric_events).to receive(:time).with(:duration_in_millis).and_return(clock)
-        expect(clock).to receive(:stop)
+        expect(counter_time).to receive(:increment).with(Integer)
         subject.multi_receive(events)
       end
     end