logstash-core 5.3.3-java → 5.4.0-java

data/spec/logstash/legacy_ruby_timestamp_spec.rb

@@ -0,0 +1,170 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/timestamp"
+require "bigdecimal"
+
+describe LogStash::Timestamp do
+
+  it "should parse its own iso8601 output" do
+    t = Time.now
+    ts = LogStash::Timestamp.new(t)
+    expect(LogStash::Timestamp.parse_iso8601(ts.to_iso8601).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce iso8601 string" do
+    t = Time.now
+    ts = LogStash::Timestamp.new(t)
+    expect(LogStash::Timestamp.coerce(ts.to_iso8601).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce Time" do
+    t = Time.now
+    expect(LogStash::Timestamp.coerce(t).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce Timestamp" do
+    t = LogStash::Timestamp.now
+    expect(LogStash::Timestamp.coerce(t).to_i).to eq(t.to_i)
+  end
+
+  it "should raise on invalid string coerce" do
+    expect{LogStash::Timestamp.coerce("foobar")}.to raise_error LogStash::TimestampParserError
+  end
+
+  it "should return nil on invalid object coerce" do
+    expect(LogStash::Timestamp.coerce(:foobar)).to be_nil
+  end
+
+  it "should support to_json" do
+    expect(LogStash::Timestamp.parse_iso8601("2014-09-23T00:00:00-0800").to_json).to eq("\"2014-09-23T08:00:00.000Z\"")
+  end
+
+  it "should support to_json and ignore arguments" do
+    expect(LogStash::Timestamp.parse_iso8601("2014-09-23T00:00:00-0800").to_json(:some => 1, :arguments => "test")).to eq("\"2014-09-23T08:00:00.000Z\"")
+  end
+
+  it "should support timestamp comparison" do
+   current = LogStash::Timestamp.new(Time.now) 
+   future = LogStash::Timestamp.new(Time.now + 100)
+
+   expect(future > current).to eq(true)
+   expect(future < current).to eq(false)
+   expect(current == current).to eq(true)
+
+   expect(current <=> current).to eq(0)
+   expect(current <=> future).to eq(-1)
+   expect(future <=> current).to eq(1)
+  end
+
+  it "should allow unary operation +" do
+    current = Time.now
+    t = LogStash::Timestamp.new(current) + 10
+    expect(t).to eq(current + 10)
+  end
+
+  describe "subtraction" do
+    it "should work on a timestamp object" do
+      t = Time.now
+      current = LogStash::Timestamp.new(t)
+      future = LogStash::Timestamp.new(t + 10)
+      expect(future - current).to eq(10)
+    end
+
+    it "should work on with time object" do
+      current = Time.now
+      t = LogStash::Timestamp.new(current + 10)
+      expect(t - current).to eq(10)
+    end
+
+    it "should work with numeric value" do
+      current = Time.now
+      t = LogStash::Timestamp.new(current + 10)
+      expect(t - 10).to eq(current)
+    end
+  end
+
+  context "identity methods" do
+    subject { LogStash::Timestamp.new }
+
+    it "should support utc" do
+      expect(subject.utc).to eq(subject)
+    end
+
+    it "should support gmtime" do
+      expect(subject.gmtime).to eq(subject)
+    end
+  end
+
+  context "numeric casting methods" do
+    let (:now) {Time.now}
+    subject { LogStash::Timestamp.new(now) }
+
+    it "should support to_i" do
+      expect(subject.to_i).to eq(now.to_i)
+    end
+
+    it "should support to_f" do
+      expect(subject.to_f).to eq(now.to_f)
+    end
+  end
+
+  context "at" do
+    context "with integer epoch" do
+      it "should convert to correct date" do
+        expect(LogStash::Timestamp.at(946702800).to_iso8601).to eq("2000-01-01T05:00:00.000Z")
+      end
+
+      it "should return zero usec" do
+        expect(LogStash::Timestamp.at(946702800).usec).to eq(0)
+      end
+
+      it "should return prior to epoch date on negative input" do
+        expect(LogStash::Timestamp.at(-1).to_iso8601).to eq("1969-12-31T23:59:59.000Z")
+      end
+    end
+
+    context "with float epoch" do
+      it "should convert to correct date" do
+        expect(LogStash::Timestamp.at(946702800.123456.to_f).to_iso8601).to eq("2000-01-01T05:00:00.123Z")
+      end
+
+      it "should return usec with a minimum of millisec precision" do
+        expect(LogStash::Timestamp.at(946702800.123456.to_f).usec).to be_within(1000).of(123456)
+      end
+    end
+
+    context "with BigDecimal epoch" do
+      it "should convert to correct date" do
+        expect(LogStash::Timestamp.at(BigDecimal.new("946702800.123456")).to_iso8601).to eq("2000-01-01T05:00:00.123Z")
+      end
+
+      it "should return usec with a minimum of millisec precision" do
+        # since Java Timestamp relies on JodaTime which supports only milliseconds precision
+        # the usec method will only be precise up to milliseconds.
+        expect(LogStash::Timestamp.at(BigDecimal.new("946702800.123456")).usec).to be_within(1000).of(123456)
+      end
+    end
+
+    context "with illegal parameters" do
+      it "should raise exception on nil input" do
+        expect{LogStash::Timestamp.at(nil)}.to raise_error
+      end
+
+      it "should raise exception on invalid input type" do
+        expect{LogStash::Timestamp.at(:foo)}.to raise_error
+      end
+    end
+  end
+
+  context "usec" do
+    it "should support millisecond precision" do
+      expect(LogStash::Timestamp.at(946702800.123).usec).to eq(123000)
+    end
+
+    it "should try to preserve and report microseconds precision if possible" do
+      # since Java Timestamp relies on JodaTime which supports only milliseconds precision
+      # the usec method will only be precise up to milliseconds.
+      expect(LogStash::Timestamp.at(946702800.123456).usec).to be_within(1000).of(123456)
+    end
+  end
+end

data/spec/logstash/output_delegator_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/output_delegator"
-require 'spec_helper'
+require "logstash/execution_context"
+require "spec_helper"
 
 describe LogStash::OutputDelegator do
   let(:logger) { double("logger") }
@@ -8,8 +9,9 @@ describe LogStash::OutputDelegator do
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
   let(:collector) { [] }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
+  let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
 
-  subject { described_class.new(logger, out_klass, metric, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
+  subject { described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
 
   context "with a plain output plugin" do
     let(:out_klass) { double("output klass") }
@@ -27,6 +29,7 @@ describe LogStash::OutputDelegator do
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)
+      allow(out_inst).to receive(:execution_context=).with(default_execution_context)
       allow(out_inst).to receive(:id).and_return("a-simple-plugin")
 
       allow(subject.metric_events).to receive(:increment).with(any_args)
@@ -39,7 +42,7 @@ describe LogStash::OutputDelegator do
 
     it "should push the name of the plugin to the metric" do
       expect(metric).to receive(:gauge).with(:name, out_klass.config_name)
-      described_class.new(logger, out_klass, metric, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
+      described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
     end
 
     context "after having received a batch of events" do

data/spec/logstash/outputs/base_spec.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/outputs/base"
+require "logstash/execution_context"
 
 # use a dummy NOOP output to test Outputs::Base
 class LogStash::Outputs::NOOPSingle < LogStash::Outputs::Base
@@ -77,6 +79,27 @@ describe "LogStash::Outputs::Base#new" do
     end
   end
 
+  context "execution context" do
+    let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
+    let(:klass) { LogStash::Outputs::NOOPSingle }
+
+    subject(:instance) { klass.new(params.dup) }
+
+    it "allow to set the context" do
+      expect(instance.execution_context).to be_nil
+      instance.execution_context = default_execution_context
+
+      expect(instance.execution_context).to eq(default_execution_context)
+    end
+
+    it "propagate the context to the codec" do
+      expect(instance.codec.execution_context).to be_nil
+      instance.execution_context = default_execution_context
+
+      expect(instance.codec.execution_context).to eq(default_execution_context)
+    end
+  end
+
   describe "dispatching multi_receive" do
     let(:event) { double("event") }
     let(:events) { [event] }

data/spec/logstash/pipeline_pq_file_spec.rb

@@ -82,6 +82,13 @@ describe LogStash::Pipeline do
   let(:queue_type) { "persisted" } #  "memory" "memory_acked"
   let(:times) { [] }
 
+  let(:pipeline_thread) do
+    # subject has to be called for the first time outside the thread because it will create a race condition
+    # with the subject.ready? call since subject is lazily initialized
+    s = subject
+    Thread.new { s.run }
+  end
+
   before :each do
     FileUtils.mkdir_p(this_queue_folder)
 
@@ -97,19 +104,22 @@ describe LogStash::Pipeline do
     pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
     pipeline_settings_obj.set("queue.page_capacity", page_capacity)
     pipeline_settings_obj.set("queue.max_bytes", max_bytes)
-    Thread.new do
-      # make sure we have received all the generated events
-      while counting_output.event_count < number_of_events do
-        sleep 1
-      end
-      subject.shutdown
-    end
     times.push(Time.now.to_f)
-    subject.run
+
+    pipeline_thread
+    sleep(0.1) until subject.ready?
+
+    # make sure we have received all the generated events
+    while counting_output.event_count < number_of_events do
+      sleep(0.5)
+    end
+
     times.unshift(Time.now.to_f - times.first)
   end
 
   after :each do
+    subject.shutdown
+    pipeline_thread.join
     # Dir.rm_rf(this_queue_folder)
   end
 

data/spec/logstash/pipeline_spec.rb

@@ -3,6 +3,7 @@ require "spec_helper"
 require "logstash/inputs/generator"
 require "logstash/filters/multiline"
 require_relative "../support/mocks_classes"
+require_relative "../logstash/pipeline_reporter_spec" # for DummyOutput class
 
 class DummyInput < LogStash::Inputs::Base
   config_name "dummyinput"
@@ -133,7 +134,7 @@ describe LogStash::Pipeline do
       CONFIG
     end
 
-    it "should not propage cancelled events from filter to output" do
+    it "should not propagate cancelled events from filter to output" do
       abort_on_exception_state = Thread.abort_on_exception
       Thread.abort_on_exception = true
 
@@ -249,7 +250,6 @@ describe LogStash::Pipeline do
       }
 
       it "starts multiple filter threads" do
-        skip("This test has been failing periodically since November 2016. Tracked as https://github.com/elastic/logstash/issues/6245")
         pipeline = TestPipeline.new(test_config_with_filters)
         pipeline.run
         expect(pipeline.worker_threads.size).to eq(worker_thread_count)
@@ -389,7 +389,7 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
       allow(logger).to receive(:warn)
 
-      # pipeline must be first called outside the thread context because it lazyly initialize and will create a
+      # pipeline must be first called outside the thread context because it lazily initialize and will create a
       # race condition if called in the thread
       p = pipeline
       t = Thread.new { p.run }
@@ -411,7 +411,7 @@ describe LogStash::Pipeline do
     end
   end
 
-  context "compiled filter funtions" do
+  context "compiled filter functions" do
     context "new events should propagate down the filters" do
       config <<-CONFIG
         filter {
@@ -630,7 +630,7 @@ describe LogStash::Pipeline do
 
     it "should handle evaluating different config" do
       # When the functions are compiled from the AST it will generate instance
-      # variables that are unique to the actual config, the intances are pointing
+      # variables that are unique to the actual config, the instances are pointing
       # to conditionals and/or plugins.
       #
       # Before the `defined_singleton_method`, the definition of the method was
@@ -854,4 +854,40 @@ describe LogStash::Pipeline do
        expect(pipeline1.instance_variables).to eq(pipeline2.instance_variables)
     end
   end
+
+  describe "#system?" do
+    after do
+      pipeline.close # close the queue
+    end
+
+    let(:pipeline) { LogStash::Pipeline.new(config_string, settings) }
+    let(:config_string) { "input { generator {} } output { null {} }" }
+
+    context "when the pipeline is a system pipeline" do
+      let(:settings) do
+        s = LogStash::SETTINGS.clone
+        s.set("pipeline.system", true)
+        s.set("config.string", config_string)
+        s
+      end
+
+
+      it "returns true" do
+        expect(pipeline.system?).to be_truthy
+      end
+    end
+
+    context "when the pipeline is not a system pipeline" do
+      let(:settings) do
+        s = LogStash::SETTINGS.clone
+        s.set("pipeline.system", false)
+        s.set("config.string", config_string)
+        s
+      end
+
+      it "returns true" do
+        expect(pipeline.system?).to be_falsey
+      end
+    end
+  end
 end

data/spec/logstash/plugin_spec.rb

@@ -5,6 +5,7 @@ require "logstash/outputs/base"
 require "logstash/codecs/base"
 require "logstash/inputs/base"
 require "logstash/filters/base"
+require "logstash/execution_context"
 
 describe LogStash::Plugin do
   context "reloadable" do
@@ -42,12 +43,23 @@ describe LogStash::Plugin do
     end
   end
 
-  it "should fail lookup on inexisting type" do
+  context "#execution_context" do
+    subject { Class.new(LogStash::Plugin).new({}) }
+
+    it "can be set and get" do
+      expect(subject.execution_context).to be_nil
+      execution_context = LogStash::ExecutionContext.new(:main)
+      subject.execution_context = execution_context
+      expect(subject.execution_context).to eq(execution_context)
+    end
+  end
+
+  it "should fail lookup on nonexistent type" do
     #expect_any_instance_of(Cabin::Channel).to receive(:debug).once
     expect { LogStash::Plugin.lookup("badbadtype", "badname") }.to raise_error(LogStash::PluginLoadingError)
   end
 
-  it "should fail lookup on inexisting name" do
+  it "should fail lookup on nonexistent name" do
     #expect_any_instance_of(Cabin::Channel).to receive(:debug).once
     expect { LogStash::Plugin.lookup("filter", "badname") }.to raise_error(LogStash::PluginLoadingError)
   end
@@ -217,7 +229,7 @@ describe LogStash::Plugin do
           config_name "simple_plugin"
 
           config :host, :validate => :string
-          config :export, :validte => :boolean
+          config :export, :validate => :boolean
 
           def register; end
         end

data/spec/logstash/plugins/hooks_registry_spec.rb

@@ -35,12 +35,12 @@ describe LogStash::Plugins::HooksRegistry do
   let(:listener) { DummyListener.new }
 
   it "allow to register an emitter" do
-    expect { subject.register_emitter(emitter.class, emitter.dispatcher) }.to change { subject.emmitters_count }.by(1)
+    expect { subject.register_emitter(emitter.class, emitter.dispatcher) }.to change { subject.emitters_count }.by(1)
   end
 
   it "allow to remove an emitter" do
     subject.register_emitter(emitter.class, emitter.dispatcher)
-    expect { subject.remove_emitter(emitter.class)}.to change { subject.emmitters_count }.by(-1)
+    expect { subject.remove_emitter(emitter.class)}.to change { subject.emitters_count }.by(-1)
   end
 
   it "allow to register hooks to emitters" do

data/spec/logstash/runner_spec.rb

@@ -141,7 +141,7 @@ describe LogStash::Runner do
 
     context "with a good configuration" do
       let(:pipeline_string) { "input { } filter { } output { }" }
-      it "should exit successfuly" do
+      it "should exit successfully" do
         expect(subject.run(args)).to eq(0)
       end
     end
@@ -165,7 +165,38 @@ describe LogStash::Runner do
       allow(pipeline).to receive(:run).and_return(task)
       allow(pipeline).to receive(:shutdown)
     end
-
+    
+    context "when :path.data is defined by the user" do
+      let(:test_data_path) { "/tmp/ls-test-data" }
+      let(:test_queue_path) { test_data_path + "/" + "queue" }
+      
+      it "should set data paths" do
+        expect(LogStash::Agent).to receive(:new) do |settings|
+          expect(settings.get("path.data")).to eq(test_data_path)
+          expect(settings.get("path.queue")).to eq(test_queue_path)
+        end
+        
+        args = ["--path.data", test_data_path, "-e", pipeline_string]
+        subject.run("bin/logstash", args)
+      end
+      
+      context "and path.queue is manually set" do
+        let(:queue_override_path) { "/tmp/queue-override_path" }
+        
+        it "should set data paths" do
+          expect(LogStash::Agent).to receive(:new) do |settings|
+            expect(settings.get("path.data")).to eq(test_data_path)
+            expect(settings.get("path.queue")).to eq(queue_override_path)
+          end
+          
+          LogStash::SETTINGS.set("path.queue", queue_override_path)
+          
+          args = ["--path.data", test_data_path, "-e", pipeline_string]
+          subject.run("bin/logstash", args)
+        end
+      end
+    end
+    
     context "when :http.host is defined by the user" do
       it "should pass the value to the webserver" do
         expect(LogStash::Agent).to receive(:new) do |settings|