logstash-core 5.0.2-java → 5.1.1.1-java

data/spec/logstash/environment_spec.rb

@@ -53,4 +53,42 @@ describe LogStash::Environment do
       expect($LOAD_PATH).to include(path)
     end
   end
+
+
+  describe "OS detection" do
+    windows_host_os = %w(bccwin cygwin mingw mswin wince)
+    linux_host_os = %w(linux)
+
+    context "windows" do
+      windows_host_os.each do |host|
+        it "#{host} returns true" do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
+          expect(LogStash::Environment.windows?).to be_truthy
+        end
+      end
+
+      linux_host_os.each do |host|
+        it "#{host} returns false" do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
+          expect(LogStash::Environment.windows?).to be_falsey
+        end
+      end
+    end
+
+    context "Linux" do
+      windows_host_os.each do |host|
+        it "#{host} returns true" do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
+          expect(LogStash::Environment.linux?).to be_falsey
+        end
+      end
+
+      linux_host_os.each do |host|
+        it "#{host} returns false" do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
+          expect(LogStash::Environment.linux?).to be_truthy
+        end
+      end
+    end
+  end
 end

data/spec/logstash/event_dispatcher_spec.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+#
+require "logstash/event_dispatcher"
+
+describe LogStash::EventDispatcher do
+  class DummyEmitter
+    attr_reader :dispatcher
+
+    def initialize
+      @dispatcher = LogStash::EventDispatcher.new(self)
+    end
+
+    def method_exists
+      dispatcher.fire(:method_exists)
+    end
+
+    def method_exists_with_arguments(argument1, argument2, argument3)
+      dispatcher.fire(:method_exists_with_arguments, argument1, argument2, argument3)
+    end
+
+    def method_do_not_exist
+      dispatcher.fire(:method_do_not_exist)
+    end
+  end
+
+  class CustomSpy
+    def method_exists
+    end
+
+    def method_exists_with_arguments(argument1, argument2, argument3)
+    end
+  end
+
+  let(:listener) { CustomSpy }
+  subject(:emitter) { DummyEmitter.new }
+
+  describe "Emits events" do
+    before do
+      emitter.dispatcher.add_listener(listener)
+    end
+
+    context "when the method exist" do
+      it "calls the method without arguments" do
+        expect(listener).to receive(:method_exists).with(emitter)
+        emitter.method_exists
+      end
+
+      it "calls the method with arguments" do
+        expect(listener).to receive(:method_exists_with_arguments).with(emitter, 1, 2, 3)
+        emitter.method_exists_with_arguments(1, 2, 3)
+      end
+    end
+
+    context "when the method doesn't exist on the listener" do
+      it "should not raise an exception" do
+        expect { emitter.method_do_not_exist }.not_to raise_error
+      end
+    end
+  end
+
+  describe "Configuring listeners" do
+    it "adds a listener to an emitter" do
+      expect(listener).to receive(:method_exists).with(emitter)
+      emitter.dispatcher.add_listener(listener)
+      emitter.method_exists
+    end
+
+    it "allows to remove a listner to an emitter" do
+      expect(listener).to receive(:method_exists).with(emitter).once
+      emitter.dispatcher.add_listener(listener)
+      emitter.method_exists
+      emitter.dispatcher.remove_listener(listener)
+      emitter.method_exists
+    end
+  end
+end

data/spec/logstash/filter_delegator_spec.rb

@@ -10,7 +10,8 @@ describe LogStash::FilterDelegator do
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
   end
-  let(:metric) { LogStash::Instrument::NullMetric.new }
+  let(:collector) { [] }
+  let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
   let(:events) { [LogStash::Event.new, LogStash::Event.new] }
 
   before :each do

data/spec/logstash/instrument/namespaced_null_metric_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "logstash/instrument/namespaced_null_metric"
+require "logstash/instrument/null_metric"
+require_relative "../../support/matchers"
+require "spec_helper"
+
+describe LogStash::Instrument::NamespacedNullMetric do
+  let(:namespace) { :root }
+  let(:collector) { [] }
+  let(:metric) { LogStash::Instrument::NullMetric.new(collector) }
+
+  subject { described_class.new(metric, namespace) }
+
+  it "defines the same interface as `Metric`" do
+    expect(described_class).to implement_interface_of(LogStash::Instrument::NamespacedMetric)
+  end
+
+  it "returns a TimedException when we call without a block" do
+    expect(subject.time(:duration_ms)).to be(LogStash::Instrument::NullMetric::NullTimedExecution)
+  end
+
+  it "returns the value of the block" do
+    expect(subject.time(:duration_ms) { "hello" }).to eq("hello")
+  end
+
+  it "its doesnt change the original `namespace` when creating a subnamespace" do
+    new_namespace = subject.namespace(:wally)
+
+    expect(subject.namespace_name).to eq([namespace])
+    expect(new_namespace.namespace_name).to eq([:root, :wally])
+  end
+
+end

data/spec/logstash/instrument/null_metric_spec.rb

@@ -2,18 +2,22 @@
 require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_metric"
 require_relative "../../support/shared_examples"
+require_relative "../../support/matchers"
+require "spec_helper"
 
 describe LogStash::Instrument::NullMetric do
-  # This is defined in the `namespaced_metric_spec`
-  include_examples "metrics commons operations"
+
+  let(:key) { "test" }
+  let(:collector) { [] }
+  subject { LogStash::Instrument::NullMetric.new(collector) }
 
   it "defines the same interface as `Metric`" do
-    expect(described_class).to implement_interface_of(LogStash::Instrument::NamespacedMetric)
+    expect(described_class).to implement_interface_of(LogStash::Instrument::Metric)
   end
 
   describe "#namespace" do
-    it "return a NullMetric" do
-      expect(subject.namespace(key)).to be_kind_of LogStash::Instrument::NullMetric
+    it "return a NamespacedNullMetric" do
+      expect(subject.namespace(key)).to be_kind_of LogStash::Instrument::NamespacedNullMetric
     end
   end
 end

data/spec/logstash/instrument/periodic_poller/jvm_spec.rb

@@ -2,6 +2,7 @@
 require "spec_helper"
 require "logstash/instrument/periodic_poller/jvm"
 require "logstash/instrument/collector"
+require "logstash/environment"
 
 describe LogStash::Instrument::PeriodicPoller::JVM::GarbageCollectorName do
   subject { LogStash::Instrument::PeriodicPoller::JVM::GarbageCollectorName }
@@ -27,6 +28,7 @@ describe LogStash::Instrument::PeriodicPoller::JVM::GarbageCollectorName do
   end
 end
 
+
 describe LogStash::Instrument::PeriodicPoller::JVM do
   let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
   let(:options) { {} }
@@ -36,6 +38,22 @@ describe LogStash::Instrument::PeriodicPoller::JVM do
     expect { jvm }.not_to raise_error
   end
 
+  describe "load average" do
+    context "on linux" do
+      context "when an exception occur reading the file" do
+        before do
+          expect(LogStash::Environment).to receive(:windows?).and_return(false)
+          expect(LogStash::Environment).to receive(:linux?).and_return(true)
+          expect(::File).to receive(:read).with("/proc/loadavg").and_raise("Didnt work out so well")
+        end
+
+        it "doesn't raise an exception" do
+          expect { subject.collect }.not_to raise_error
+        end
+      end
+    end
+  end
+
   describe "collections" do
     subject(:collection) { jvm.collect }
     it "should run cleanly" do
@@ -70,6 +88,28 @@ describe LogStash::Instrument::PeriodicPoller::JVM do
           expect(mval(*path)).to be_a(Numeric)
         end
       end
+
+      context "real system" do
+        if LogStash::Environment.linux?
+          context "Linux" do
+            it "returns the load avg" do
+              expect(subject[:process][:cpu][:load_average].value).to include(:"1m" => a_kind_of(Numeric), :"5m" => a_kind_of(Numeric), :"15m" => a_kind_of(Numeric))
+            end
+          end
+        elsif LogStash::Environment.windows?
+          context "Window" do
+            it "returns nothing" do
+              expect(subject[:process][:cpu].has_key?(:load_average)).to be_falsey
+            end
+          end
+        else
+          context "Other" do
+            it "returns 1m only" do
+              expect(subject[:process][:cpu][:load_average].value).to include(:"1m" => a_kind_of(Numeric))
+            end
+          end
+        end
+      end
     end
   end
 end

data/spec/logstash/instrument/periodic_poller/load_average_spec.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/load_average"
+describe LogStash::Instrument::PeriodicPoller::LoadAverage do
+  subject { described_class.create }
+
+  context "on mocked system" do
+    context "on Linux" do
+      before do
+        expect(LogStash::Environment).to receive(:windows?).and_return(false)
+        expect(LogStash::Environment).to receive(:linux?).and_return(true)
+      end
+
+      context "when it can read the file" do
+        let(:proc_loadavg) { "0.00 0.01 0.05 3/180 29727" }
+
+        before do
+          expect(::File).to receive(:read).with("/proc/loadavg").and_return(proc_loadavg)
+        end
+
+        it "return the 3 load average from `/proc/loadavg`" do
+          avg_1m, avg_5m, avg_15m = proc_loadavg.chomp.split(" ")
+
+          expect(subject.get).to include(:"1m" => avg_1m.to_f, :"5m" => avg_5m.to_f, :"15m" => avg_15m.to_f)
+        end
+      end
+    end
+
+    context "on windows" do
+      before do
+        expect(LogStash::Environment).to receive(:windows?).and_return(true)
+      end
+
+      it "Xreturns nil" do
+        expect(subject.get).to be_nil
+      end
+    end
+
+    context "on other" do
+      before do
+        expect(LogStash::Environment).to receive(:windows?).and_return(false)
+        expect(LogStash::Environment).to receive(:linux?).and_return(false)
+      end
+
+      context "when 'OperatingSystemMXBean.getSystemLoadAverage' return something" do
+        let(:load_avg) { 5 }
+
+        before do
+          expect(ManagementFactory).to receive(:getOperatingSystemMXBean).and_return(double("OperatingSystemMXBean", :getSystemLoadAverage => load_avg))
+        end
+
+        it "returns the value" do
+          expect(subject.get).to include(:"1m" => 5)
+        end
+      end
+
+      context "when 'OperatingSystemMXBean.getSystemLoadAverage' doesn't return anything" do
+        before do
+          expect(ManagementFactory).to receive(:getOperatingSystemMXBean).and_return(double("OperatingSystemMXBean", :getSystemLoadAverage => nil))
+        end
+
+        it "returns nothing" do
+          expect(subject.get).to be_nil
+        end
+      end
+    end
+  end
+
+  # Since we are running this on macos and linux I think it make sense to have real test
+  # insteadof only mock
+  context "real system" do
+    if LogStash::Environment.linux?
+      context "Linux" do
+        it "returns the load avg" do
+          expect(subject.get).to include(:"1m" => a_kind_of(Numeric), :"5m" => a_kind_of(Numeric), :"15m" => a_kind_of(Numeric))
+        end
+      end
+    elsif LogStash::Environment.windows?
+      context "window" do
+        it "returns nothing" do
+          expect(subject.get).to be_nil
+        end
+      end
+    else
+      context "Other" do
+        it "returns 1m only" do
+          expect(subject.get).to include(:"1m" => a_kind_of(Numeric))
+        end
+      end
+    end
+  end
+end

data/spec/logstash/output_delegator_spec.rb

@@ -6,7 +6,8 @@ describe LogStash::OutputDelegator do
   let(:logger) { double("logger") }
   let(:events) { 7.times.map { LogStash::Event.new }}
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
-  let(:metric) { LogStash::Instrument::NullMetric.new }
+  let(:collector) { [] }
+  let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
 
   subject { described_class.new(logger, out_klass, metric, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
 

data/spec/logstash/patches_spec.rb

@@ -2,17 +2,18 @@
 require "socket"
 require "logstash/patches"
 require "flores/pki"
+require "logstash/json"
 
 describe "OpenSSL defaults" do
   subject { OpenSSL::SSL::SSLContext.new }
 
-  # OpenSSL::SSL::SSLContext#ciphers returns an array of 
+  # OpenSSL::SSL::SSLContext#ciphers returns an array of
   # [ [ ciphername, version, bits, alg_bits ], [ ... ], ... ]
- 
+
   # List of cipher names
   let(:ciphers) { subject.ciphers.map(&:first) }
 
-  # List of cipher encryption bit strength. 
+  # List of cipher encryption bit strength.
   let(:encryption_bits) { subject.ciphers.map { |_, _, _, a| a } }
 
   it "should not include any export ciphers" do
@@ -35,7 +36,7 @@ describe "OpenSSL defaults" do
     # https://github.com/jordansissel/ruby-flores/blob/master/spec/flores/pki_integration_spec.rb
     # since these helpers were created to fix this particular issue
     let(:csr) { Flores::PKI::CertificateSigningRequest.new }
-    # Here, I use a 1024-bit key for faster tests. 
+    # Here, I use a 1024-bit key for faster tests.
     # Please do not use such small keys in production.
     let(:key_bits) { 1024 }
     let(:key) { OpenSSL::PKey::RSA.generate(key_bits, 65537) }
@@ -88,3 +89,13 @@ describe "OpenSSL defaults" do
     end
   end
 end
+
+describe "exceptions used json logging hashes" do
+  let(:exception) { ArgumentError.new("so you want an argument, huh?") }
+  let(:result) { [] }
+
+  it "should not raise errors" do
+    expect { result << LogStash::Json.dump({"error" => exception}) }.not_to raise_error
+    expect(result[0]).to match(/ArgumentError.*so you want an argument/)
+  end
+end

data/spec/logstash/pipeline_pq_file_spec.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/inputs/generator"
+require "logstash/filters/multiline"
+
+class PipelinePqFileOutput < LogStash::Outputs::Base
+  config_name "pipelinepqfileoutput"
+  milestone 2
+
+  attr_reader :num_closes, :event_count
+
+  def self.make_shared
+    @concurrency = :shared
+  end
+
+  def initialize(params={})
+    super
+    @num_closes = 0
+    @event_count = 0
+    @mutex = Mutex.new
+  end
+
+  def register
+    self.class.make_shared
+  end
+
+  def receive(event)
+    @mutex.synchronize do
+      @event_count = @event_count.succ
+    end
+  end
+
+  def close
+    @num_closes = 1
+  end
+end
+
+describe LogStash::Pipeline do
+  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_id) { "main" }
+
+  let(:multiline_id) { "my-multiline" }
+  let(:output_id) { "my-pipelinepqfileoutput" }
+  let(:generator_id) { "my-generator" }
+  let(:config) do
+    <<-EOS
+    input {
+      generator {
+        count => #{number_of_events}
+        id => "#{generator_id}"
+      }
+    }
+    filter {
+      multiline {
+        id => "#{multiline_id}"
+        pattern => "hello"
+        what => next
+      }
+    }
+    output {
+      pipelinepqfileoutput {
+        id => "#{output_id}"
+      }
+    }
+    EOS
+  end
+
+   let(:pipeline_settings) { { "queue.type" => queue_type, "pipeline.workers" => worker_thread_count, "pipeline.id" => pipeline_id} }
+
+  subject { described_class.new(config, pipeline_settings_obj, metric) }
+
+  let(:counting_output) { PipelinePqFileOutput.new({ "id" => output_id }) }
+  let(:metric_store) { subject.metric.collector.snapshot_metric.metric_store }
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:base_queue_path) { pipeline_settings_obj.get("path.queue") }
+  let(:this_queue_folder) { File.join(base_queue_path, SecureRandom.hex(8)) }
+
+  let(:worker_thread_count) { 8 } # 1 4 8
+  let(:number_of_events) { 100_000 }
+  let(:page_capacity) { 1 * 1024 * 512 } # 1 128
+  let(:max_bytes) { 1024 * 1024 * 1024 } # 1 gb
+  let(:queue_type) { "persisted" } #  "memory" "memory_acked"
+  let(:times) { [] }
+
+  before :each do
+    FileUtils.mkdir_p(this_queue_folder)
+
+    pipeline_settings_obj.set("path.queue", this_queue_folder)
+    allow(PipelinePqFileOutput).to receive(:new).with(any_args).and_return(counting_output)
+    allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+    allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
+    allow(LogStash::Plugin).to receive(:lookup).with("output", "pipelinepqfileoutput").and_return(PipelinePqFileOutput)
+
+    pipeline_workers_setting = LogStash::SETTINGS.get_setting("pipeline.workers")
+    allow(pipeline_workers_setting).to receive(:default).and_return(worker_thread_count)
+    pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
+    pipeline_settings_obj.set("queue.page_capacity", page_capacity)
+    pipeline_settings_obj.set("queue.max_bytes", max_bytes)
+    Thread.new do
+      # make sure we have received all the generated events
+      while counting_output.event_count < number_of_events do
+        sleep 1
+      end
+      subject.shutdown
+    end
+    times.push(Time.now.to_f)
+    subject.run
+    times.unshift(Time.now.to_f - times.first)
+  end
+
+  after :each do
+    # Dir.rm_rf(this_queue_folder)
+  end
+
+  let(:collected_metric) { metric_store.get_with_path("stats/pipelines/") }
+
+  it "populates the pipelines core metrics" do
+    _metric = collected_metric[:stats][:pipelines][:main][:events]
+    expect(_metric[:duration_in_millis].value).not_to be_nil
+    expect(_metric[:in].value).to eq(number_of_events)
+    expect(_metric[:filtered].value).to eq(number_of_events)
+    expect(_metric[:out].value).to eq(number_of_events)
+    STDOUT.puts "  queue.type: #{pipeline_settings_obj.get("queue.type")}"
+    STDOUT.puts "  queue.page_capacity: #{pipeline_settings_obj.get("queue.page_capacity") / 1024}KB"
+    STDOUT.puts "  queue.max_bytes: #{pipeline_settings_obj.get("queue.max_bytes") / 1024}KB"
+    STDOUT.puts "  workers: #{worker_thread_count}"
+    STDOUT.puts "  events: #{number_of_events}"
+    STDOUT.puts "  took: #{times.first}s"
+  end
+end