logstash-core 5.5.3-java → 5.6.0-java

data/spec/logstash/config/string_escape_spec.rb

@@ -0,0 +1,24 @@
+
+require "logstash/config/string_escape"
+
+describe LogStash::Config::StringEscape do
+  let(:result) { described_class.process_escapes(text) }
+
+  table = {
+    '\\"' => '"',
+    "\\'" => "'",
+    "\\n" => "\n",
+    "\\r" => "\r",
+    "\\t" => "\t",
+    "\\\\" => "\\",
+  }
+
+  table.each do |input, expected|
+    context "when processing #{input.inspect}" do
+      let(:text) { input }
+      it "should produce #{expected.inspect}" do
+        expect(result).to be == expected
+      end
+    end
+  end
+end

data/spec/logstash/event_spec.rb

@@ -84,6 +84,15 @@ describe LogStash::Event do
       expect(e.get("foo")).to eq("bar")
     end
 
+    it "should propagate changes to mutable strings to java APIs" do
+      e = LogStash::Event.new()
+      e.to_java.setField("foo", "bar")
+      expect(e.get("foo")).to eq("bar")
+      e.get("foo").gsub!(/bar/, 'pff')
+      expect(e.get("foo")).to eq("pff")
+      expect(e.to_java.getField("foo")).to eq("pff")
+    end
+
     it "should set deep hash values" do
       e = LogStash::Event.new()
       expect(e.set("[foo][bar]", "baz")).to eq("baz")

data/spec/logstash/filter_delegator_spec.rb

@@ -6,12 +6,23 @@ require "logstash/event"
 require "logstash/execution_context"
 
 describe LogStash::FilterDelegator do
+
+  class MockGauge
+    def increment(_)
+    end
+  end
+
+  include_context "execution_context"
+
   let(:logger) { double(:logger) }
   let(:filter_id) { "my-filter" }
   let(:config) do
     { "host" => "127.0.0.1", "id" => filter_id }
   end
   let(:collector) { [] }
+  let(:counter_in) { MockGauge.new }
+  let(:counter_out) { MockGauge.new }
+  let(:counter_time) { MockGauge.new }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
   let(:events) { [LogStash::Event.new, LogStash::Event.new] }
   let(:default_execution_context) { LogStash::ExecutionContext.new(:main, filter_id, "filter",
@@ -19,6 +30,9 @@ describe LogStash::FilterDelegator do
 
   before :each do
     allow(metric).to receive(:namespace).with(anything).and_return(metric)
+    allow(metric).to receive(:counter).with(:in).and_return(counter_in)
+    allow(metric).to receive(:counter).with(:out).and_return(counter_out)
+    allow(metric).to receive(:counter).with(:duration_in_millis).and_return(counter_time)
   end
 
   let(:plugin_klass) do
@@ -58,7 +72,7 @@ describe LogStash::FilterDelegator do
     context "when the flush return events" do
       it "increments the out" do
         subject.multi_filter([LogStash::Event.new])
-        expect(metric).to receive(:increment).with(:out, 1)
+        expect(counter_out).to receive(:increment).with(1)
         subject.flush({})
       end
     end
@@ -76,12 +90,12 @@ describe LogStash::FilterDelegator do
       end
 
       it "has incremented :in" do
-        expect(metric).to receive(:increment).with(:in, events.size)
+        expect(counter_in).to receive(:increment).with(events.size)
         subject.multi_filter(events)
       end
 
       it "has not incremented :out" do
-        expect(metric).not_to receive(:increment).with(:out, anything)
+        expect(counter_out).not_to receive(:increment).with(anything)
         subject.multi_filter(events)
       end
     end
@@ -107,8 +121,8 @@ describe LogStash::FilterDelegator do
       end
 
       it "increments the in/out of the metric" do
-        expect(metric).to receive(:increment).with(:in, events.size)
-        expect(metric).to receive(:increment).with(:out, events.size * 2)
+        expect(counter_in).to receive(:increment).with(events.size)
+        expect(counter_out).to receive(:increment).with(events.size * 2)
 
         subject.multi_filter(events)
       end
@@ -136,8 +150,8 @@ describe LogStash::FilterDelegator do
     end
 
     it "increments the in/out of the metric" do
-      expect(metric).to receive(:increment).with(:in, events.size)
-      expect(metric).to receive(:increment).with(:out, events.size)
+      expect(counter_in).to receive(:increment).with(events.size)
+      expect(counter_out).to receive(:increment).with(events.size)
 
       subject.multi_filter(events)
     end

data/spec/logstash/instrument/periodic_poller/dlq_spec.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/instrument/periodic_poller/dlq"
+require "logstash/instrument/collector"
+
+describe LogStash::Instrument::PeriodicPoller::DeadLetterQueue do
+  subject { LogStash::Instrument::PeriodicPoller::DeadLetterQueue }
+
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:agent) { double("agent")}
+  let(:options) { {} }
+  subject(:dlq) { described_class.new(metric, agent, options) }
+
+  it "should initialize cleanly" do
+    expect { dlq }.not_to raise_error
+  end
+end

data/spec/logstash/instrument/periodic_poller/jvm_spec.rb

@@ -44,7 +44,7 @@ describe LogStash::Instrument::PeriodicPoller::JVM do
         before do
           expect(LogStash::Environment).to receive(:windows?).and_return(false)
           expect(LogStash::Environment).to receive(:linux?).and_return(true)
-          expect(::File).to receive(:read).with("/proc/loadavg").and_raise("Didnt work out so well")
+          expect(::File).to receive(:read).with("/proc/loadavg").at_least(:once).and_raise("Didnt work out so well")
         end
 
         it "doesn't raise an exception" do

data/spec/logstash/legacy_ruby_event_spec.rb

@@ -399,17 +399,17 @@ describe LogStash::Event do
       end
 
       it "should assign current time when no timestamp" do
-        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of (Time.now.to_i)
+        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(2).of (Time.now.to_i)
       end
 
       it "should tag for invalid value" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
 
         event = LogStash::Event.new("@timestamp" => 666)
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq(666)
       end
@@ -421,7 +421,7 @@ describe LogStash::Event do
 
       it "should tag for invalid string format" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
       end

data/spec/logstash/modules/logstash_config_spec.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "logstash/modules/logstash_config"
+
+describe LogStash::Modules::LogStashConfig do
+  let(:mod) { instance_double("Modules", :directory => Stud::Temporary.directory, :module_name => "testing") }
+  let(:settings) { {"var.logstash.testing.pants" => "fancy" }}
+  subject { described_class.new(mod, settings) }
+
+  describe "configured inputs" do
+    context "when no inputs is send" do
+      it "returns the default" do
+        expect(subject.configured_inputs(["kafka"])).to include("kafka")
+      end
+    end
+
+    context "when inputs are send" do
+      let(:settings) { { "var.inputs" => "tcp" } }
+
+      it "returns the configured inputs" do
+        expect(subject.configured_inputs(["kafka"])).to include("tcp")
+      end
+
+      context "when alias is specified" do
+        let(:settings) { { "var.inputs" => "smartconnector" } }
+
+        it "returns the configured inputs" do
+          expect(subject.configured_inputs(["kafka"], { "smartconnector" => "tcp"  })).to include("tcp", "smartconnector")
+        end
+      end
+    end
+  end
+
+  describe "array to logstash array string" do
+    it "return an escaped string" do
+      expect(subject.array_to_string(["hello", "ninja"])).to eq("['hello', 'ninja']")
+    end
+  end
+
+  describe "alias modules options" do
+    let(:alias_table) do
+      { "var.logstash.testing" => "var.logstash.better" }
+    end
+
+    before do
+      subject.alias_settings_keys!(alias_table)
+    end
+
+    it "allow to retrieve settings" do
+      expect(subject.setting("var.logstash.better.pants", "dont-exist")).to eq("fancy")
+    end
+
+    it "allow to retrieve settings with the original name" do
+      expect(subject.setting("var.logstash.testing.pants", "dont-exist")).to eq("fancy")
+    end
+  end
+end

data/spec/logstash/modules/scaffold_spec.rb

@@ -0,0 +1,234 @@
+# encoding: utf-8
+#
+require "logstash/namespace"
+require "logstash/elasticsearch_client"
+require "logstash/modules/kibana_client"
+require "logstash/modules/kibana_config"
+require "logstash/modules/scaffold"
+require "logstash/modules/elasticsearch_importer"
+require "logstash/modules/kibana_importer"
+
+require_relative "../../support/helpers"
+
+describe LogStash::Modules::Scaffold do
+  let(:base_dir) { "gem-home" }
+  let(:mname) { "foo" }
+  subject(:test_module) { described_class.new(mname, base_dir) }
+  let(:module_settings) do
+    {
+      "var.elasticsearch.hosts" => "es.mycloud.com:9200",
+      "var.elasticsearch.user" => "foo",
+      "var.elasticsearch.password" => "password",
+      "var.input.tcp.port" => 5606,
+    }
+  end
+  let(:dashboard_hash) do
+    {
+      "hits" => 0,
+      "timeRestore" => false,
+      "description" => "",
+      "title" => "Filebeat Apache2 Dashboard",
+      "uiStateJSON" => "{}",
+      "panelsJSON" => '[{"col":1,"id":"foo-c","panelIndex":1,"row":1,"size_x":12,"size_y":3,"type":"visualization"},{"id":"foo-d","type":"search","panelIndex":7,"size_x":12,"size_y":3,"col":1,"row":11,"columns":["apache2.error.client","apache2.error.level","apache2.error.module","apache2.error.message"],"sort":["@timestamp","desc"]}]',
+      "optionsJSON" => "{}",
+      "version" => 1,
+      "kibanaSavedObjectMeta" => {
+        "searchSourceJSON" => "{}"
+      }
+    }
+  end
+  let(:viz_hash) do
+    {
+      "visState" => "",
+      "description" => "",
+      "title" => "foo-c",
+      "uiStateJSON" => "",
+      "version" => 1,
+      "savedSearchId" => "foo-e",
+      "kibanaSavedObjectMeta" => {}
+    }
+  end
+  let(:index_pattern_hash) do
+    {
+      "title" => "foo-*",
+      "timeFieldName" =>"time",
+      "fieldFormatMap" => "{some map}",
+      "fields" => "[some array]"
+    }
+  end
+  context "logstash operation" do
+    let(:ls_conf) do
+<<-ERB
+input {
+  tcp {
+    port => <%= setting("var.input.tcp.port", 45) %>
+    host => <%= setting("var.input.tcp.host", "localhost") %>
+    type => <%= setting("var.input.tcp.type", "server") %>
+  }
+}
+filter {
+
+}
+output {
+  <%= elasticsearch_output_config() %>
+}
+ERB
+    end
+
+    before do
+      allow(LogStash::Modules::FileReader).to receive(:read).and_return(ls_conf)
+    end
+
+    it "provides a logstash config" do
+      expect(test_module.logstash_configuration).to be_nil
+      test_module.with_settings(module_settings)
+      expect(test_module.logstash_configuration).not_to be_nil
+      config_string = test_module.config_string
+      expect(config_string).to include("port => 5606")
+      expect(config_string).to include("hosts => ['es.mycloud.com:9200']")
+    end
+  end
+
+  context "elasticsearch operation" do
+    it "provides the elasticsearch mapping file paths" do
+      test_module.with_settings(module_settings)
+      expect(test_module.elasticsearch_configuration).not_to be_nil
+      files = test_module.elasticsearch_configuration.resources
+      expect(files.size).to eq(1)
+      expect(files.first).to be_a(LogStash::Modules::ElasticsearchResource)
+      expect(files.first.content_path).to eq("gem-home/elasticsearch/foo.json")
+      expect(files.first.import_path).to eq("_template/foo")
+    end
+  end
+
+  context "kibana operation" do
+    before do
+      # allow(LogStash::Modules::FileReader).to receive(:read_json).and_return({})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/dashboard/foo.json").and_return(["Foo-Dashboard"])
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/dashboard/Foo-Dashboard.json").and_return(dashboard_hash)
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/visualization/foo-c.json").and_return(viz_hash)
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/search/foo-d.json").and_return({"d" => "search"})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/search/foo-e.json").and_return({"e" => "search"})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/index-pattern/foo.json").and_return(index_pattern_hash)
+    end
+
+    it "provides a list of importable files" do
+      expect(test_module.kibana_configuration).to be_nil
+      test_module.with_settings(module_settings)
+      expect(test_module.kibana_configuration).not_to be_nil
+      resources = test_module.kibana_configuration.resources
+      expect(resources.size).to eq(2)
+      resource1 = resources[0]
+      resource2 = resources[1]
+      expect(resource1).to be_a(LogStash::Modules::KibanaSettings)
+      expect(resource2).to be_a(LogStash::Modules::KibanaDashboards)
+      expect(resource1.import_path).to eq("api/kibana/settings")
+      expect(resource1.content).to be_a(Array)
+      expect(resource1.content.size).to eq(1)
+
+      test_object = resource1.content[0]
+      expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
+      expect(test_object.name).to eq("defaultIndex")
+      expect(test_object.value).to eq("foo-*")
+
+      expect(resource2.import_path).to eq("api/kibana/dashboards/import")
+      expect(resource2.content).to be_a(Array)
+      expect(resource2.content.size).to eq(5)
+      expect(resource2.content.map{|o| o.class}.uniq).to eq([LogStash::Modules::KibanaResource])
+
+      test_object = resource2.content[0]
+      expect(test_object.content_id).to eq("foo-*")
+      expect(test_object.content_type).to eq("index-pattern")
+      expect(test_object.content_as_object).to eq(index_pattern_hash)
+
+      test_object = resource2.content[1]
+      expect(test_object.content_id).to eq("Foo-Dashboard")
+      expect(test_object.content_type).to eq("dashboard")
+      expect(test_object.content_as_object).to eq(dashboard_hash)
+
+      test_object = resource2.content[2]
+      expect(test_object.content_id).to eq("foo-c") #<- the panels can contain items from other folders
+      expect(test_object.content_type).to eq("visualization")
+      expect(test_object.content_as_object).to eq(viz_hash)
+      expect(test_object.content_as_object["savedSearchId"]).to eq("foo-e")
+
+      test_object = resource2.content[3]
+      expect(test_object.content_id).to eq("foo-d") #<- the panels can contain items from other folders
+      expect(test_object.content_type).to eq("search")
+      expect(test_object.content_as_object).to eq("d"=>"search")
+
+      test_object = resource2.content[4]
+      expect(test_object.content_id).to eq("foo-e") # <- the visualization can contain items from the search folder
+      expect(test_object.content_type).to eq("search")
+      expect(test_object.content_as_object).to eq("e"=>"search")
+    end
+  end
+
+  context "importing to elasticsearch stubbed client" do
+    let(:mname) { "tester" }
+    let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "modules", "#{mname}", "configuration")) }
+    let(:response) { double(:response) }
+    let(:client) { double(:client) }
+    let(:kbnclient) { double(:kbnclient) }
+    let(:paths) { [] }
+    let(:expected_paths) { ["_template/tester", "api/kibana/settings", "api/kibana/dashboards/import"] }
+    let(:contents) { [] }
+    let(:expected_objects) do
+      [
+        "index-pattern tester-*",
+        "dashboard FW-Dashboard",
+        "visualization FW-Viz-1",
+        "visualization FW-Viz-2",
+        "search Search-Tester"
+      ]
+    end
+
+    before do
+      allow(response).to receive(:status).and_return(404)
+      allow(client).to receive(:head).and_return(response)
+      allow(kbnclient).to receive(:version).and_return("9.8.7-6")
+    end
+
+    it "calls the import method" do
+      expect(client).to receive(:put).once do |path, content|
+        paths << path
+        LogStash::ElasticsearchClient::Response.new(201, "", {})
+      end
+      expect(kbnclient).to receive(:post).twice do |path, content|
+        paths << path
+        contents << content
+        LogStash::Modules::KibanaClient::Response.new(201, "", {})
+      end
+      test_module.with_settings(module_settings)
+      test_module.import(LogStash::Modules::ElasticsearchImporter.new(client), LogStash::Modules::KibanaImporter.new(kbnclient))
+      expect(paths).to eq(expected_paths)
+      expect(contents[0]).to eq({"changes"=>{"defaultIndex"=>"tester-*"}})
+      second_kbn_post = contents[1]
+      expect(second_kbn_post[:version]).to eq("9.8.7-6")
+      expect(second_kbn_post[:objects]).to be_a(Array)
+      expect(second_kbn_post[:objects].size).to eq(5)
+      objects_types_ids = second_kbn_post[:objects].map {|h| "#{h["type"]} #{h["id"]}"}
+      expect(objects_types_ids).to eq(expected_objects)
+    end
+  end
+
+  context "import 4 realz", :skip => "integration" do
+    let(:mname) { "cef" }
+    let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "#{mname}")) }
+    let(:module_settings) do
+      {
+        "var.elasticsearch.hosts" => "localhost:9200",
+        "var.elasticsearch.user" => "foo",
+        "var.elasticsearch.password" => "password",
+        "var.input.tcp.port" => 5606,
+      }
+    end
+    it "puts stuff in ES" do
+      test_module.with_settings(module_settings)
+      client = LogStash::ElasticsearchClient.build(module_settings)
+      import_engine = LogStash::Modules::Importer.new(client)
+      test_module.import(import_engine)
+      expect(1).to eq(1)
+    end
+  end
+end