logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/spec/logstash/instrument/periodic_poller/dlq_spec.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/instrument/periodic_poller/dlq"
+require "logstash/instrument/collector"
+
+describe LogStash::Instrument::PeriodicPoller::DeadLetterQueue do
+  subject { LogStash::Instrument::PeriodicPoller::DeadLetterQueue }
+
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:agent) { double("agent")}
+  let(:options) { {} }
+  subject(:dlq) { described_class.new(metric, agent, options) }
+
+  it "should initialize cleanly" do
+    expect { dlq }.not_to raise_error
+  end
+end

data/spec/logstash/instrument/periodic_poller/jvm_spec.rb

@@ -44,7 +44,7 @@ describe LogStash::Instrument::PeriodicPoller::JVM do
         before do
           expect(LogStash::Environment).to receive(:windows?).and_return(false)
           expect(LogStash::Environment).to receive(:linux?).and_return(true)
-          expect(::File).to receive(:read).with("/proc/loadavg").and_raise("Didnt work out so well")
+          expect(::File).to receive(:read).with("/proc/loadavg").at_least(:once).and_raise("Didnt work out so well")
         end
 
         it "doesn't raise an exception" do

data/spec/logstash/legacy_ruby_event_spec.rb

@@ -590,15 +590,6 @@ describe LogStash::Event do
   describe "#to_s" do
     let(:timestamp) { LogStash::Timestamp.new }
     let(:event1) { LogStash::Event.new({ "@timestamp" => timestamp, "host" => "foo", "message" => "bar"}) }
-    let(:event2) { LogStash::Event.new({ "host" => "bar", "message" => "foo"}) }
-
-    it "should cache only one template" do
-      LogStash::StringInterpolation.clear_cache
-      expect {
-        event1.to_s
-        event2.to_s
-      }.to change { LogStash::StringInterpolation.cache_size }.by(1)
-    end
 
     it "return the string containing the timestamp, the host and the message" do
       expect(event1.to_s).to eq("#{timestamp.to_iso8601} #{event1.get("host")} #{event1.get("message")}")

data/spec/logstash/legacy_ruby_timestamp_spec.rb

@@ -4,7 +4,12 @@ require "logstash/timestamp"
 require "bigdecimal"
 
 describe LogStash::Timestamp do
-
+  # Via JRuby 9k time see logstash/issues/7463
+  # JRuby 9k now uses Java 8 Time with nanosecond precision but
+  # our Timestamp use Joda with millisecond precision
+  #        expected: 10
+  #             got: 9.999000001
+  # we may need to use `be_within(0.000999999).of()` in other places too
   it "should parse its own iso8601 output" do
     t = Time.now
     ts = LogStash::Timestamp.new(t)
@@ -12,7 +17,7 @@ describe LogStash::Timestamp do
   end
 
   it "should coerce iso8601 string" do
-    t = Time.now
+    t = DateTime.now.to_time
     ts = LogStash::Timestamp.new(t)
     expect(LogStash::Timestamp.coerce(ts.to_iso8601).to_i).to eq(t.to_i)
   end
@@ -44,7 +49,7 @@ describe LogStash::Timestamp do
   end
 
   it "should support timestamp comparison" do
-   current = LogStash::Timestamp.new(Time.now) 
+   current = LogStash::Timestamp.new(Time.now)
    future = LogStash::Timestamp.new(Time.now + 100)
 
    expect(future > current).to eq(true)
@@ -57,29 +62,29 @@ describe LogStash::Timestamp do
   end
 
   it "should allow unary operation +" do
-    current = Time.now
+    current = DateTime.now.to_time
     t = LogStash::Timestamp.new(current) + 10
-    expect(t).to eq(current + 10)
+    expect(t).to be_within(0.000999999).of(current + 10)
   end
 
   describe "subtraction" do
     it "should work on a timestamp object" do
-      t = Time.now
+      t = DateTime.now.to_time
       current = LogStash::Timestamp.new(t)
       future = LogStash::Timestamp.new(t + 10)
-      expect(future - current).to eq(10)
+      expect(future - current).to be_within(0.000999999).of(10)
     end
 
     it "should work on with time object" do
-      current = Time.now
+      current = DateTime.now.to_time
       t = LogStash::Timestamp.new(current + 10)
-      expect(t - current).to eq(10)
+      expect(t - current).to be_within(0.000999999).of(10)
     end
 
     it "should work with numeric value" do
-      current = Time.now
+      current = DateTime.now.to_time
       t = LogStash::Timestamp.new(current + 10)
-      expect(t - 10).to eq(current)
+      expect(t - 10).to be_within(0.000999999).of(current)
     end
   end
 
@@ -96,15 +101,15 @@ describe LogStash::Timestamp do
   end
 
   context "numeric casting methods" do
-    let (:now) {Time.now}
+    let(:now) { Time.now }
     subject { LogStash::Timestamp.new(now) }
 
     it "should support to_i" do
-      expect(subject.to_i).to eq(now.to_i)
+      expect(subject.to_i).to be_kind_of(Integer)
     end
 
     it "should support to_f" do
-      expect(subject.to_f).to eq(now.to_f)
+      expect(subject.to_f).to be_kind_of(Float)
     end
   end
 

data/spec/logstash/modules/cli_parser_spec.rb

@@ -0,0 +1,129 @@
+require "spec_helper"
+require "logstash/logging"
+require "logstash/errors"
+require "logstash/modules/cli_parser"
+
+describe LogStash::Modules::CLIParser do
+
+  subject { LogStash::Modules::CLIParser.new(module_names, module_variables) }
+  let(:logger) { double("logger") }
+  let(:module_name) { "foo" }
+  let(:module_names) { [ module_name, "bar" ] }
+  let(:proto_key_value) { "var.input.stdin.type=example" }
+  let(:proto_mod_vars) { module_name + "." + proto_key_value }
+  let(:module_variables) { [ proto_mod_vars ] }
+  let(:expected_output) { { "name" => module_name, "var.input.stdin.type" => "example" } }
+
+  describe ".parse_modules" do
+    let(:module1) { "module1" }
+    let(:module2) { "module2" }
+    let(:csv_modules) { "#{module1},#{module2}" }
+    let(:list_with_csv) { [ module_name, csv_modules ] }
+    let(:post_parse) { [ module_name, module1, module2 ] }
+
+    context "when it receives an array without a csv entry" do
+      it "return the array unaltered" do
+        expect(subject.parse_modules(module_names)).to eq(module_names)
+      end
+    end
+
+    context "when it receives an empty array" do
+      it "return an empty array" do
+        expect(subject.parse_modules([])).to eq([])
+      end
+    end
+
+    context "when it receives an array with a csv entry" do
+      it "return the original array with the csv values split into elements" do
+        expect(subject.parse_modules(list_with_csv)).to eq(post_parse)
+      end
+    end
+
+    context "when it receives an array with a bad csv entry" do
+      let(:bad_modules) { [ "-Minvalid", module1 ] }
+      it "raise a LogStash::ConfigLoadingError exception" do
+        expect { subject.parse_modules(bad_modules) }.to raise_error LogStash::ConfigLoadingError
+      end
+    end
+
+    context "when it receives a nil value in an array" do
+      let(:array_with_nil) { list_with_csv << nil }
+      it "skip it" do
+        expect(subject.parse_modules(array_with_nil)).to eq(post_parse)
+      end
+    end
+  end
+
+  describe ".get_kv" do
+    context "when it receives a valid string" do
+      let(:expected_key) { "var.input.stdin.type" }
+      let(:expected_value) { "example" }
+      let(:unparsed) { expected_key + "=" + expected_value }
+      it "split it into a key value pair" do
+        expect(subject.get_kv(module_name,unparsed)).to eq([expected_key,expected_value])
+      end
+    end
+
+    context "when it receives an invalid string" do
+      let(:bad_example) { "var.fail" }
+      it "raise a LogStash::ConfigLoadingError exception" do
+        expect { subject.get_kv(module_name,bad_example) }.to raise_error LogStash::ConfigLoadingError
+      end
+    end
+  end
+
+  describe ".name_splitter" do
+    context "when it receives a valid string" do
+      let(:expected) { "var.input.stdin.type=example" }
+      it "split the module name from the rest of the string" do
+        expect(subject.name_splitter(proto_mod_vars)).to eq([module_name,expected])
+      end
+    end
+
+    context "when it receives an invalid string" do
+      let(:bad_example) { "var.fail" }
+      it "raise a LogStash::ConfigLoadingError exception" do
+        expect { subject.name_splitter(bad_example) }.to raise_error LogStash::ConfigLoadingError
+      end
+    end
+  end
+
+  describe ".parse_vars" do
+    context "when it receives a vars_list with valid strings" do
+      it "return a hash with the module name and associated variables as key value pairs" do
+        expect(subject.parse_vars(module_name, module_variables)).to eq(expected_output)
+      end
+    end
+
+    context "when it receives a string that doesn't start with module_name" do
+      let(:has_unrelated) { module_variables << "bar.var.input.stdin.type=different" }
+      it "skips it" do
+        expect(subject.parse_vars(module_name, has_unrelated)).to eq(expected_output)
+      end
+    end
+
+    context "when it receives an empty vars_list" do
+      let(:name_only) { { "name" => module_name } }
+      it "return a hash with only 'name => module_name'" do
+        expect(subject.parse_vars(module_name, [])).to eq(name_only)
+      end
+    end
+  end
+
+  describe ".parse_it" do
+    context "when it receives a valid module_list and module_variable_list" do
+      let(:module_names) { [ module_name ]}
+      it "@output is array of hashes with the module name and associated variables as key value pairs" do
+        expect(subject.output).to eq([expected_output])
+      end
+    end
+
+    context "when it receives a non-array value for module_list" do
+      let(:module_names) { "string value" }
+      it "return an empty array" do
+        expect(subject.output).to eq([])
+      end
+    end
+  end
+
+end

data/spec/logstash/modules/logstash_config_spec.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "logstash/modules/logstash_config"
+
+describe LogStash::Modules::LogStashConfig do
+  let(:mod) { instance_double("module", :directory => Stud::Temporary.directory, :module_name => "testing") }
+  let(:settings) { {"var.logstash.testing.pants" => "fancy" }}
+  subject { described_class.new(mod, settings) }
+
+  describe "configured inputs" do
+    context "when no inputs is send" do
+      it "returns the default" do
+        expect(subject.configured_inputs(["kafka"])).to include("kafka")
+      end
+    end
+
+    context "when inputs are send" do
+      let(:settings) { { "var.inputs" => "tcp" } }
+
+      it "returns the configured inputs" do
+        expect(subject.configured_inputs(["kafka"])).to include("tcp")
+      end
+
+      context "when alias is specified" do
+        let(:settings) { { "var.inputs" => "smartconnector" } }
+
+        it "returns the configured inputs" do
+          expect(subject.configured_inputs(["kafka"], { "smartconnector" => "tcp"  })).to include("tcp", "smartconnector")
+        end
+      end
+    end
+  end
+
+  describe "array to logstash array string" do
+    it "return an escaped string" do
+      expect(subject.array_to_string(["hello", "ninja"])).to eq("['hello', 'ninja']")
+    end
+  end
+
+  describe "alias modules options" do
+    let(:alias_table) do
+      { "var.logstash.testing" => "var.logstash.better" }
+    end
+
+    before do
+      subject.alias_settings_keys!(alias_table)
+    end
+
+    it "allow to retrieve settings" do
+      expect(subject.setting("var.logstash.better.pants", "dont-exist")).to eq("fancy")
+    end
+
+    it "allow to retrieve settings with the original name" do
+      expect(subject.setting("var.logstash.testing.pants", "dont-exist")).to eq("fancy")
+    end
+  end
+end

data/spec/logstash/modules/scaffold_spec.rb

@@ -0,0 +1,239 @@
+# encoding: utf-8
+#
+require "logstash/namespace"
+require "logstash/elasticsearch_client"
+require "logstash/modules/kibana_client"
+require "logstash/modules/kibana_config"
+require "logstash/modules/scaffold"
+require "logstash/modules/elasticsearch_importer"
+require "logstash/modules/kibana_importer"
+
+require_relative "../../support/helpers"
+
+describe LogStash::Modules::Scaffold do
+  let(:base_dir) { "gem-home" }
+  let(:mname) { "foo" }
+  subject(:test_module) { described_class.new(mname, base_dir) }
+  let(:module_settings) do
+    {
+      "var.elasticsearch.hosts" => "es.mycloud.com:9200",
+      "var.elasticsearch.user" => "foo",
+      "var.elasticsearch.password" => "password",
+      "var.input.tcp.port" => 5606,
+    }
+  end
+  let(:dashboard_hash) do
+    {
+      "hits" => 0,
+      "timeRestore" => false,
+      "description" => "",
+      "title" => "Filebeat Apache2 Dashboard",
+      "uiStateJSON" => "{}",
+      "panelsJSON" => '[{"col":1,"id":"foo-c","panelIndex":1,"row":1,"size_x":12,"size_y":3,"type":"visualization"},{"id":"foo-d","type":"search","panelIndex":7,"size_x":12,"size_y":3,"col":1,"row":11,"columns":["apache2.error.client","apache2.error.level","apache2.error.module","apache2.error.message"],"sort":["@timestamp","desc"]}]',
+      "optionsJSON" => "{}",
+      "version" => 1,
+      "kibanaSavedObjectMeta" => {
+        "searchSourceJSON" => "{}"
+      }
+    }
+  end
+  let(:viz_hash) do
+    {
+      "visState" => "",
+      "description" => "",
+      "title" => "foo-c",
+      "uiStateJSON" => "",
+      "version" => 1,
+      "savedSearchId" => "foo-e",
+      "kibanaSavedObjectMeta" => {}
+    }
+  end
+  let(:index_pattern_hash) do
+    {
+      "title" => "foo-*",
+      "timeFieldName" =>"time",
+      "fieldFormatMap" => "{some map}",
+      "fields" => "[some array]"
+    }
+  end
+  context "logstash operation" do
+    let(:ls_conf) do
+<<-ERB
+input {
+  tcp {
+    port => <%= setting("var.input.tcp.port", 45) %>
+    host => <%= setting("var.input.tcp.host", "localhost") %>
+    type => <%= setting("var.input.tcp.type", "server") %>
+  }
+}
+filter {
+
+}
+output {
+  <%= elasticsearch_output_config() %>
+}
+ERB
+    end
+
+    before do
+      allow(LogStash::Modules::FileReader).to receive(:read).and_return(ls_conf)
+    end
+
+    it "provides a logstash config" do
+      expect(test_module.logstash_configuration).to be_nil
+      test_module.with_settings(module_settings)
+      expect(test_module.logstash_configuration).not_to be_nil
+      config_string = test_module.config_string
+      expect(config_string).to include("port => 5606")
+      expect(config_string).to include("hosts => ['es.mycloud.com:9200']")
+    end
+  end
+
+  context "elasticsearch operation" do
+    it "provides the elasticsearch mapping file paths" do
+      test_module.with_settings(module_settings)
+      expect(test_module.elasticsearch_configuration).not_to be_nil
+      files = test_module.elasticsearch_configuration.resources
+      expect(files.size).to eq(1)
+      expect(files.first).to be_a(LogStash::Modules::ElasticsearchResource)
+      expect(files.first.content_path).to eq("gem-home/elasticsearch/foo.json")
+      expect(files.first.import_path).to eq("_template/foo")
+    end
+  end
+
+  context "kibana operation" do
+    before do
+      # allow(LogStash::Modules::FileReader).to receive(:read_json).and_return({})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/dashboard/foo.json").and_return(["Foo-Dashboard"])
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/dashboard/Foo-Dashboard.json").and_return(dashboard_hash)
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/visualization/foo-c.json").and_return(viz_hash)
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/search/foo-d.json").and_return({"d" => "search"})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/search/foo-e.json").and_return({"e" => "search"})
+      allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/index-pattern/foo.json").and_return(index_pattern_hash)
+    end
+
+    it "provides a list of importable files" do
+      expect(test_module.kibana_configuration).to be_nil
+      test_module.with_settings(module_settings)
+      expect(test_module.kibana_configuration).not_to be_nil
+      resources = test_module.kibana_configuration.resources
+      expect(resources.size).to eq(2)
+      resource1 = resources[0]
+      resource2 = resources[1]
+      expect(resource1).to be_a(LogStash::Modules::KibanaSettings)
+      expect(resource2).to be_a(LogStash::Modules::KibanaDashboards)
+      expect(resource1.import_path).to eq("api/kibana/settings")
+      expect(resource1.content).to be_a(Array)
+      expect(resource1.content.size).to eq(2)
+
+      test_object = resource1.content[0]
+      expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
+      expect(test_object.name).to eq("defaultIndex")
+      expect(test_object.value).to eq("foo-*")
+
+      test_object = resource1.content[1]
+      expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
+      expect(test_object.name).to eq("metrics:max_buckets")
+      expect(test_object.value).to eq(86400)
+
+      expect(resource2.import_path).to eq("api/kibana/dashboards/import")
+      expect(resource2.content).to be_a(Array)
+      expect(resource2.content.size).to eq(5)
+      expect(resource2.content.map{|o| o.class}.uniq).to eq([LogStash::Modules::KibanaResource])
+
+      test_object = resource2.content[0]
+      expect(test_object.content_id).to eq("foo-*")
+      expect(test_object.content_type).to eq("index-pattern")
+      expect(test_object.content_as_object).to eq(index_pattern_hash)
+
+      test_object = resource2.content[1]
+      expect(test_object.content_id).to eq("Foo-Dashboard")
+      expect(test_object.content_type).to eq("dashboard")
+      expect(test_object.content_as_object).to eq(dashboard_hash)
+
+      test_object = resource2.content[2]
+      expect(test_object.content_id).to eq("foo-c") #<- the panels can contain items from other folders
+      expect(test_object.content_type).to eq("visualization")
+      expect(test_object.content_as_object).to eq(viz_hash)
+      expect(test_object.content_as_object["savedSearchId"]).to eq("foo-e")
+
+      test_object = resource2.content[3]
+      expect(test_object.content_id).to eq("foo-d") #<- the panels can contain items from other folders
+      expect(test_object.content_type).to eq("search")
+      expect(test_object.content_as_object).to eq("d"=>"search")
+
+      test_object = resource2.content[4]
+      expect(test_object.content_id).to eq("foo-e") # <- the visualization can contain items from the search folder
+      expect(test_object.content_type).to eq("search")
+      expect(test_object.content_as_object).to eq("e"=>"search")
+    end
+  end
+
+  context "importing to elasticsearch stubbed client" do
+    let(:mname) { "tester" }
+    let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "modules", "#{mname}", "configuration")) }
+    let(:response) { double(:response) }
+    let(:client) { double(:client) }
+    let(:kbnclient) { double(:kbnclient) }
+    let(:paths) { [] }
+    let(:expected_paths) { ["_template/tester", "api/kibana/settings", "api/kibana/dashboards/import"] }
+    let(:contents) { [] }
+    let(:expected_objects) do
+      [
+        "index-pattern tester-*",
+        "dashboard FW-Dashboard",
+        "visualization FW-Viz-1",
+        "visualization FW-Viz-2",
+        "search Search-Tester"
+      ]
+    end
+
+    before do
+      allow(response).to receive(:status).and_return(404)
+      allow(client).to receive(:head).and_return(response)
+      allow(kbnclient).to receive(:version).and_return("9.8.7-6")
+    end
+
+    it "calls the import method" do
+      expect(client).to receive(:put).once do |path, content|
+        paths << path
+        LogStash::ElasticsearchClient::Response.new(201, "", {})
+      end
+      expect(kbnclient).to receive(:post).twice do |path, content|
+        paths << path
+        contents << content
+        LogStash::Modules::KibanaClient::Response.new(201, "", {})
+      end
+      test_module.with_settings(module_settings)
+      test_module.import(LogStash::Modules::ElasticsearchImporter.new(client), LogStash::Modules::KibanaImporter.new(kbnclient))
+      expect(paths).to eq(expected_paths)
+      expect(contents[0]).to eq({"changes"=>{"defaultIndex"=>"tester-*", "metrics:max_buckets"=>"86400"}})
+      second_kbn_post = contents[1]
+      expect(second_kbn_post[:version]).to eq("9.8.7-6")
+      expect(second_kbn_post[:objects]).to be_a(Array)
+      expect(second_kbn_post[:objects].size).to eq(5)
+      objects_types_ids = second_kbn_post[:objects].map {|h| "#{h["type"]} #{h["id"]}"}
+      expect(objects_types_ids).to eq(expected_objects)
+    end
+  end
+
+  context "import 4 realz", :skip => "integration" do
+    let(:mname) { "cef" }
+    let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "#{mname}")) }
+    let(:module_settings) do
+      {
+        "var.elasticsearch.hosts" => "localhost:9200",
+        "var.elasticsearch.user" => "foo",
+        "var.elasticsearch.password" => "password",
+        "var.input.tcp.port" => 5606,
+      }
+    end
+    it "puts stuff in ES" do
+      test_module.with_settings(module_settings)
+      client = LogStash::ElasticsearchClient.build(module_settings)
+      import_engine = LogStash::Modules::Importer.new(client)
+      test_module.import(import_engine)
+      expect(1).to eq(1)
+    end
+  end
+end