logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/logstash-core.gemspec

@@ -24,14 +24,14 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
-  gem.add_runtime_dependency "concurrent-ruby", "1.0.0"
+  gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
 
   # Later versions are ruby 2.0 only. We should remove the rack dep once we support 9k
   gem.add_runtime_dependency "rack", '1.6.6'
-  
+
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2.16'
-  gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "jruby-openssl", "0.9.20" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
@@ -43,7 +43,7 @@ Gem::Specification.new do |gem|
 
   # filetools and rakelib
   gem.add_runtime_dependency "minitar", "~> 0.5.4"
-  gem.add_runtime_dependency "rubyzip", "~> 1.1.7"
+  gem.add_runtime_dependency "rubyzip", "~> 1.2.1"
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
   gem.add_runtime_dependency "jrjackson", "~> 0.4.2" #(Apache 2.0 license)
@@ -53,6 +53,8 @@ Gem::Specification.new do |gem|
   # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
   # which does not have this problem.
   gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
+  gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
+  gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
 
   eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
 end

data/spec/logstash/agent/converge_spec.rb

@@ -131,7 +131,7 @@ describe LogStash::Agent do
       end
 
       context "is set to `TRUE`" do
-        let(:interval) { 0.01 }
+        let(:interval) { "10ms" }
         let(:agent_settings) do
           mock_settings(
             "config.reload.automatic" => true,
@@ -165,7 +165,7 @@ describe LogStash::Agent do
           it "it will keep trying to converge" do
             agent_task = start_agent(subject)
 
-            sleep(interval * 20) # let the interval reload a few times
+            sleep(agent_settings.get("config.reload.interval") / 1_000_000_000.0 * 20) # let the interval reload a few times
             expect(subject.pipelines_count).to eq(0)
             expect(source_loader.fetch_count).to be > 1
 

data/spec/logstash/agent_spec.rb

@@ -9,6 +9,8 @@ require "fileutils"
 require_relative "../support/helpers"
 require_relative "../support/matchers"
 
+java_import org.logstash.Timestamp
+
 describe LogStash::Agent do
   let(:agent_settings) { mock_settings({}) }
   let(:agent_args) { {} }
@@ -81,6 +83,12 @@ describe LogStash::Agent do
     end
   end
 
+  describe "ephemeral_id" do
+    it "create a ephemeral id at creation time" do
+      expect(subject.ephemeral_id).to_not be_nil
+    end
+  end
+
   describe "#execute" do
     let(:config_string) { "input { generator { id => 'old'} } output { }" }
     let(:mock_config_pipeline) { mock_pipeline_config(:main, config_string, pipeline_settings) }
@@ -202,7 +210,7 @@ describe LogStash::Agent do
     let(:pipeline_config) { "input { generator { message => '${FOO}-bar' count => 1 } } filter { } output { file { path => '#{temporary_file}' } }" }
     let(:agent_args) { {
       "config.reload.automatic" => false,
-      "config.reload.interval" => 0.01,
+      "config.reload.interval" => "10ms",
       "config.string" => pipeline_config
     } }
 
@@ -386,7 +394,7 @@ describe LogStash::Agent do
       it "sets the success reload timestamp" do
         snapshot = subject.metric.collector.snapshot_metric
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
-        expect(value).to be_a(LogStash::Timestamp)
+        expect(value).to be_a(Timestamp)
       end
 
       it "does not set the last reload error" do
@@ -415,7 +423,7 @@ describe LogStash::Agent do
       it "sets the failure reload timestamp" do
         snapshot = subject.metric.collector.snapshot_metric
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
-        expect(value).to be_a(LogStash::Timestamp)
+        expect(value).to be_a(Timestamp)
       end
 
       it "sets the last reload error" do

data/spec/logstash/api/modules/logging_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 require "spec_helper"
+require "json-schema"
 require "sinatra"
 require "logstash/api/modules/logging"
-require "logstash/json"
 
 describe LogStash::Api::Modules::Logging do
   include_context "api setup"
@@ -12,20 +12,26 @@ describe LogStash::Api::Modules::Logging do
     context "when setting a logger's log level" do
       it "should return a positive acknowledgement on success" do
         put '/', '{"logger.logstash": "ERROR"}'
-        payload = LogStash::Json.load(last_response.body)
-        expect(payload['acknowledged']).to eq(true)
+        expect(JSON::Validator.fully_validate(
+          { "properties" => { "acknowledged" => { "enum" => [true] } } },
+          last_response.body)
+        ).to be_empty
       end
 
       it "should throw error when level is invalid" do
         put '/', '{"logger.logstash": "invalid"}'
-        payload = LogStash::Json.load(last_response.body)
-        expect(payload['error']).to eq("invalid level[invalid] for logger[logstash]")
+        expect(JSON::Validator.fully_validate(
+          { "properties" => { "error" => { "enum" => ["invalid level[invalid] for logger[logstash]"] } } },
+          last_response.body)
+        ).to be_empty
       end
 
       it "should throw error when key logger is invalid" do
         put '/', '{"invalid" : "ERROR"}'
-        payload = LogStash::Json.load(last_response.body)
-        expect(payload['error']).to eq("unrecognized option [invalid]")
+        expect(JSON::Validator.fully_validate(
+          { "properties" => { "error" => { "enum" => ["unrecognized option [invalid]"] } } },
+          last_response.body)
+        ).to be_empty
       end
     end
   end

data/spec/logstash/api/modules/node_plugins_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 require "spec_helper"
+require "json-schema"
 require "sinatra"
 require "logstash/api/modules/plugins"
-require "logstash/json"
 
 describe LogStash::Api::Modules::Plugins do
   include_context "api setup"
@@ -14,19 +14,37 @@ describe LogStash::Api::Modules::Plugins do
     get "/"
   end
 
-  let(:payload) { LogStash::Json.load(last_response.body) }
-
   describe "retrieving plugins" do
     it "should return OK" do
       expect(last_response).to be_ok
     end
 
     it "should return a list of plugins" do
-      expect(payload["plugins"]).to be_a(Array)
+      expect(JSON::Validator.fully_validate(
+        {
+          "properties" => {
+            "plugins" => {
+              "type" => "array"
+            },
+            "required" => ["plugins"]
+          }
+        },
+        last_response.body)
+      ).to be_empty
     end
 
     it "should return the total number of plugins" do
-      expect(payload["total"]).to be_a(Numeric)
+      expect(JSON::Validator.fully_validate(
+        {
+          "properties" => {
+            "total" => {
+              "type" => "number"
+            },
+            "required" => ["total"]
+          }
+        },
+        last_response.body)
+      ).to be_empty
     end
   end
 end

data/spec/logstash/api/modules/node_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 require "spec_helper"
+require "json-schema"
 require "sinatra"
 require "logstash/api/modules/node"
-require "logstash/json"
 
 describe LogStash::Api::Modules::Node do
   include_context "api setup"
@@ -19,7 +19,7 @@ describe LogStash::Api::Modules::Node do
     end
 
     it "should return a JSON object" do
-      expect{ LogStash::Json.load(last_response.body) }.not_to raise_error
+      expect(JSON::Validator.validate({}, last_response.body)).to eq(true)
     end
 
     context "#threads count" do
@@ -28,14 +28,21 @@ describe LogStash::Api::Modules::Node do
         get "/hot_threads?threads=5"
       end
 
-      let(:payload) { LogStash::Json.load(last_response.body) }
-
       it "should return a json payload content type" do
         expect(last_response.content_type).to eq("application/json")
       end
 
       it "should return information for <= # requested threads" do
-        expect(payload["hot_threads"]["threads"].count).to be <= 5
+        expect(JSON::Validator.fully_validate(
+          {
+            "properties" => {
+              "hot_threads" => {
+                "properties" => { "threads" => { "type" => "array", "maxItems" => 5 } }
+              }
+            }
+          },
+          last_response.body
+        )).to be_empty
       end
     end
 
@@ -51,14 +58,12 @@ describe LogStash::Api::Modules::Node do
           get path
         end
 
-        let(:payload) { last_response.body }
-
         it "should return a text/plain content type" do
           expect(last_response.content_type).to eq("text/plain;charset=utf-8")
         end
 
         it "should return a plain text payload" do
-          expect{ JSON.parse(payload) }.to raise_error
+          expect {JSON::Validator.fully_validate({}, payload)}.to raise_error
         end
       end
     end
@@ -76,10 +81,8 @@ describe LogStash::Api::Modules::Node do
         @threads.each { |t| t.kill } rescue nil
       end
 
-      let(:payload) { last_response.body }
-
       it "should return information for <= # requested threads" do
-        expect(payload.scan(/thread name/).size).to eq(2)
+        expect(last_response.body.scan(/thread name/).size).to eq(2)
       end
     end
 
@@ -97,10 +100,8 @@ describe LogStash::Api::Modules::Node do
           expect(last_response.content_type).to eq("application/json")
         end
 
-        let(:payload) { last_response.body }
-
         it "should return a json payload" do
-          expect{ JSON.parse(payload) }.not_to raise_error
+          expect(JSON::Validator.validate({}, last_response.body)).to eq(true)
         end
       end
     end
@@ -115,7 +116,8 @@ describe LogStash::Api::Modules::Node do
             "batch_size" => Numeric,
             "batch_delay" => Numeric,
             "config_reload_automatic" => Boolean,
-            "config_reload_interval" => Numeric
+            "config_reload_interval" => Numeric,
+            "dead_letter_queue_enabled" => Boolean
           }
         },
         "os" => {

data/spec/logstash/api/modules/node_stats_spec.rb

@@ -3,7 +3,6 @@ require "spec_helper"
 
 require "sinatra"
 require "logstash/api/modules/node_stats"
-require "logstash/json"
 
 describe LogStash::Api::Modules::NodeStats do
   include_context "api setup"

data/spec/logstash/api/modules/plugins_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "spec_helper"
+require "json-schema"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"
@@ -12,8 +13,6 @@ describe LogStash::Api::Modules::Plugins do
     get "/"
   end
 
-  let(:payload) { LogStash::Json.load(last_response.body) }
-
   it "respond to plugins resource" do
     expect(last_response).to be_ok
   end
@@ -24,15 +23,31 @@ describe LogStash::Api::Modules::Plugins do
 
   context "#schema" do
     it "return the expected schema" do
-      expect(payload.keys).to include("plugins", "total")
-      payload["plugins"].each do |plugin|
-        expect(plugin.keys).to include("name", "version")
-      end
+      expect(JSON::Validator.fully_validate(
+        {
+          "properties" => {
+            "plugins" => {
+              "type" => "array",
+              "items" => [
+                {
+                  "type" => "object",
+                  "required" => ["version", "name"]
+                }
+              ]
+            },
+            "total" => { "type" => "number" } 
+          },
+          "required" => ["plugins", "total"]
+        },
+        last_response.body)
+      ).to be_empty
     end
   end
 
   context "#values" do
 
+    let(:payload) { LogStash::Json.load(last_response.body) }
+
     it "return totals of plugins" do
       expect(payload["total"]).to eq(payload["plugins"].count)
     end
@@ -46,9 +61,25 @@ describe LogStash::Api::Modules::Plugins do
     end
 
     it "return non empty version values" do
-      payload["plugins"].each do |plugin|
-        expect(plugin["version"]).not_to be_empty
-      end
+      expect(JSON::Validator.fully_validate(
+        { "properties" => { "plugins" => {
+          "type" => "array",
+          "items" => [
+            {
+              "type" => "object",
+              "properties" => {
+                "version" => {
+                  "type" => "string",
+                  "minLength" => 1
+                }
+              },
+              "required" => ["version"]
+            }
+          ],
+          "minItems" => 1
+        } } },
+        last_response.body)
+      ).to be_empty
     end
   end
 end

data/spec/logstash/api/modules/root_spec.rb

@@ -3,7 +3,6 @@ require "spec_helper"
 
 require "sinatra"
 require "logstash/api/modules/root"
-require "logstash/json"
 
 describe LogStash::Api::Modules::Root do
   include_context "api setup"

data/spec/logstash/api/rack_app_spec.rb

@@ -1,4 +1,5 @@
 require "logstash/api/rack_app"
+require "json-schema"
 require "rack/test"
 
 describe LogStash::Api::RackApp do
@@ -55,7 +56,7 @@ describe LogStash::Api::RackApp do
       end
 
       it "should return valid JSON" do
-        expect { LogStash::Json.load(last_response.body) }.not_to raise_error
+        expect(JSON::Validator.validate({}, last_response.body)).to eq(true)
       end
 
       it "should log the error" do

data/spec/logstash/compiler/compiler_spec.rb

@@ -10,6 +10,8 @@ describe LogStash::Compiler do
 
   let(:source_protocol) { "test_proto" }
 
+  let(:settings) { mock_settings({}) }
+
   # Static import of these useful enums
   INPUT = Java::OrgLogstashConfigIr::PluginDefinition::Type::INPUT
   FILTER = Java::OrgLogstashConfigIr::PluginDefinition::Type::FILTER
@@ -28,8 +30,8 @@ describe LogStash::Compiler do
 
   describe "compiling to Pipeline" do
     subject(:source_id) { "fake_sourcefile" }
-    let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, source) }
-    subject(:compiled) { described_class.compile_pipeline(source_with_metadata) }
+    let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
+    subject(:compiled) { puts "PCOMP"; described_class.compile_pipeline(source_with_metadata, settings) }
 
     describe "compiling multiple sources" do
       let(:sources) do
@@ -38,13 +40,18 @@ describe LogStash::Compiler do
           "input { input_1 {} } filter { filter_1 {} } output { output_1 {} }"
         ]
       end
+
       let(:sources_with_metadata) do
         sources.map.with_index do |source, idx|
-          org.logstash.common.SourceWithMetadata.new("#{source_protocol}_#{idx}", "#{source_id}_#{idx}", source)
+          org.logstash.common.SourceWithMetadata.new("#{source_protocol}_#{idx}", "#{source_id}_#{idx}", 0, 0, source)
         end
       end
 
-      subject(:pipeline) { described_class.compile_sources(*sources_with_metadata) }
+      subject(:pipeline) { described_class.compile_sources(sources_with_metadata, settings) }
+
+      it "should generate a hash" do
+        expect(pipeline.unique_hash).to be_a(String)
+      end
 
       it "should compile cleanly" do
         expect(pipeline).to be_a(org.logstash.config.ir.PipelineIR)
@@ -56,7 +63,8 @@ describe LogStash::Compiler do
 
       describe "applying protocol and id metadata" do
         it "should apply the correct source metadata to all components" do
-          pipeline.plugin_vertices.each do |pv| 
+          # TODO: seems to be a jruby regression we cannot currently call each on a stream
+          pipeline.get_plugin_vertices.each do |pv|
             name_idx = pv.plugin_definition.name.split("_").last
             source_protocol_idx = pv.source_with_metadata.protocol.split("_").last
             source_id_idx = pv.source_with_metadata.id.split("_").last
@@ -91,9 +99,48 @@ describe LogStash::Compiler do
 
   describe "compiling imperative" do
     let(:source_id) { "fake_sourcefile" }
-    let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, source) }
-    subject(:compiled) { described_class.compile_imperative(source_with_metadata) }
+    let(:source_with_metadata) { org.logstash.common.SourceWithMetadata.new(source_protocol, source_id, 0, 0, source) }
+    subject(:compiled) { described_class.compile_imperative(source_with_metadata, settings) }
+
+    context "when config.support_escapes" do
+      let(:parser) { LogStashCompilerLSCLGrammarParser.new }
+
+      let(:processed_value)  { 'The computer says, "No"' }
+
+      let(:source) { 
+        %q(
+          input {
+            foo {
+              bar => "The computer says, \"No\""
+            }
+          }
+        )
+      }
+
+      let(:compiled_string) do
+        compiled[:input].toGraph.vertices.toArray.first.getPluginDefinition.arguments["bar"]
+      end
+
+      before do
+        settings.set_value("config.support_escapes", process_escape_sequences)
+      end
 
+      context "is enabled" do
+        let(:process_escape_sequences) { true }
+
+        it "should process escape sequences" do
+          expect(compiled_string).to be == processed_value
+        end
+      end
+
+      context "is false" do
+        let(:process_escape_sequences) { false }
+
+        it "should not process escape sequences" do
+          expect(compiled_string).not_to be == processed_value
+        end
+      end
+    end
     describe "an empty file" do
       let(:source) { "input {} output {}" }