logstash-core 5.4.3-java → 5.5.0-java

data/lib/logstash/util/thread_dump.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+java_import 'org.logstash.instrument.reports.ThreadsReport'
+
 module LogStash
   module Util
     class ThreadDump
@@ -10,7 +12,7 @@ module LogStash
 
       def initialize(options={})
         @options   = options
-        @dump = options.fetch(:dump, JRMonitor.threads.generate({}))
+        @dump = options.fetch(:dump, ThreadsReport.generate({}))
         @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
         @ignore    = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
       end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -127,7 +127,12 @@ module LogStash; module Util
       end
 
       def empty?
-        @mutex.synchronize { @queue.is_fully_acked? }
+        @mutex.lock
+        begin
+          @queue.is_fully_acked?
+        ensure
+          @mutex.unlock
+        end
       end
 
       def set_batch_dimensions(batch_size, wait_for)
@@ -152,8 +157,11 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           yield(@inflight_batches)
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -173,16 +181,24 @@ module LogStash; module Util
         end
 
         batch = new_batch
-        @mutex.synchronize { batch.read_next }
+        @mutex.lock
+        begin
+          batch.read_next
+        ensure
+          @mutex.unlock
+        end
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
           start_clock
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -191,12 +207,14 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           batch.close
-
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
+        ensure
+          @mutex.unlock
         end
       end
 

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -98,8 +98,11 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           yield(@inflight_batches)
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -115,16 +118,24 @@ module LogStash; module Util
 
       def read_batch
         batch = new_batch
-        @mutex.synchronize { batch.read_next }
+        @mutex.lock
+        begin
+          batch.read_next
+        ensure
+          @mutex.unlock
+        end
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.synchronize do
-          # there seems to be concurrency issues with metrics, keep it in the mutex
+        @mutex.lock
+        # there seems to be concurrency issues with metrics, keep it in the mutex
+        begin
           set_current_thread_inflight_batch(batch)
           start_clock
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -133,10 +144,13 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
+        ensure
+          @mutex.unlock
         end
       end
 

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.4.3"
+LOGSTASH_VERSION = "5.5.0"

data/locales/en.yml

@@ -83,10 +83,31 @@ en:
       logging:
         unrecognized_option: |-
           unrecognized option [%{option}]
+    modules:
+      configuration:
+        parse-failed: |-
+          Failed to parse the module configuration: [%{error}]
+        modules-must-be-unique: >-
+          Only a single instance of any module can be run at a time. Duplicate
+          modules: %{duplicate_modules}
+        modules-invalid-name: >-
+          Invalid module name: %{module_name}
+        modules-variables-malformed: >-
+          Failed to parse module variable %{rawvar}.  Must be in -M
+          "MODULE_NAME.KEY.SUBKEY=VALUE" format
+        modules-unavailable: >-
+          The modules specified are not available yet.
+          Specified modules: %{specified_modules}
+          Available modules: %{available_modules}
+        elasticsearch_connection_failed: >-
+          Failed to import module configurations to Elasticsearch.
+          Module: %{module_name} has hosts: %{hosts}
+
     runner:
       short-help: |-
         usage:
           bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
+          bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
           bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
           bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
           bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
@@ -96,6 +117,13 @@ en:
       missing-configuration: >-
         No configuration file was specified. Perhaps you forgot to provide
         the '-f yourlogstash.conf' flag?
+      config-module-exclusive: >-
+        Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
+        (--modules) or the "modules:" block in the logstash.yml file.
+      cli-module-override: >-
+        Both command-line and logstash.yml modules configurations detected.
+        Using command-line module configuration and ignoring logstash.yml module
+        configuration.
       reload-without-config-path: >-
         Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
       locked-data-path: >-
@@ -181,6 +209,24 @@ en:
           "%{default_output}"
           If you wish to use both defaults, please use
           the empty string for the '-e' flag.
+        modules: |+
+          Load Logstash modules.
+          Modules can be defined using multiple instances
+          '--modules module1 --modules module2',
+             or comma-separated syntax
+          '--modules=module1,module2'
+          Cannot be used in conjunction with '-e' or '-f'
+          Use of '--modules' will override modules declared
+          in the 'logstash.yml' file.
+        modules_variable: |+
+          Load variables for module template.
+          Multiple instances of '-M' or
+          '--modules.variable' are supported.
+          Ignored if '--modules' flag is not used.
+          Should be in the format of
+          '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
+          as in
+          '-M "example.var.filter.mutate.fieldname=fieldvalue"'
         configtest: |+
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host

data/logstash-core.gemspec

@@ -25,11 +25,14 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
   gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
+
+  # Later versions are ruby 2.0 only. We should remove the rack dep once we support 9k
+  gem.add_runtime_dependency "rack", '1.6.6'
+  
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2.16'
-  gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "jruby-openssl", "0.9.19" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
-  gem.add_runtime_dependency "jrmonitor", '~> 0.4.2'
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan
@@ -50,6 +53,8 @@ Gem::Specification.new do |gem|
   # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
   # which does not have this problem.
   gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
+  gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
+  gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
 
   eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
 end

data/spec/{api/lib/commands/stats.rb → logstash/api/commands/stats_spec.rb}

@@ -1,10 +1,15 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
+require "spec_helper"
 
 describe LogStash::Api::Commands::Stats do
+  include_context "api setup"
 
   let(:report_method) { :run }
-  subject(:report) { do_request { report_class.new.send(report_method) } }
+  subject(:report) do
+    factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
+   
+    factory.build(:stats).send(report_method)
+  end
 
   let(:report_class) { described_class }
 

data/spec/{api/lib → logstash/api}/errors_spec.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require_relative "../spec_helper"
+require "spec_helper"
 require "logstash/api/errors"
 
 describe LogStash::Api::ApiError do

data/spec/{api/lib/api → logstash/api/modules}/logging_spec.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
+require "spec_helper"
 require "sinatra"
 require "logstash/api/modules/logging"
 require "logstash/json"
@@ -10,15 +10,6 @@ describe LogStash::Api::Modules::Logging do
   describe "#logging" do
 
     context "when setting a logger's log level" do
-      before(:all) do
-        @runner = LogStashRunner.new
-        @runner.start
-      end
-
-      after(:all) do
-        @runner.stop
-      end
-
       it "should return a positive acknowledgement on success" do
         put '/', '{"logger.logstash": "ERROR"}'
         payload = LogStash::Json.load(last_response.body)

data/spec/{api/lib/api → logstash/api/modules}/node_plugins_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require_relative "../../../support/shared_examples"
-require_relative "../../spec_helper"
+require "spec_helper"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"
@@ -12,7 +11,7 @@ describe LogStash::Api::Modules::Plugins do
   extend ResourceDSLMethods
 
   before(:each) do
-    do_request { get "/" }
+    get "/"
   end
 
   let(:payload) { LogStash::Json.load(last_response.body) }

data/spec/{api/lib/api → logstash/api/modules}/node_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
-require_relative "../../../support/shared_examples"
+require "spec_helper"
 require "sinatra"
 require "logstash/api/modules/node"
 require "logstash/json"
@@ -12,7 +11,7 @@ describe LogStash::Api::Modules::Node do
   describe "#hot threads" do
 
     before(:all) do
-      do_request { get "/hot_threads" }
+      get "/hot_threads"
     end
 
     it "respond OK" do
@@ -26,7 +25,7 @@ describe LogStash::Api::Modules::Node do
     context "#threads count" do
 
       before(:all) do
-        do_request { get "/hot_threads?threads=5" }
+        get "/hot_threads?threads=5"
       end
 
       let(:payload) { LogStash::Json.load(last_response.body) }
@@ -49,7 +48,7 @@ describe LogStash::Api::Modules::Node do
       ].each do |path|
 
         before(:all) do
-          do_request { get path }
+          get path
         end
 
         let(:payload) { last_response.body }
@@ -70,7 +69,7 @@ describe LogStash::Api::Modules::Node do
         @threads = []
         5.times { @threads << Thread.new { loop {} } }
 
-        do_request { get "/hot_threads?human=t&threads=2"}
+        get "/hot_threads?human=t&threads=2"
       end
 
       after(:all) do
@@ -91,7 +90,7 @@ describe LogStash::Api::Modules::Node do
         "/hot_threads?human=f",
       ].each do |path|
         before(:all) do
-          do_request { get path }
+          get path
         end
 
         it "should return a json payload content type" do

data/spec/{api/lib/api → logstash/api/modules}/node_stats_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
-require_relative "../../../support/shared_examples"
+require "spec_helper"
+
 require "sinatra"
 require "logstash/api/modules/node_stats"
 require "logstash/json"

data/spec/{api/lib/api → logstash/api/modules}/plugins_spec.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
-require_relative "../../../support/shared_examples"
+require "spec_helper"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"
@@ -40,7 +39,9 @@ describe LogStash::Api::Modules::Plugins do
 
     it "return a list of available plugins" do
       payload["plugins"].each do |plugin|
-        expect(plugin).to be_available?
+        expect do 
+          Gem::Specification.find_by_name(plugin["name"])
+        end.not_to raise_error
       end
     end
 

data/spec/{api/lib/api → logstash/api/modules}/root_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
-require_relative "../../spec_helper"
-require_relative "../../../support/shared_examples"
+require "spec_helper"
+
 require "sinatra"
 require "logstash/api/modules/root"
 require "logstash/json"
@@ -9,7 +9,7 @@ describe LogStash::Api::Modules::Root do
   include_context "api setup"
 
   it "should respond to root resource" do
-    do_request { get "/" }
+    get "/"
     expect(last_response).to be_ok
   end
 

data/spec/logstash/config/mixin_spec.rb

@@ -192,8 +192,8 @@ describe LogStash::Config::Mixin do
         expect(clone.uri.to_s).to eql(uri_hidden)
       end
 
-      it "should make the real URI object available under #uri" do
-        expect(subject.uri.uri).to be_a(::URI)
+      it "should make the real java.net.URI object available under #uri" do
+        expect(subject.uri.uri).to be_a(java.net.URI)
       end
 
       it "should obfuscate original_params" do

data/spec/logstash/execution_context_spec.rb

@@ -1,13 +1,32 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/util/dead_letter_queue_manager"
 require "logstash/execution_context"
 
 describe LogStash::ExecutionContext do
+  let(:pipeline) { double("pipeline") }
   let(:pipeline_id) { :main }
+  let(:plugin_id) { "plugin_id" }
+  let(:plugin_type) { "plugin_type" }
+  let(:dlq_writer) { LogStash::Util::DummyDeadLetterQueueWriter.new }
 
-  subject { described_class.new(pipeline_id) }
+  before do
+    allow(pipeline).to receive(:pipeline_id).and_return(pipeline_id)
+  end
+
+  subject { described_class.new(pipeline, plugin_id, plugin_type, dlq_writer) }
 
   it "returns the `pipeline_id`" do
     expect(subject.pipeline_id).to eq(pipeline_id)
   end
+
+  it "returns the pipeline" do
+    expect(subject.pipeline).to eq(pipeline)
+  end
+
+  it "returns the plugin-specific dlq writer" do
+    expect(subject.dlq_writer.plugin_type).to eq(plugin_type)
+    expect(subject.dlq_writer.plugin_id).to eq(plugin_id)
+    expect(subject.dlq_writer.inner_writer).to eq(dlq_writer)
+  end
 end