logstash-core 7.3.2-java → 7.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9f056b670518c9c84a1080c87e4ada7f16729ceb93dc8fc4cbe62c71ef47e86
-  data.tar.gz: 8abadb91f916ae4e32ccc0dba9b7fb1ba0917948d7da3c5de97519dbf4f7dd1c
+  metadata.gz: 33325349b0d132777a029d1ae9da41b912d4831b2d54404b2333f27e97ce6f1d
+  data.tar.gz: 0cbb379d60437e13d9e886e00c9b0c5d68a24a2d120920f9c50afe7610d77920
 SHA512:
-  metadata.gz: 3db1f0158ea7160a663d7f54ed9137ed52112ded806657393397eb35e69ecbb994d19dcf15e590823326e821737a723233441a16ed2234ab5c1e0aef6e9bde7d
-  data.tar.gz: 99861090b2d27b2c48a5f041147566e4c821027da0de42981b824645600ea267c2f63e5a6f04ccc6b415404e6e856dbed15075fd25d5b52a2c50caa813b8cff4
+  metadata.gz: 7859b91d732f6fbeb17b9996182ca2a7d1f688bf6c18d1c45ea3652693e441e6ee902b9317604cdfbeb064c3dd9df23b1d3ff4f894debffa5a29e222933c19c2
+  data.tar.gz: 5e4f036d28299357262b0fb989898d184f3b0db8f99038dc5bb5d1e3ac5c1e41c7f5323fb18fdd32a7964f873142440d2c0fd7e4197320127c686b4cb08ef712

data/lib/logstash/agent.rb

@@ -89,6 +89,7 @@ class LogStash::Agent
 
   def execute
     @thread = Thread.current # this var is implicitly used by Stud.stop?
+    LogStash::Util.set_thread_name("Agent thread")
     logger.debug("Starting agent")
 
     transition_to_running
@@ -307,7 +308,7 @@ class LogStash::Agent
 
     pipeline_actions.map do |action|
       Thread.new(action, converge_result) do |action, converge_result|
-        java.lang.Thread.currentThread().setName("Converge #{action}");
+        LogStash::Util.set_thread_name("Converge #{action}")
         # We execute every task we need to converge the current state of pipelines
         # for every task we will record the action result, that will help us
         # the results of all the task will determine if the converge was successful or not

data/lib/logstash/config/source/multi_local.rb

@@ -4,6 +4,7 @@ require "logstash/settings"
 
 module LogStash module Config module Source
   class MultiLocal < Local
+    include LogStash::Util::SubstitutionVariables
     include LogStash::Util::Loggable
 
     def initialize(settings)
@@ -13,7 +14,7 @@ module LogStash module Config module Source
     end
 
     def pipeline_configs
-      pipelines = retrieve_yaml_pipelines()
+      pipelines = deep_replace(retrieve_yaml_pipelines())
       pipelines_settings = pipelines.map do |pipeline_settings|
         clone = @original_settings.clone
         clone.merge_pipeline_settings(pipeline_settings)

data/lib/logstash/java_pipeline.rb

@@ -8,6 +8,8 @@ require "logstash/instrument/collector"
 require "logstash/compiler"
 require "logstash/config/lir_serializer"
 
+java_import org.apache.logging.log4j.ThreadContext
+
 module LogStash; class JavaPipeline < JavaBasePipeline
   include LogStash::Util::Loggable
   attr_reader \
@@ -102,6 +104,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
+        ThreadContext.put("pipeline.id", pipeline_id)
         run
         @finished_run.make_true
       rescue => e
@@ -236,6 +239,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       pipeline_workers.times do |t|
         thread = Thread.new do
           Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
+          ThreadContext.put("pipeline.id", pipeline_id)
           org.logstash.execution.WorkerLoop.new(
               lir_execution, filter_queue_client, @events_filtered, @events_consumed,
               @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
@@ -305,6 +309,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def inputworker(plugin)
     Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
+    ThreadContext.put("pipeline.id", pipeline_id)
     begin
       plugin.run(wrapped_write_client(plugin.id.to_sym))
     rescue => e

data/lib/logstash/pipeline.rb

@@ -12,6 +12,8 @@ require "logstash/instrument/collector"
 require "logstash/filter_delegator"
 require "logstash/compiler"
 
+java_import org.apache.logging.log4j.ThreadContext
+
 module LogStash; class BasePipeline < AbstractPipeline
   include LogStash::Util::Loggable
 
@@ -172,7 +174,8 @@ module LogStash; class Pipeline < BasePipeline
 
     @thread = Thread.new do
       begin
-        LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
+        LogStash::Util.set_thread_name("[#{pipeline_id}]-manager")
+        ThreadContext.put("pipeline.id", pipeline_id)
         run
         @finished_run.make_true
       rescue => e
@@ -300,7 +303,8 @@ module LogStash; class Pipeline < BasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
-          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
+          LogStash::Util::set_thread_name("[#{pipeline_id}]>worker#{t}")
+          ThreadContext.put("pipeline.id", pipeline_id)
           _pipeline.worker_loop(_b_size, _b_delay)
         end
         @worker_threads << thread
@@ -430,6 +434,7 @@ module LogStash; class Pipeline < BasePipeline
 
   def inputworker(plugin)
     Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
+    ThreadContext.put("pipeline.id", pipeline_id)
     begin
       plugin.run(wrapped_write_client(plugin.id.to_sym))
     rescue => e
@@ -535,6 +540,8 @@ module LogStash; class Pipeline < BasePipeline
     raise "Attempted to start flusher on a stopped pipeline!" if stopped?
 
     @flusher_thread = Thread.new do
+      LogStash::Util.set_thread_name("[#{pipeline_id}]-flusher-thread")
+      ThreadContext.put("pipeline.id", pipeline_id)
       while Stud.stoppable_sleep(5, 0.1) { stopped? }
         flush
         break if stopped?

data/lib/logstash/runner.rb

@@ -336,7 +336,8 @@ class LogStash::Runner < Clamp::StrictCommand
         # TODO(ph): make it better for multiple pipeline
         if results.success?
           results.response.each do |pipeline_config|
-            LogStash::BasePipeline.new(pipeline_config)
+            pipeline_class = pipeline_config.settings.get_value("pipeline.java_execution") ? LogStash::JavaPipeline : LogStash::BasePipeline
+            pipeline_class.new(pipeline_config)
           end
           puts "Configuration OK"
           logger.info "Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash"

data/spec/logstash/api/commands/default_metadata_spec.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe LogStash::Api::Commands::DefaultMetadata do
+  include_context "api setup"
+
+  let(:report_method) { :all }
+  subject(:report) do
+    factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
+    factory.build(:default_metadata).send(report_method)
+  end
+
+  let(:report_class) { described_class }
+
+  describe "#plugins_stats_report" do
+    let(:report_method) { :all }
+    # Enforce just the structure
+    it "check keys" do
+      expect(report.keys).to include(
+        :host,
+        :version,
+        :http_address,
+        :id,
+        :name,
+        :ephemeral_id,
+        :status,
+        :snapshot,
+        :pipeline
+      )
+      expect(report[:pipeline].keys).to include(
+        :workers,
+        :batch_size,
+        :batch_delay,
+      )
+    end
+  end
+end

data/spec/logstash/api/commands/node_spec.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe LogStash::Api::Commands::Node do
+  include_context "api setup"
+
+  let(:report_method) { :all }
+  let(:pipeline_id) { nil }
+  let(:opts) { {} }
+  let(:mocked_vertex) {{:config_name=>"elasticsearch",
+                       :plugin_type=>"output",
+                       :meta=>{
+                         :source=>{
+                           :protocol=>"str",
+                           :id=>"pipeline",
+                           :line=>1,
+                           :column=>64
+                         }
+                       },
+                       :id=>"2d2270426a2e8d7976b972b6a5318624331fa0d39fa3f903d2f3490e58a7d25a",
+                       :explicit_id=>false,
+                       :type=>"plugin"}
+                  }
+  subject(:report) do
+    factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
+    if report_method == :pipelines
+      factory.build(:node).send(report_method, opts)
+    elsif report_method == :pipeline
+      factory.build(:node).send(report_method, pipeline_id, opts)
+    elsif report_method == :decorate_with_cluster_uuids
+      factory.build(:node).send(report_method, mocked_vertex)
+    else
+      factory.build(:node).send(report_method)
+    end
+  end
+
+  let(:report_class) { described_class }
+
+  describe "#all" do
+    let(:report_method) { :all }
+    # Enforce just the structure
+    it "check keys" do
+      expect(report.keys).to include(
+        :pipelines,
+        :os,
+        :jvm
+      )
+    end
+  end
+
+
+  describe "#pipeline" do
+    let(:report_method) { :pipeline }
+    let(:pipeline_id) { "main" }
+    # Enforce just the structure
+    it "check keys" do
+      expect(report.keys).to include(
+        :ephemeral_id,
+        :hash,
+        :workers,
+        :batch_size,
+        :batch_delay,
+        :config_reload_automatic,
+        :config_reload_interval,
+        :dead_letter_queue_enabled,
+        # :dead_letter_queue_path is nil in tests
+        # so it is ignored
+      )
+    end
+  end
+
+  describe "#pipeline?opts" do
+    let(:report_method) { :pipeline }
+    let(:pipeline_id) { "main" }
+    let(:opts) { { :graph=>true } }
+    # Enforce just the structure
+    it "check keys" do
+      expect(report.keys).to include(
+        :ephemeral_id,
+        :hash,
+        :workers,
+        :batch_size,
+        :batch_delay,
+        :config_reload_automatic,
+        :config_reload_interval,
+        :dead_letter_queue_enabled,
+        # Be sure we display a graph when we set the option to
+        :graph
+      )
+    end
+  end
+
+  describe "#os" do
+    let(:report_method) { :os }
+    it "check_keys" do
+      expect(report.keys).to include(
+        :name,
+        :arch,
+        :version,
+        :available_processors
+      )
+    end
+  end
+
+  describe "#jvm" do
+    let(:report_method) { :jvm }
+    it "check_keys" do
+      expect(report.keys).to include(
+        :pid,
+        :version,
+        :vm_version,
+        :vm_vendor,
+        :vm_name,
+        :start_time_in_millis,
+        :mem,
+        :gc_collectors
+      )
+    expect(report[:mem].keys).to include(
+      :heap_init_in_bytes,
+      :heap_max_in_bytes,
+      :non_heap_init_in_bytes,
+      :non_heap_max_in_bytes
+    )
+    end
+  end
+end

data/spec/logstash/api/commands/stats_spec.rb

@@ -5,25 +5,63 @@ describe LogStash::Api::Commands::Stats do
   include_context "api setup"
 
   let(:report_method) { :run }
+  let(:extended_pipeline) { nil }
+  let(:opts) { {} }
   subject(:report) do
     factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
-   
-    factory.build(:stats).send(report_method)
+    if extended_pipeline
+      factory.build(:stats).send(report_method, "main", extended_pipeline, opts)
+    else
+      factory.build(:stats).send(report_method)
+    end
   end
 
   let(:report_class) { described_class }
 
+  describe "#plugins_stats_report" do
+    let(:report_method) { :plugins_stats_report }
+    # Enforce just the structure
+    let(:extended_pipeline) {
+      {
+      :queue => "fake_queue",
+      :hash => "fake_hash",
+      :ephemeral_id => "fake_epehemeral_id",
+      :vertices => "fake_vertices"
+      }
+    }
+    # TODO pass in a real sample vertex
+#    let(:opts) {
+#      {
+#        :vertices => "fake vertices"
+#      }
+#    }
+    it "check keys" do
+      expect(report.keys).to include(
+        :queue,
+        :hash,
+        :ephemeral_id,
+        # TODO re-add vertices -- see above
+#        :vertices
+      )
+    end
+  end
+
   describe "#events" do
     let(:report_method) { :events }
 
     it "return events information" do
-      expect(report.keys).to include(:in, :filtered, :out)
+      expect(report.keys).to include(
+        :in,
+        :filtered,
+        :out,
+        :duration_in_millis,
+        :queue_push_duration_in_millis)
     end
   end
-  
+
   describe "#hot_threads" do
     let(:report_method) { :hot_threads }
-    
+
     it "should return hot threads information as a string" do
       expect(report.to_s).to be_a(String)
     end
@@ -35,19 +73,50 @@ describe LogStash::Api::Commands::Stats do
 
   describe "memory stats" do
     let(:report_method) { :memory }
-      
+
     it "return hot threads information" do
       expect(report).not_to be_empty
     end
 
-    it "return heap information" do
-      expect(report.keys).to include(:heap_used_in_bytes)
+    it "return memory information" do
+      expect(report.keys).to include(
+        :heap_used_percent,
+        :heap_committed_in_bytes,
+        :heap_max_in_bytes,
+        :heap_used_in_bytes,
+        :non_heap_used_in_bytes,
+        :non_heap_committed_in_bytes,
+        :pools
+      )
     end
+  end
+
+  describe "jvm stats" do
+    let(:report_method) { :jvm }
 
-    it "return non heap information" do
-      expect(report.keys).to include(:non_heap_used_in_bytes)
+    it "return jvm information" do
+      expect(report.keys).to include(
+        :threads,
+        :mem,
+        :gc,
+        :uptime_in_millis
+      )
+      expect(report[:threads].keys).to include(
+        :count,
+        :peak_count
+      )
     end
+  end
 
+  describe "reloads stats" do
+    let(:report_method) { :reloads }
+
+    it "return reloads information" do
+      expect(report.keys).to include(
+      :successes,
+      :failures,
+      )
+    end
   end
 
   describe "pipeline stats" do
@@ -74,6 +143,7 @@ describe LogStash::Api::Commands::Stats do
         )
       end
     end
+
     context "when using multiple pipelines" do
       before(:each) do
         expect(LogStash::Config::PipelinesInfo).to receive(:format_pipelines_info).and_return([

data/spec/logstash/api/modules/node_spec.rb

@@ -123,6 +123,7 @@ describe LogStash::Api::Modules::Node do
       root_structure = {
         "pipelines" => {
           "main" => {
+            "ephemeral_id" => String,
             "workers" => Numeric,
             "batch_size" => Numeric,
             "batch_delay" => Numeric,

data/spec/logstash/api/modules/node_stats_spec.rb

@@ -80,12 +80,22 @@ describe LogStash::Api::Modules::NodeStats do
          "filtered" => Numeric,
          "out" => Numeric,
          "queue_push_duration_in_millis" => Numeric
-       }
+       },
+       "plugins" => {
+          "inputs" => Array,
+          "codecs" => Array,
+          "filters" => Array,
+          "outputs" => Array,
+       },
      }
    },
    "reloads" => {
      "successes" => Numeric,
      "failures" => Numeric
+   },
+   "os" => Hash,
+   "queue" => {
+      "events_count" => Numeric
    }
   }
 

data/spec/logstash/jruby_version_spec.rb

@@ -9,7 +9,7 @@ describe "JRuby Version" do
     # ref:
     #   https://github.com/logstash-plugins/logstash-filter-dns/issues/51
     #   https://github.com/jruby/jruby/pull/5722
-    expect(JRUBY_VERSION).to eq("9.2.7.0")
+    expect(JRUBY_VERSION).to eq("9.2.8.0")
   end
 end
 

data/versions-gem-copy.yml

@@ -1,14 +1,14 @@
 ---
 # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var
-logstash: 7.3.2
-logstash-core: 7.3.2
+logstash: 7.4.0
+logstash-core: 7.4.0
 logstash-core-plugin-api: 2.1.16
 
 # jruby must reference a *released* version of jruby which can be downloaded from the official download url
 # *and* for which jars artifacts are published for compile-time
 jruby:
-  version: 9.2.7.0
-  sha1: dc35f9bb991f526f058bf6b9591c460f98cffe9e
+  version: 9.2.8.0
+  sha1: 5b0b73b3d696afaeac92e6f8879dedcc63ac39d8
 
 # jruby-runtime-override, if specified, will override the jruby version installed in vendor/jruby for logstash runtime only,
 # not for the compile-time jars

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 7.3.2
+  version: 7.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-06 00:00:00.000000000 Z
+date: 2019-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,8 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.12'
   name: pry
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -31,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.19
   name: stud
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -45,8 +45,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: clamp
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -59,8 +59,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.2'
   name: filesize
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -73,8 +73,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: gems
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -87,8 +87,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -104,8 +104,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.6.11
   name: rack
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -124,8 +124,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.4.6
   name: sinatra
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -141,8 +141,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
   name: puma
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -155,8 +155,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.10'
   name: jruby-openssl
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -169,8 +169,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.10'
   name: chronic_duration
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -183,8 +183,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: treetop
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -197,8 +197,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: i18n
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -211,8 +211,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.8'
   name: minitar
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -225,8 +225,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: rubyzip
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -239,8 +239,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.3.6
   name: thread_safe
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -253,8 +253,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.4.9
   name: jrjackson
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
@@ -267,8 +267,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5'
   name: elasticsearch
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -281,8 +281,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: manticore
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -464,6 +464,8 @@ files:
 - spec/logstash/agent/converge_spec.rb
 - spec/logstash/agent/metrics_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/api/commands/default_metadata_spec.rb
+- spec/logstash/api/commands/node_spec.rb
 - spec/logstash/api/commands/stats_spec.rb
 - spec/logstash/api/errors_spec.rb
 - spec/logstash/api/modules/logging_spec.rb
@@ -608,6 +610,8 @@ test_files:
 - spec/logstash/agent/converge_spec.rb
 - spec/logstash/agent/metrics_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/api/commands/default_metadata_spec.rb
+- spec/logstash/api/commands/node_spec.rb
 - spec/logstash/api/commands/stats_spec.rb
 - spec/logstash/api/errors_spec.rb
 - spec/logstash/api/modules/logging_spec.rb