logstash-core 6.5.4-java → 6.6.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 742ed9d61d2a33fc1d8918701800cc4a06f738f5cdf5bdda270310c75bb3bf4c
-  data.tar.gz: d846918c5a9efe9b1e9a32e195f34d3a6fd8a87a6c747013e5980741c3ab10b6
+  metadata.gz: cbe096068df3326925926019e2137eb56527f92c91c9d14861616d85e19ac46b
+  data.tar.gz: 8a5a795fbb82efc19a01232f8ef173d668948a33f75aaa749df73e0cc075e206
 SHA512:
-  metadata.gz: 74a63b5a650d4fe41662dbdc168a37d58fcf9ab311461506559ff94d570fca1fe3f08a0e7314825373e1a1e268f58118cbb6a05cd3b050ec1cb8f4c54c1c0c61
-  data.tar.gz: 18f27ce7eab9b0a966ef9cd52344e68d6670a74268ad98236f3fd19d128265ddce2cd899985d297456295dbd1c269dd33d3087abffda8b909d76b08e6ec2e1e8
+  metadata.gz: 99c5e85d59c293bd4479ca1d4805cb0dff72b225ed97aa7351060c452c1e73e495a24bf3fbc4142df0e67f01bb1eaa90893af1cd8c30d22f6a61d4fed2a0cfd0
+  data.tar.gz: 977df2140b2d9596ad4d08e784bd03cbb063c2c40b9a158771eaf91b6b4f09f80e0804dce33f801c002dfe2cf4664b8a544f588565c4aa492fa2f83c7c964339

data/lib/logstash/api/modules/stats.rb

@@ -10,7 +10,7 @@ module LogStash
         # return hot threads information
         get "/jvm/hot_threads" do
           begin
-            top_threads_count = params["threads"] || 3
+            top_threads_count = params["threads"] || 10
             ignore_idle_threads = params["ignore_idle_threads"] || true
             options = {
               :threads => top_threads_count.to_i,

data/lib/logstash/config/config_ast.rb

@@ -136,7 +136,7 @@ module LogStash; module Config; module AST
               events.each{|e| block.call(e)}
             end
 
-            if @generated_objects[:#{name}].respond_to?(:flush)
+            if !@generated_objects[:#{name}].nil? && @generated_objects[:#{name}].has_flush
               @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
               @shutdown_flushers << @generated_objects[:#{name}_flush]
             end

data/lib/logstash/environment.rb

@@ -62,6 +62,7 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::Boolean.new("queue.checkpoint.retry", false),
             Setting::Boolean.new("dead_letter_queue.enable", false),
             Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),

data/lib/logstash/filter_delegator.rb

@@ -1,69 +1,2 @@
-# encoding: utf-8
-#
-module LogStash
-  class FilterDelegator
-    extend Forwardable
-    DELEGATED_METHODS = [
-      :register,
-      :close,
-      :threadsafe?,
-      :do_close,
-      :do_stop,
-      :periodic_flush,
-      :reloadable?
-    ]
-    def_delegators :@filter, *DELEGATED_METHODS
-
-    attr_reader :id
-
-    def initialize(filter, id)
-      @klass = filter.class
-      @id = id
-      @filter = filter
-
-      # Scope the metrics to the plugin
-      namespaced_metric = filter.metric
-      @metric_events = namespaced_metric.namespace(:events)
-      @metric_events_in = @metric_events.counter(:in)
-      @metric_events_out = @metric_events.counter(:out)
-      @metric_events_time = @metric_events.counter(:duration_in_millis)
-      namespaced_metric.gauge(:name, config_name)
-
-      # Not all the filters will do bufferings
-      define_flush_method if @filter.respond_to?(:flush)
-    end
-
-    def config_name
-      @klass.config_name
-    end
-
-    def multi_filter(events)
-      @metric_events_in.increment(events.size)
-
-      start_time = java.lang.System.nano_time
-      new_events = @filter.multi_filter(events)
-      @metric_events_time.increment((java.lang.System.nano_time - start_time) / 1_000_000)
-
-      # There is no guarantee in the context of filter
-      # that EVENTS_IN == EVENTS_OUT, see the aggregates and
-      # the split filter
-      c = new_events.count { |event| !event.cancelled? }
-      @metric_events_out.increment(c) if c > 0
-      new_events
-    end
-
-    private
-    def define_flush_method
-      define_singleton_method(:flush) do |options = {}|
-        # we also need to trace the number of events
-        # coming from a specific filters.
-        new_events = @filter.flush(options)
-
-        # Filter plugins that does buffering or spooling of events like the
-        # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
-        @metric_events_out.increment(new_events.size) if new_events && new_events.size > 0
-        new_events
-      end
-    end
-  end
-end
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly include it.

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -20,8 +20,8 @@ java_import 'org.logstash.instrument.reports.ProcessReport'
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
     class GarbageCollectorName
-      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation", "scavenge"])
-      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "global"])
+      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation", "scavenge", "GPGC New"])
+      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "global", "GPGC Old"])
 
       YOUNG = :young
       OLD = :old
@@ -68,7 +68,7 @@ module LogStash module Instrument module PeriodicPoller
         logger.debug("collector name", :name => collector_name)
         name = GarbageCollectorName.get(collector_name)
         if name.nil?
-          logger.error("Unknown garbage collector name", :name => name)
+          logger.error("Unknown garbage collector name", :name => collector_name)
         else
           metric.gauge([:jvm, :gc, :collectors, name], :collection_count, collector.getCollectionCount())
           metric.gauge([:jvm, :gc, :collectors, name], :collection_time_in_millis, collector.getCollectionTime())

data/lib/logstash/java_pipeline.rb

@@ -26,6 +26,8 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     @worker_threads = []
 
+    @java_inputs_controller = org.logstash.execution.InputsController.new(lir_execution.javaInputs)
+
     @drain_queue =  settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
 
     @events_filtered = java.util.concurrent.atomic.LongAdder.new
@@ -216,11 +218,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new do
+          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           org.logstash.execution.WorkerLoop.new(
               lir_execution, filter_queue_client, @events_filtered, @events_consumed,
               @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
         end
-        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 
@@ -242,6 +244,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def wait_inputs
     @input_threads.each(&:join)
+    @java_inputs_controller.awaitStop
   end
 
   def start_inputs
@@ -260,6 +263,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     # then after all input plugins are successfully registered, start them
     inputs.each { |input| start_input(input) }
+    @java_inputs_controller.startInputs(self)
   end
 
   def start_input(plugin)
@@ -325,6 +329,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   def stop_inputs
     @logger.debug("Closing inputs", default_logging_keys)
     inputs.each(&:do_stop)
+    @java_inputs_controller.stopInputs
     @logger.debug("Closed inputs", default_logging_keys)
   end
 

data/lib/logstash/pipeline.rb

@@ -283,9 +283,9 @@ module LogStash; class Pipeline < BasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
+          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           _pipeline.worker_loop(_b_size, _b_delay)
         end
-        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 

data/lib/logstash/pipeline_settings.rb

@@ -28,6 +28,7 @@ module LogStash
       "queue.checkpoint.acks",
       "queue.checkpoint.interval",
       "queue.checkpoint.writes",
+      "queue.checkpoint.retry",
       "queue.drain",
       "queue.max_bytes",
       "queue.max_events",

data/lib/logstash/plugins/registry.rb

@@ -262,11 +262,14 @@ module LogStash module Plugins
     # @param name [String] plugin name
     # @return [Boolean] true if klass is a valid plugin for name
     def is_a_plugin?(klass, name)
-      klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
+      (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
+      (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
     end
 
     def add_plugin(type, name, klass)
-      if !exists?(type, name)
+      if klass.respond_to?("javaClass", true)
+        @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
+      elsif !exists?(type, name)
         specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
         @registry[key_for(type, name)] = specification_klass.new(type, name, klass)
       else

data/lib/logstash/util/thread_dump.rb

@@ -5,7 +5,7 @@ module LogStash
   module Util
     class ThreadDump
       SKIPPED_THREADS             = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
-      THREADS_COUNT_DEFAULT       = 3.freeze
+      THREADS_COUNT_DEFAULT       = 10.freeze
       IGNORE_IDLE_THREADS_DEFAULT = true.freeze
 
       attr_reader :top_count, :ignore, :dump

data/locales/en.yml

@@ -298,7 +298,7 @@ en:
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         java-execution: |+
-          (Beta) Use new Java execution engine.
+          Use Java execution engine.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+

data/spec/logstash/acked_queue_concurrent_stress_spec.rb

@@ -14,7 +14,7 @@ describe LogStash::WrappedAckedQueue, :stress_test => true do
     let(:reject_memo_keys) { [:reject_memo_keys, :path, :queue, :writer_threads, :collector, :metric, :reader_threads, :output_strings] }
 
     let(:queue) do
-      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, queue_capacity)
+      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, false, queue_capacity)
     end
 
     let(:writer_threads) do

data/spec/logstash/filter_delegator_spec.rb

@@ -51,7 +51,7 @@ describe LogStash::FilterDelegator do
     end
 
     it "defines a flush method" do
-      expect(subject.respond_to?(:flush)).to be_truthy
+      expect(subject.has_flush).to be_truthy
     end
 
     context "when the flush return events" do
@@ -128,7 +128,7 @@ describe LogStash::FilterDelegator do
     end
 
     it "doesnt define a flush method" do
-      expect(subject.respond_to?(:flush)).to be_falsey
+      expect(subject.has_flush).to be_falsey
     end
 
     it "increments the in/out of the metric" do
@@ -145,14 +145,4 @@ describe LogStash::FilterDelegator do
     end
   end
 
-  context "delegate methods to the original plugin" do
-    # I am not testing the behavior of these methods
-    # this is done in the plugin tests. I just want to make sure
-    # the proxy delegates the methods.
-    LogStash::FilterDelegator::DELEGATED_METHODS.each do |method|
-      it "delegate method: `#{method}` to the filter" do
-        expect(subject.respond_to?(method))
-      end
-    end
-  end
 end

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -110,7 +110,7 @@ describe LogStash::WrappedWriteClient do
 
   context "WrappedAckedQueue" do
     let(:path) { Stud::Temporary.directory }
-    let(:queue) { LogStash::WrappedAckedQueue.new(path, 1024, 10, 1024, 1024, 1024, 4096) }
+    let(:queue) { LogStash::WrappedAckedQueue.new(path, 1024, 10, 1024, 1024, 1024, false, 4096) }
 
     before do
       read_client.set_events_metric(metric.namespace([:stats, :events]))

data/spec/logstash/java_filter_delegator_spec.rb

@@ -6,7 +6,7 @@ require "support/shared_contexts"
 
 java_import org.logstash.RubyUtil
 
-describe LogStash::JavaFilterDelegator do
+describe LogStash::FilterDelegator do
 
   class MockGauge
     def increment(_)
@@ -182,14 +182,4 @@ describe LogStash::JavaFilterDelegator do
     end
   end
 
-  context "delegate methods to the original plugin" do
-    # I am not testing the behavior of these methods
-    # this is done in the plugin tests. I just want to make sure
-    # the proxy delegates the methods.
-    LogStash::FilterDelegator::DELEGATED_METHODS.each do |method|
-      it "delegate method: `#{method}` to the filter" do
-        expect(subject.respond_to?(method))
-      end
-    end
-  end
 end

data/spec/logstash/queue_factory_spec.rb

@@ -14,6 +14,7 @@ describe LogStash::QueueFactory do
       LogStash::Setting::Numeric.new("queue.checkpoint.acks", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.writes", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.interval", 1000),
+      LogStash::Setting::Boolean.new("queue.checkpoint.retry", false),
       LogStash::Setting::String.new("pipeline.id", pipeline_id),
       LogStash::Setting::PositiveInteger.new("pipeline.batch.size", 125),
       LogStash::Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum)

data/spec/logstash/util/wrapped_acked_queue_spec.rb

@@ -38,7 +38,7 @@ describe LogStash::WrappedAckedQueue do
     let(:checkpoint_writes) { 1024 }
     let(:checkpoint_interval) { 0 }
     let(:path) { Stud::Temporary.directory }
-    let(:queue) { LogStash::WrappedAckedQueue.new(path, page_capacity, max_events, checkpoint_acks, checkpoint_writes, checkpoint_interval, max_bytes) }
+    let(:queue) { LogStash::WrappedAckedQueue.new(path, page_capacity, max_events, checkpoint_acks, checkpoint_writes, checkpoint_interval, false, max_bytes) }
 
     after do
       queue.close

data/versions-gem-copy.yml

@@ -1,6 +1,6 @@
 ---
-logstash: 6.5.4
-logstash-core: 6.5.4
+logstash: 6.6.0
+logstash-core: 6.6.0
 logstash-core-plugin-api: 2.1.16
 
 # jruby must reference a *released* version of jruby which can be downloaded from the official download url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 6.5.4
+  version: 6.6.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-17 00:00:00.000000000 Z
+date: 2019-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement