logstash-core 6.8.14-java → 7.0.0.alpha1-java

data/lib/logstash/event.rb

@@ -1,12 +1,6 @@
 # encoding: utf-8
 
-require "logstash/json"
-
-# transient pipeline events for normal in-flow signaling as opposed to
-# flow altering exceptions. for now having base classes is adequate and
-# in the future it might be necessary to refactor using like a BaseEvent
-# class to have a common interface for all pipeline events to support
-# eventual queueing persistence for example, TBD.
+# used only in the Ruby execution engine
 module LogStash
   class SignalEvent
     def flush?; raise "abstract method"; end;
@@ -31,21 +25,4 @@ module LogStash
   FLUSH = FlushEvent.new
   SHUTDOWN = ShutdownEvent.new
   NO_SIGNAL = NoSignal.new
-
-  class Event
-    MSG_BRACKETS_METHOD_MISSING = "Direct event field references (i.e. event['field']) have been disabled in favor of using event get and set methods (e.g. event.get('field')). Please consult the Logstash 5.0 breaking changes documentation for more details.".freeze
-    MSG_BRACKETS_EQUALS_METHOD_MISSING = "Direct event field references (i.e. event['field'] = 'value') have been disabled in favor of using event get and set methods (e.g. event.set('field', 'value')). Please consult the Logstash 5.0 breaking changes documentation for more details.".freeze
-    RE_BRACKETS_METHOD = /^\[\]$/.freeze
-    RE_BRACKETS_EQUALS_METHOD = /^\[\]=$/.freeze
-
-    def method_missing(method_name, *arguments, &block)
-      if RE_BRACKETS_METHOD.match(method_name.to_s)
-        raise NoMethodError.new(MSG_BRACKETS_METHOD_MISSING)
-      end
-      if RE_BRACKETS_EQUALS_METHOD.match(method_name.to_s)
-        raise NoMethodError.new(MSG_BRACKETS_EQUALS_METHOD_MISSING)
-      end
-      super
-    end
-  end
 end

data/lib/logstash/filter_delegator.rb

@@ -1,2 +1,69 @@
-# The contents of this file have been ported to Java. It is included for for compatibility
-# with plugins that directly include it.
+# encoding: utf-8
+#
+module LogStash
+  class FilterDelegator
+    extend Forwardable
+    DELEGATED_METHODS = [
+      :register,
+      :close,
+      :threadsafe?,
+      :do_close,
+      :do_stop,
+      :periodic_flush,
+      :reloadable?
+    ]
+    def_delegators :@filter, *DELEGATED_METHODS
+
+    attr_reader :id
+
+    def initialize(filter, id)
+      @klass = filter.class
+      @id = id
+      @filter = filter
+
+      # Scope the metrics to the plugin
+      namespaced_metric = filter.metric
+      @metric_events = namespaced_metric.namespace(:events)
+      @metric_events_in = @metric_events.counter(:in)
+      @metric_events_out = @metric_events.counter(:out)
+      @metric_events_time = @metric_events.counter(:duration_in_millis)
+      namespaced_metric.gauge(:name, config_name)
+
+      # Not all the filters will do bufferings
+      define_flush_method if @filter.respond_to?(:flush)
+    end
+
+    def config_name
+      @klass.config_name
+    end
+
+    def multi_filter(events)
+      @metric_events_in.increment(events.size)
+
+      start_time = java.lang.System.nano_time
+      new_events = @filter.multi_filter(events)
+      @metric_events_time.increment((java.lang.System.nano_time - start_time) / 1_000_000)
+
+      # There is no guarantee in the context of filter
+      # that EVENTS_IN == EVENTS_OUT, see the aggregates and
+      # the split filter
+      c = new_events.count { |event| !event.cancelled? }
+      @metric_events_out.increment(c) if c > 0
+      new_events
+    end
+
+    private
+    def define_flush_method
+      define_singleton_method(:flush) do |options = {}|
+        # we also need to trace the number of events
+        # coming from a specific filters.
+        new_events = @filter.flush(options)
+
+        # Filter plugins that does buffering or spooling of events like the
+        # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
+        @metric_events_out.increment(new_events.size) if new_events && new_events.size > 0
+        new_events
+      end
+    end
+  end
+end

data/lib/logstash/filters/base.rb

@@ -194,6 +194,8 @@ class LogStash::Filters::Base < LogStash::Plugin
     # this is important because a construct like event["tags"].delete(tag) will not work
     # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
 
+    return if @remove_tag.empty?
+
     tags = event.get("tags")
     return unless tags
 

data/lib/logstash/instrument/metric_store.rb

@@ -302,7 +302,7 @@ module LogStash module Instrument
     #
     # @param [Concurrent::Map] Map to search for the key
     # @param [Array] List of path to create
-    # @param [Integer] Which part from the list to create
+    # @param [Fixnum] Which part from the list to create
     #
     def fetch_or_store_namespace_recursively(map, namespaces_path, idx = 0)
       current = namespaces_path[idx]

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -10,11 +10,13 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.running_user_defined_pipelines
-      pipelines.each do |_, pipeline|
-        unless pipeline.nil?
-          pipeline.collect_dlq_stats
-        end
+      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
+      unless pipelines.nil?
+        pipelines.each {|_, pipeline|
+          unless pipeline.nil?
+            pipeline.collect_dlq_stats
+          end
+        }
       end
     end
   end

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -20,8 +20,8 @@ java_import 'org.logstash.instrument.reports.ProcessReport'
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
     class GarbageCollectorName
-      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation", "scavenge", "GPGC New"])
-      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "global", "GPGC Old"])
+      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation", "scavenge"])
+      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "global"])
 
       YOUNG = :young
       OLD = :old
@@ -68,7 +68,7 @@ module LogStash module Instrument module PeriodicPoller
         logger.debug("collector name", :name => collector_name)
         name = GarbageCollectorName.get(collector_name)
         if name.nil?
-          logger.error("Unknown garbage collector name", :name => collector_name)
+          logger.error("Unknown garbage collector name", :name => name)
         else
           metric.gauge([:jvm, :gc, :collectors, name], :collection_count, collector.getCollectionCount())
           metric.gauge([:jvm, :gc, :collectors, name], :collection_time_in_millis, collector.getCollectionTime())

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,12 +11,14 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.running_user_defined_pipelines
-      pipelines.each do |_, pipeline|
-        unless pipeline.nil?
-          pipeline.collect_stats
-        end
+      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
+      unless pipelines.nil?
+        pipelines.each {|_, pipeline|
+          unless pipeline.nil?
+            pipeline.collect_stats
+          end
+        }
       end
     end
   end
-end end end
+end; end; end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -11,12 +11,12 @@ module LogStash module Instrument
   class PeriodicPollers
     attr_reader :metric
 
-    def initialize(metric, queue_type, agent)
+    def initialize(metric, queue_type, pipelines)
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
-                           PeriodicPoller::DeadLetterQueue.new(metric, agent)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
+                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
     end
 
     def start

data/lib/logstash/java_pipeline.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "thread"
 require "concurrent"
-require "logstash/event"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
@@ -39,23 +38,9 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
-
-    # @finished_execution signals that the pipeline thread has finished its execution
-    # regardless of any exceptions; it will always be true when the thread completes
     @finished_execution = Concurrent::AtomicBoolean.new(false)
-
-    # @finished_run signals that the run methods called in the pipeline thread was completed
-    # without errors and it will NOT be set if the run method exits from an exception; this
-    # is by design and necessary for the wait_until_started semantic
-    @finished_run = Concurrent::AtomicBoolean.new(false)
-
-    @thread = nil
   end # def initialize
 
-  def finished_execution?
-    @finished_execution.true?
-  end
-
   def ready?
     @ready.value
   end
@@ -97,18 +82,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @logger.debug("Starting pipeline", default_logging_keys)
 
     @finished_execution.make_false
-    @finished_run.make_false
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_run.make_true
+        @finished_execution.make_true
       rescue => e
         close
         logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
-      ensure
-        @finished_execution.make_true
       end
     end
 
@@ -123,14 +105,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def wait_until_started
     while true do
-      if @finished_run.true?
-        # it completed run without exception
+      # This should be changed with an appropriate FSM
+      # It's an edge case, if we have a pipeline with
+      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
+      # because the execution of the thread was successful and complete
+      if @finished_execution.true?
         return true
       elsif thread.nil? || !thread.alive?
-        # some exception occurred and the thread is dead
         return false
       elsif running?
-        # fully initialized and running
         return true
       else
         sleep 0.01
@@ -232,11 +215,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new do
-          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           org.logstash.execution.WorkerLoop.new(
               lir_execution, filter_queue_client, @events_filtered, @events_consumed,
               @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
         end
+        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 
@@ -257,13 +240,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def wait_inputs
-    @input_threads.each do |thread|
-      if thread.class == Java::JavaObject
-        thread.to_java.join
-      else
-        thread.join
-      end
-    end
+    @input_threads.each(&:join)
   end
 
   def start_inputs
@@ -285,11 +262,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def start_input(plugin)
-    if plugin.class == LogStash::JavaInputDelegator
-      @input_threads << plugin.start
-    else
-      @input_threads << Thread.new { inputworker(plugin) }
-    end
+    @input_threads << Thread.new { inputworker(plugin) }
   end
 
   def inputworker(plugin)
@@ -335,7 +308,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     stop_inputs
 
-    # We make this call blocking, so we know for sure when the method return the shtudown is
+    # We make this call blocking, so we know for sure when the method return the shutdown is
     # stopped
     wait_for_workers
     clear_pipeline_metrics
@@ -399,7 +372,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def plugin_threads_info
-    input_threads = @input_threads.select {|t| t.class == Thread && t.alive? }
+    input_threads = @input_threads.select {|t| t.alive? }
     worker_threads = @worker_threads.select {|t| t.alive? }
     (input_threads + worker_threads).map {|t| Util.thread_info(t) }
   end

data/lib/logstash/modules/kibana_config.rb

@@ -8,7 +8,7 @@ module LogStash module Modules class KibanaConfig
   include LogStash::Util::Loggable
 
   ALLOWED_DIRECTORIES = ["search", "visualization"]
-  attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
+  attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig
 
   # We name it `modul` here because `module` has meaning in Ruby.
   def initialize(modul, settings)

data/lib/logstash/modules/logstash_config.rb

@@ -39,7 +39,7 @@ module LogStash module Modules class LogStashConfig
   def get_setting(setting_class)
     raw_value = @settings[setting_class.name]
     # If we dont check for NIL, the Settings class will try to coerce the value
-    # and most of the it will fails when a NIL value is explicitely set.
+    # and most of the it will fails when a NIL value is explicitly set.
     # This will be fixed once we wrap the plugins settings into a Settings class
     setting_class.set(raw_value) unless raw_value.nil?
     setting_class.value

data/lib/logstash/patches/resolv.rb

@@ -1,25 +1,40 @@
 require "resolv"
 
-# ref:
-#   https://github.com/logstash-plugins/logstash-filter-dns/issues/51
-#   https://github.com/jruby/jruby/pull/5722
+# ref: https://github.com/logstash-plugins/logstash-filter-dns/issues/40
 #
-# JRuby versions starting at 9.2.0.0 have a bug in resolv.rb with a leak between the
-# DNS.allocate_request_id/DNS.free_request_id methods.
+# JRuby 9k versions prior to 9.1.16.0 have a bug which crashes IP address
+# resolution after 64k unique IP addresses resolutions.
 #
-# We are opting to load a patched full resolv.rb instead of monkey patching the
-# offending methods because we do not yet know in which upcoming version of JRuby
-# this will be fixed and we want to avoid potential conflicting monkey patches.
-# A spec which will break on JRuby upgrade will redirect here
-# to make a manual verification and eventually remove that patch here once the fix is
-# made in the JRuby version of resolv.rb.
+# Note that the oldest JRuby version in LS 6 is 9.1.13.0 and
+# JRuby 1.7.25 and 1.7.27 (the 2 versions used across LS 5) are not affected by this bug.
 
-if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.2.0.0")
-  # save verbose level and mute the "warning: already initialized constant"
-  warn_level = $VERBOSE
-  $VERBOSE = nil
+# make sure we abort if a known correct JRuby version is installed 
+# to avoid having an unnecessary legacy patch being applied in the future.
+raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.1.16.0")
 
-  require_relative "resolv_9270"
+# The code below is copied from JRuby 9.1.16.0 resolv.rb:
+# https://github.com/jruby/jruby/blob/9.1.16.0/lib/ruby/stdlib/resolv.rb#L775-L784
+#
+# JRuby is Copyright (c) 2007-2017 The JRuby project, and is released
+# under a tri EPL/GPL/LGPL license.
+# Full license available at https://github.com/jruby/jruby/blob/9.1.16.0/COPYING
 
-  $VERBOSE = warn_level
+class Resolv
+  class DNS
+    class Requester
+      class UnconnectedUDP
+        def sender(msg, data, host, port=Port)
+          sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"]
+          return nil if !sock
+          service = [IPAddr.new(host), port]
+          id = DNS.allocate_request_id(service[0], service[1])
+          request = msg.encode
+          request[0,2] = [id].pack('n')
+          return @senders[[service, id]] =
+              Sender.new(request, data, sock, host, port)
+        end
+      end
+    end
+  end
 end
+

data/lib/logstash/pipeline.rb

@@ -107,23 +107,8 @@ module LogStash; class Pipeline < BasePipeline
     @flushing = Concurrent::AtomicReference.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
     @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
-
-    # @finished_execution signals that the pipeline thread has finished its execution
-    # regardless of any exceptions; it will always be true when the thread completes
-    @finished_execution = Concurrent::AtomicBoolean.new(false)
-
-    # @finished_run signals that the run methods called in the pipeline thread was completed
-    # without errors and it will NOT be set if the run method exits from an exception; this
-    # is by design and necessary for the wait_until_started semantic
-    @finished_run = Concurrent::AtomicBoolean.new(false)
-
-    @thread = nil
   end # def initialize
 
-  def finished_execution?
-    @finished_execution.true?
-  end
-
   def ready?
     @ready.value
   end
@@ -167,19 +152,16 @@ module LogStash; class Pipeline < BasePipeline
       "pipeline.batch.size" => settings.get("pipeline.batch.size"),
       "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
 
-    @finished_execution.make_false
-    @finished_run.make_false
+    @finished_execution = Concurrent::AtomicBoolean.new(false)
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_run.make_true
+        @finished_execution.make_true
       rescue => e
         close
         @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
-      ensure
-        @finished_execution.make_true
       end
     end
 
@@ -194,14 +176,15 @@ module LogStash; class Pipeline < BasePipeline
 
   def wait_until_started
     while true do
-      if @finished_run.true?
-        # it completed run without exception
+      # This should be changed with an appropriate FSM
+      # It's an edge case, if we have a pipeline with
+      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
+      # because the execution of the thread was successful and complete
+      if @finished_execution.true?
         return true
-      elsif thread.nil? || !thread.alive?
-        # some exception occured and the thread is dead
+      elsif !thread.alive?
         return false
       elsif running?
-        # fully initialized and running
         return true
       else
         sleep 0.01
@@ -300,9 +283,9 @@ module LogStash; class Pipeline < BasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
-          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           _pipeline.worker_loop(_b_size, _b_delay)
         end
+        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 
@@ -463,7 +446,7 @@ module LogStash; class Pipeline < BasePipeline
 
     stop_inputs
 
-    # We make this call blocking, so we know for sure when the method return the shtudown is
+    # We make this call blocking, so we know for sure when the method return the shutdown is
     # stopped
     wait_for_workers
     clear_pipeline_metrics
@@ -542,7 +525,7 @@ module LogStash; class Pipeline < BasePipeline
 
   # Calculate the uptime in milliseconds
   #
-  # @return [Integer] Uptime in milliseconds, 0 if the pipeline is not started
+  # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
   def uptime
     return 0 if started_at.nil?
     ((Time.now.to_f - started_at.to_f) * 1000.0).to_i