logstash-core 5.3.3-java → 5.4.0-java

data/lib/logstash/execution_context.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+module LogStash
+  class ExecutionContext
+    attr_reader :pipeline_id
+
+    def initialize(pipeline_id)
+      @pipeline_id = pipeline_id
+    end
+  end
+end

data/lib/logstash/filter_delegator.rb

@@ -14,7 +14,7 @@ module LogStash
     ]
     def_delegators :@filter, *DELEGATED_METHODS
 
-    def initialize(logger, klass, metric, plugin_args)
+    def initialize(logger, klass, metric, execution_context, plugin_args)
       @logger = logger
       @klass = klass
       @id = plugin_args["id"]
@@ -23,6 +23,7 @@ module LogStash
       # Scope the metrics to the plugin
       namespaced_metric = metric.namespace(@id.to_sym)
       @filter.metric = namespaced_metric
+      @filter.execution_context = execution_context
 
       @metric_events = namespaced_metric.namespace(:events)
       namespaced_metric.gauge(:name, config_name)
@@ -42,7 +43,7 @@ module LogStash
       new_events = @filter.multi_filter(events)
       clock.stop
 
-      # There is no garantee in the context of filter
+      # There is no guarantee in the context of filter
       # that EVENTS_INT == EVENTS_OUT, see the aggregates and
       # the split filter
       c = new_events.count { |event| !event.cancelled? }

data/lib/logstash/inputs/base.rb

@@ -89,11 +89,25 @@ class LogStash::Inputs::Base < LogStash::Plugin
     stop
   end
 
-  # stop? should never be overriden
+  # stop? should never be overridden
   public
   def stop?
     @stop_called.value
   end
+  
+  def clone
+    cloned = super
+    cloned.codec = @codec.clone if @codec
+    cloned
+  end
+
+  def execution_context=(context)
+    super
+    # There is no easy way to propage an instance variable into the codec, because the codec
+    # are created at the class level
+    @codec.execution_context = context
+    context
+  end
 
   protected
   def decorate(event)

data/lib/logstash/instrument/collector.rb

@@ -29,7 +29,7 @@ module LogStash module Instrument
     # of update the metric
     #
     # If there is a problem with the key or the type of metric we will record an error
-    # but we wont stop processing events, theses errors are not considered fatal.
+    # but we won't stop processing events, theses errors are not considered fatal.
     #
     def push(namespaces_path, key, type, *metric_type_params)
       begin

data/lib/logstash/instrument/metric.rb

@@ -77,7 +77,7 @@ module LogStash module Instrument
     private
     # Allow to calculate the execution of a block of code.
     # This class support 2 differents syntax a block or the return of
-    # the object itself, but in the later case the metric wont be recorded
+    # the object itself, but in the later case the metric won't be recorded
     # Until we call `#stop`.
     #
     # @see LogStash::Instrument::Metric#time
@@ -96,7 +96,9 @@ module LogStash module Instrument
       end
 
       def stop
-        @metric.report_time(@namespace, @key, (MILLISECONDS * (Time.now - @start_time)).to_i)
+        execution_time = (MILLISECONDS * (Time.now - @start_time)).to_i
+        @metric.report_time(@namespace, @key, execution_time)
+        execution_time
       end
     end
   end

data/lib/logstash/instrument/metric_store.rb

@@ -52,7 +52,7 @@ module LogStash module Instrument
       # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
       # `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
       # have it in the `@metric_store` for structured search so we add it there too.
-      if found_value = @fast_lookup.put_if_absent([namespaces, key], provided_value)
+      if found_value = @fast_lookup.put_if_absent(namespaces.dup << key, provided_value)
         return found_value
       else
         @structured_lookup_mutex.synchronize do
@@ -73,7 +73,7 @@ module LogStash module Instrument
     # If you use the `,` on a key the metric store will return the both values at that level
     #
     # The returned hash will keep the same structure as it had in the `Concurrent::Map`
-    # but will be a normal ruby hash. This will allow the api to easily seriliaze the content
+    # but will be a normal ruby hash. This will allow the api to easily serialize the content
     # of the map
     #
     # @param [Array] The path where values should be located
@@ -131,7 +131,7 @@ module LogStash module Instrument
     # }
     def extract_metrics(path, *keys)
       keys.reduce({}) do |acc,k|
-        # Simplifiy 1-length keys
+        # Simplify 1-length keys
         k = k.first if k.is_a?(Array) && k.size == 1
 
         # If we have array values here we need to recurse
@@ -162,6 +162,10 @@ module LogStash module Instrument
       end
     end    
 
+    def has_metric?(*path)
+      @fast_lookup[path]
+    end
+
     # Return all the individuals Metric,
     # This call mimic a Enum's each if a block is provided
     #
@@ -179,9 +183,9 @@ module LogStash module Instrument
     alias_method :all, :each
 
     def prune(path)
-      key_paths = key_paths(path).map {|k| k.to_sym }
+      key_paths = key_paths(path).map(&:to_sym)
       @structured_lookup_mutex.synchronize do
-        keys_to_delete = @fast_lookup.keys.select {|namespace, _| (key_paths - namespace).empty? }
+        keys_to_delete = @fast_lookup.keys.select {|namespace| (key_paths - namespace[0..-2]).empty? }
         keys_to_delete.each {|k| @fast_lookup.delete(k) }
         delete_from_map(@store, key_paths)
       end

data/lib/logstash/instrument/null_metric.rb

@@ -54,6 +54,7 @@ module LogStash module Instrument
     # @see LogStash::Instrument::TimedExecution`
     class NullTimedExecution
       def self.stop
+        0
       end
     end
   end

data/lib/logstash/instrument/periodic_poller/cgroup.rb

@@ -9,7 +9,7 @@ module LogStash module Instrument module PeriodicPoller
     include LogStash::Util::Loggable
 
     CONTROL_GROUP_RE = Regexp.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)");
-    CONTROLLER_SEPERATOR_RE = ","
+    CONTROLLER_SEPARATOR_RE = ","
 
     PROC_SELF_CGROUP_FILE = Pathname.new("/proc/self/cgroup")
     PROC_CGROUP_CPU_DIR = Pathname.new("/sys/fs/cgroup/cpu")
@@ -36,8 +36,8 @@ module LogStash module Instrument module PeriodicPoller
 
         read_proc_self_cgroup_lines.each do |line|
           matches = CONTROL_GROUP_RE.match(line)
-          # multiples controlles, same hierachy
-          controllers = matches[1].split(CONTROLLER_SEPERATOR_RE)
+          # multiples controls, same hierarchy
+          controllers = matches[1].split(CONTROLLER_SEPARATOR_RE)
           controllers.each_with_object(response) { |controller| response[controller] = matches[2] }
         end
 

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -34,6 +34,13 @@ module LogStash module Instrument module PeriodicPoller
       end
     end
 
+    MEMORY_TRANSPOSE_MAP = {
+      "usage.used" => :used_in_bytes,
+      "usage.committed" => :committed_in_bytes,
+      "usage.max" => :max_in_bytes,
+      "peak.max" => :peak_max_in_bytes,
+      "peak.used" => :peak_used_in_bytes
+    }
 
     attr_reader :metric
 
@@ -52,8 +59,6 @@ module LogStash module Instrument module PeriodicPoller
       collect_load_average
     end
 
-    private
-
     def collect_gc_stats
       garbage_collectors = ManagementFactory.getGarbageCollectorMXBeans()
 
@@ -141,7 +146,6 @@ module LogStash module Instrument module PeriodicPoller
       end
     end
 
-
     def build_pools_metrics(data)
       heap = data["heap"]
       old  = {}
@@ -164,11 +168,10 @@ module LogStash module Instrument module PeriodicPoller
       collection.reduce(default_information_accumulator) do |m,e|
         e = { e[0] => e[1] } if e.is_a?(Array)
         e.each_pair do |k,v|
-          m[:used_in_bytes] += v       if k.include?("used")
-          m[:committed_in_bytes] += v  if k.include?("committed")
-          m[:max_in_bytes] += v        if k.include?("max")
-          m[:peak_max_in_bytes] += v   if k.include?("peak.max")
-          m[:peak_used_in_bytes] += v  if k.include?("peak.used")
+          if MEMORY_TRANSPOSE_MAP.include?(k)
+            transpose_key = MEMORY_TRANSPOSE_MAP[k]
+            m[transpose_key] += v
+          end
         end
         m
       end

data/lib/logstash/instrument/periodic_poller/load_average.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+java_import "java.lang.management.ManagementFactory"
+
 module LogStash module Instrument module PeriodicPoller
   class LoadAverage
     class Windows
@@ -11,8 +13,8 @@ module LogStash module Instrument module PeriodicPoller
       LOAD_AVG_FILE = "/proc/loadavg"
       TOKEN_SEPARATOR = " "
 
-      def self.get
-        load_average = ::File.read(LOAD_AVG_FILE).chomp.split(TOKEN_SEPARATOR)
+      def self.get(content = ::File.read(LOAD_AVG_FILE))
+        load_average = content.chomp.split(TOKEN_SEPARATOR)
 
         {
           :"1m" => load_average[0].to_f,

data/lib/logstash/instrument/wrapped_write_client.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+module LogStash module Instrument
+  class WrappedWriteClient
+    def initialize(write_client, pipeline, metric, plugin)
+      @write_client = write_client
+
+      pipeline_id = pipeline.pipeline_id.to_s.to_sym
+      plugin_type = "#{plugin.class.plugin_type}s".to_sym
+
+      @events_metrics = metric.namespace([:stats, :events])
+      @pipeline_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :events])
+      @plugin_events_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :plugins, plugin_type, plugin.id.to_sym, :events])
+
+      define_initial_metrics_values
+    end
+
+    def get_new_batch
+      @write_client.get_new_batch
+    end
+
+    def push(event)
+      record_metric { @write_client.push(event) }
+    end
+    alias_method(:<<, :push)
+
+    def push_batch(batch)
+      record_metric(batch.size) { @write_client.push_batch(batch) }
+    end
+
+    private
+    def record_metric(size = 1)
+      @events_metrics.increment(:in, size)
+      @pipeline_metrics.increment(:in, size)
+      @plugin_events_metrics.increment(:out, size)
+
+      clock = @events_metrics.time(:queue_push_duration_in_millis)
+
+      result = yield
+
+      # Reuse the same values for all the endpoints to make sure we don't have skew in times.
+      execution_time = clock.stop
+
+      @pipeline_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+
+      result
+    end
+
+    def define_initial_metrics_values
+      @events_metrics.increment(:in, 0)
+      @pipeline_metrics.increment(:in, 0)
+      @plugin_events_metrics.increment(:out, 0)
+
+      @events_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @pipeline_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, 0)
+    end
+  end
+end end

data/lib/logstash/java_integration.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require "java"
 
-# this is mainly for usage with JrJackson json parsing in :raw mode which genenerates
+# this is mainly for usage with JrJackson json parsing in :raw mode which generates
 # Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.
 # these object already quacks like their Ruby equivalents Array and Hash but they will
 # not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
@@ -35,7 +35,7 @@ map_mixin = lambda do
   # this bug makes has_key? (and all its aliases) return false for a key that has a nil value.
   # Only LinkedHashMap is patched here because patching the Map interface is not working.
   # TODO find proper fix, and submit upstream
-  # releavant JRuby files:
+  # relevant JRuby files:
   # https://github.com/jruby/jruby/blob/master/core/src/main/ruby/jruby/java/java_ext/java.util.rb
   # https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/java/proxies/MapJavaProxy.java
   def has_key?(key)

data/lib/logstash/output_delegator.rb

@@ -7,7 +7,7 @@ require "logstash/output_delegator_strategies/legacy"
 module LogStash class OutputDelegator
   attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events, :id
 
-  def initialize(logger, output_class, metric, strategy_registry, plugin_args)
+  def initialize(logger, output_class, metric, execution_context, strategy_registry, plugin_args)
     @logger = logger
     @output_class = output_class
     @metric = metric
@@ -22,7 +22,7 @@ module LogStash class OutputDelegator
 
     @strategy = strategy_registry.
                   class_for(self.concurrency).
-                  new(@logger, @output_class, @namespaced_metric, plugin_args)
+                  new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
   end
 
   def config_name

data/lib/logstash/output_delegator_strategies/legacy.rb

@@ -2,10 +2,13 @@
 module LogStash module OutputDelegatorStrategies class Legacy
   attr_reader :worker_count, :workers
   
-  def initialize(logger, klass, metric, plugin_args)
+  def initialize(logger, klass, metric, execution_context, plugin_args)
     @worker_count = (plugin_args["workers"] || 1).to_i
     @workers = @worker_count.times.map { klass.new(plugin_args) }
-    @workers.each {|w| w.metric = metric }
+    @workers.each do |w|
+      w.metric = metric
+      w.execution_context = execution_context
+    end
     @worker_queue = SizedQueue.new(@worker_count)
     @workers.each {|w| @worker_queue << w}
   end

data/lib/logstash/output_delegator_strategies/shared.rb

@@ -1,7 +1,8 @@
 module LogStash module OutputDelegatorStrategies class Shared
-  def initialize(logger, klass, metric, plugin_args)
+  def initialize(logger, klass, metric, execution_context, plugin_args)
     @output = klass.new(plugin_args)
     @output.metric = metric
+    @output.execution_context = execution_context
   end
   
   def register

data/lib/logstash/output_delegator_strategies/single.rb

@@ -1,7 +1,8 @@
 module LogStash module OutputDelegatorStrategies class Single
-  def initialize(logger, klass, metric, plugin_args)
+  def initialize(logger, klass, metric, execution_context, plugin_args)
     @output = klass.new(plugin_args)
     @output.metric = metric
+    @output.execution_context = execution_context
     @mutex = Mutex.new
   end
 

data/lib/logstash/outputs/base.rb

@@ -105,6 +105,14 @@ class LogStash::Outputs::Base < LogStash::Plugin
     self.class.concurrency
   end
 
+  def execution_context=(context)
+    super
+    # There is no easy way to propage an instance variable into the codec, because the codec
+    # are created at the class level
+    @codec.execution_context = context
+    context
+  end
+
   private
   def output?(event)
     # TODO: noop for now, remove this once we delete this call from all plugins

data/lib/logstash/patches/cabin.rb

@@ -9,7 +9,7 @@ if ENV["PROFILE_BAD_LOG_CALLS"] || ($DEBUGLIST || []).include?("log")
   # Basically, the following is wastes tons of effort creating objects that are
   # never used if the log level hides the log:
   #
-  #     logger.debug("something happend", :what => Happened)
+  #     logger.debug("something happened", :what => Happened)
   #
   # This is shown to be 4x faster:
   #

data/lib/logstash/patches/stronger_openssl_defaults.rb

@@ -54,7 +54,7 @@ class OpenSSL::SSL::SSLContext
   #
   # This monkeypatch doesn't enforce a `VERIFY_MODE` on the SSLContext,
   # SSLContext are both used for the client and the server implementation,
-  # If set the `verify_mode` to peer the server wont accept any connection,
+  # If set the `verify_mode` to peer the server won't accept any connection,
   # because it will try to verify the client certificate, this is a protocol
   # details implemented at the plugin level.
   #

data/lib/logstash/pipeline.rb

@@ -17,9 +17,11 @@ require "logstash/instrument/namespaced_metric"
 require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
+require "logstash/instrument/wrapped_write_client"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
 require "logstash/queue_factory"
+require "logstash/execution_context"
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
@@ -41,6 +43,7 @@ module LogStash; class BasePipeline
     @inputs = nil
     @filters = nil
     @outputs = nil
+    @execution_context = LogStash::ExecutionContext.new(@pipeline_id)
 
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
@@ -88,12 +91,15 @@ module LogStash; class BasePipeline
     klass = Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric,  OutputDelegatorStrategyRegistry.instance, args)
+      OutputDelegator.new(@logger, klass, type_scoped_metric, @execution_context, OutputDelegatorStrategyRegistry.instance, args)
     elsif plugin_type == "filter"
-      FilterDelegator.new(@logger, klass, type_scoped_metric, args)
+      FilterDelegator.new(@logger, klass, type_scoped_metric, @execution_context, args)
     else # input
       input_plugin = klass.new(args)
-      input_plugin.metric = type_scoped_metric.namespace(id)
+      scoped_metric = type_scoped_metric.namespace(id.to_sym)
+      scoped_metric.gauge(:name, input_plugin.config_name)
+      input_plugin.metric = scoped_metric
+      input_plugin.execution_context = @execution_context
       input_plugin
     end
   end
@@ -148,12 +154,13 @@ module LogStash; class Pipeline < BasePipeline
     @input_queue_client = @queue.write_client
     @filter_queue_client = @queue.read_client
     @signal_queue = Queue.new
-    # Note that @infilght_batches as a central mechanism for tracking inflight
+    # Note that @inflight_batches as a central mechanism for tracking inflight
     # batches will fail if we have multiple read clients here.
     @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
     @filter_queue_client.set_pipeline_metric(
         metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
     )
+    @drain_queue =  @settings.get_value("queue.drain")
 
     @events_filtered = Concurrent::AtomicFixnum.new(0)
     @events_consumed = Concurrent::AtomicFixnum.new(0)
@@ -250,6 +257,10 @@ module LogStash; class Pipeline < BasePipeline
     @running.false?
   end
 
+  def system?
+    settings.get_value("pipeline.system")
+  end
+
   # register_plugin simply calls the plugin #register method and catches & logs any error
   # @param plugin [Plugin] the plugin to register
   # @return [Plugin] the registered plugin
@@ -326,26 +337,32 @@ module LogStash; class Pipeline < BasePipeline
   # Main body of what a worker thread does
   # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
-    running = true
+    shutdown_requested = false
 
     @filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
 
-    while running
-      batch = @filter_queue_client.take_batch
+    while true
       signal = @signal_queue.empty? ? NO_SIGNAL : @signal_queue.pop
-      running = !signal.shutdown?
+      shutdown_requested |= signal.shutdown? # latch on shutdown signal
 
+      batch = @filter_queue_client.read_batch # metrics are started in read_batch
       @events_consumed.increment(batch.size)
-
       filter_batch(batch)
-
-      if signal.flush? || signal.shutdown?
-        flush_filters_to_batch(batch, :final => signal.shutdown?)
-      end
-
+      flush_filters_to_batch(batch, :final => false) if signal.flush?
       output_batch(batch)
       @filter_queue_client.close_batch(batch)
+
+      # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
+      break if shutdown_requested && !draining_queue?
     end
+
+    # we are shutting down, queue is drained if it was required, now  perform a final flush.
+    # for this we need to create a new empty batch to contain the final flushed events
+    batch = @filter_queue_client.new_batch
+    @filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
+    flush_filters_to_batch(batch, :final => true)
+    output_batch(batch)
+    @filter_queue_client.close_batch(batch)
   end
 
   def filter_batch(batch)
@@ -411,7 +428,7 @@ module LogStash; class Pipeline < BasePipeline
     # first make sure we can register all input plugins
     register_plugins(@inputs)
 
-    # then after all input plugins are sucessfully registered, start them
+    # then after all input plugins are successfully registered, start them
     @inputs.each { |input| start_input(input) }
   end
 
@@ -422,7 +439,8 @@ module LogStash; class Pipeline < BasePipeline
   def inputworker(plugin)
     Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
     begin
-      plugin.run(@input_queue_client)
+      input_queue_client = wrapped_write_client(plugin)
+      plugin.run(input_queue_client)
     rescue => e
       if plugin.stop?
         @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
@@ -456,11 +474,11 @@ module LogStash; class Pipeline < BasePipeline
   # @param before_stop [Proc] code block called before performing stop operation on input plugins
   def shutdown(&before_stop)
     # shutdown can only start once the pipeline has completed its startup.
-    # avoid potential race conditoon between the startup sequence and this
+    # avoid potential race condition between the startup sequence and this
     # shutdown method which can be called from another thread at any time
     sleep(0.1) while !ready?
 
-    # TODO: should we also check against calling shutdown multiple times concurently?
+    # TODO: should we also check against calling shutdown multiple times concurrently?
 
     before_stop.call if block_given?
 
@@ -604,4 +622,14 @@ module LogStash; class Pipeline < BasePipeline
       :flushing => @flushing
     }
   end
-end end
+
+  private
+
+  def draining_queue?
+    @drain_queue ? !@filter_queue_client.empty? : false
+  end
+
+  def wrapped_write_client(plugin)
+    LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
+  end
+end; end