logstash-core 5.0.2-java → 5.1.1.1-java

data/lib/logstash/instrument/metric_store.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "concurrent"
-require "logstash/event"
 require "logstash/instrument/metric_type"
 require "thread"
 

data/lib/logstash/instrument/metric_type/base.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/event"
 require "logstash/util"
 
 module LogStash module Instrument module MetricType

data/lib/logstash/instrument/namespaced_null_metric.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require "logstash/instrument/null_metric"
+
+module LogStash module Instrument
+  # This class acts a a proxy between the metric library and the user calls.
+  #
+  # This is the class that plugins authors will use to interact with the `MetricStore`
+  # It has the same public interface as `Metric` class but doesnt require to send
+  # the namespace on every call.
+  #
+  # @see Logstash::Instrument::Metric
+  class NamespacedNullMetric
+    attr_reader :namespace_name
+    # Create metric with a specific namespace
+    #
+    # @param metric [LogStash::Instrument::Metric] The metric instance to proxy
+    # @param namespace [Array] The namespace to use
+    def initialize(metric = nil, namespace_name = :null)
+      @metric = metric
+      @namespace_name = Array(namespace_name)
+    end
+
+    def increment(key, value = 1)
+    end
+
+    def decrement(key, value = 1)
+    end
+
+    def gauge(key, value)
+    end
+
+    def report_time(key, duration)
+    end
+
+    def time(key, &block)
+      if block_given?
+        yield
+      else
+        ::LogStash::Instrument::NullMetric::NullTimedExecution
+      end
+    end
+
+    def collector
+      @metric.collector
+    end
+
+    def namespace(name)
+      NamespacedNullMetric.new(metric, namespace_name + Array(name))
+    end
+
+    private
+    attr_reader :metric
+  end
+end; end

data/lib/logstash/instrument/null_metric.rb

@@ -2,50 +2,59 @@
 require "logstash/instrument/metric"
 
 module LogStash module Instrument
- # This class is used in the context when we disable the metric collection
- # for specific plugin to replace the `NamespacedMetric` class with this one
- # which doesn't produce any metric to the collector.
- class NullMetric
-   attr_reader :namespace_name, :collector
-
-   def increment(key, value = 1)
-     Metric.validate_key!(key)
-   end
-
-   def decrement(key, value = 1)
-     Metric.validate_key!(key)
-   end
-
-   def gauge(key, value)
-     Metric.validate_key!(key)
-   end
-
-   def report_time(key, duration)
-     Metric.validate_key!(key)
-   end
-
-   # We have to manually redefine this method since it can return an
-   # object this object also has to be implemented as a NullObject
-   def time(key)
-     Metric.validate_key!(key)
-     if block_given?
-       yield
-     else
-       NullTimedExecution
-     end
-   end
-
-   def namespace(key)
-     self.class.new
-   end
-
-   private
-   # Null implementation of the internal timer class
-   #
-   # @see LogStash::Instrument::TimedExecution`
-   class NullTimedExecution
-     def self.stop
-     end
-   end
- end
+  # This class is used in the context when we disable the metric collection
+  # for specific plugin to replace the `NamespacedMetric` class with this one
+  # which doesn't produce any metric to the collector.
+  class NullMetric
+    attr_reader :namespace_name, :collector
+
+    def initialize(collector = nil)
+      @collector = collector
+    end
+
+    def increment(namespace, key, value = 1)
+      Metric.validate_key!(key)
+    end
+
+    def decrement(namespace, key, value = 1)
+      Metric.validate_key!(key)
+    end
+
+    def gauge(namespace, key, value)
+      Metric.validate_key!(key)
+    end
+
+    def report_time(namespace, key, duration)
+      Metric.validate_key!(key)
+    end
+
+    # We have to manually redefine this method since it can return an
+    # object this object also has to be implemented as a NullObject
+    def time(namespace, key)
+      Metric.validate_key!(key)
+      if block_given?
+        yield
+      else
+        NullTimedExecution
+      end
+    end
+
+    def namespace(name)
+      raise MetricNoNamespaceProvided if name.nil? || name.empty?
+      NamespacedNullMetric.new(self, name)
+    end
+
+    def self.validate_key!(key)
+      raise MetricNoKeyProvided if key.nil? || key.empty?
+    end
+
+    private
+    # Null implementation of the internal timer class
+    #
+    # @see LogStash::Instrument::TimedExecution`
+    class NullTimedExecution
+      def self.stop
+      end
+    end
+  end
 end; end

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -1,12 +1,14 @@
-
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
+require "logstash/instrument/periodic_poller/load_average"
+require "logstash/environment"
 require "jrmonitor"
 require "set"
 
 java_import 'java.lang.management.ManagementFactory'
 java_import 'java.lang.management.OperatingSystemMXBean'
 java_import 'java.lang.management.GarbageCollectorMXBean'
+java_import 'java.lang.management.RuntimeMXBean'
 java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'javax.management.MBeanServer'
 java_import 'javax.management.ObjectName'
@@ -32,21 +34,23 @@ module LogStash module Instrument module PeriodicPoller
       end
     end
 
+
     attr_reader :metric
 
     def initialize(metric, options = {})
       super(metric, options)
       @metric = metric
+      @load_average = LoadAverage.create
     end
 
     def collect
       raw = JRMonitor.memory.generate
-      collect_heap_metrics(raw)
-      collect_non_heap_metrics(raw)
+      collect_jvm_metrics(raw)      
       collect_pools_metrics(raw)
       collect_threads_metrics
       collect_process_metrics
       collect_gc_stats
+      collect_load_average
     end
 
     private
@@ -97,6 +101,25 @@ module LogStash module Instrument module PeriodicPoller
       metric.gauge(cpu_path, :total_in_millis, cpu_metrics["total_in_millis"])
 
       metric.gauge(path + [:mem], :total_virtual_in_bytes, process_metrics["mem"]["total_virtual_in_bytes"])
+
+    end
+
+    def collect_load_average
+      begin
+        load_average = @load_average.get
+      rescue => e
+        logger.debug("Can't retrieve load average", :exception => e.class.name, :message => e.message)
+        load_average = nil
+      end
+
+      metric.gauge([:jvm, :process, :cpu], :load_average, load_average) unless load_average.nil?
+    end
+    
+    def collect_jvm_metrics(data)
+      runtime_mx_bean = ManagementFactory.getRuntimeMXBean()
+      metric.gauge([:jvm], :uptime_in_millis, runtime_mx_bean.getUptime())
+      collect_heap_metrics(data)
+      collect_non_heap_metrics(data)
     end
 
     def collect_heap_metrics(data)

data/lib/logstash/instrument/periodic_poller/load_average.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+module LogStash module Instrument module PeriodicPoller
+  class LoadAverage
+    class Windows
+      def self.get
+        nil
+      end
+    end
+
+    class Linux
+      LOAD_AVG_FILE = "/proc/loadavg"
+      TOKEN_SEPARATOR = " "
+
+      def self.get
+        load_average = ::File.read(LOAD_AVG_FILE).chomp.split(TOKEN_SEPARATOR)
+
+        {
+          :"1m" => load_average[0].to_f,
+          :"5m" => load_average[1].to_f,
+          :"15m" => load_average[2].to_f
+        }
+      end
+    end
+
+    class Other
+      def self.get()
+        load_average_1m = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage()
+
+        return nil if load_average_1m.nil?
+
+        {
+          :"1m" => load_average_1m
+        }
+      end
+    end
+
+    def self.create
+      if LogStash::Environment.windows?
+        Windows
+      elsif LogStash::Environment.linux?
+        Linux
+      else
+        Other
+      end
+    end
+  end
+end end end

data/lib/logstash/instrument/snapshot.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-require "logstash/event"
 
 module LogStash module Instrument
   class Snapshot

data/lib/logstash/java_integration.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "java"
-require "jars"
 
 # this is mainly for usage with JrJackson json parsing in :raw mode which genenerates
 # Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.

data/lib/logstash/logging/logger.rb

@@ -3,11 +3,12 @@ require "uri"
 
 module LogStash
   module Logging
+    java_import org.apache.logging.log4j.Level
+    java_import org.apache.logging.log4j.LogManager
+    java_import org.apache.logging.log4j.core.config.Configurator
+    java_import org.apache.logging.log4j.core.config.DefaultConfiguration
+
     class Logger
-      java_import org.apache.logging.log4j.Level
-      java_import org.apache.logging.log4j.LogManager
-      java_import org.apache.logging.log4j.core.config.Configurator
-      java_import org.apache.logging.log4j.core.config.DefaultConfiguration
       @@config_mutex = Mutex.new
       @@logging_context = nil
 
@@ -90,5 +91,37 @@ module LogStash
         return @@logging_context
       end
     end
+
+    class SlowLogger
+      def initialize(name, warn_threshold, info_threshold, debug_threshold, trace_threshold)
+        slowlog_name = ["slowlog", name].join('.')
+        @slowlogger = LogManager.getLogger(slowlog_name)
+        @warn_threshold = warn_threshold
+        @info_threshold = info_threshold
+        @debug_threshold = debug_threshold
+        @trace_threshold = trace_threshold
+      end
+
+      def as_data(plugin_params, event, took_in_nanos)
+        {
+          :plugin_params => plugin_params,
+          :took_in_nanos => took_in_nanos,
+          :took_in_millis => took_in_nanos / 1000,
+          :event => event.to_json
+        }
+      end
+
+      def on_event(message, plugin_params, event, took_in_nanos)
+        if @warn_threshold >= 0 and took_in_nanos > @warn_threshold
+          @slowlogger.warn(message, as_data(plugin_params, event, took_in_nanos))
+        elsif @info_threshold >= 0 and took_in_nanos > @info_threshold
+          @slowlogger.info(message, as_data(plugin_params, event, took_in_nanos))
+        elsif @debug_threshold >= 0 and took_in_nanos > @debug_threshold
+          @slowlogger.debug(message, as_data(plugin_params, event, took_in_nanos))
+        elsif @trace_threshold >= 0 and took_in_nanos > @trace_threshold
+          @slowlogger.trace(message, as_data(plugin_params, event, took_in_nanos))
+        end
+      end
+    end
   end
 end

data/lib/logstash/outputs/base.rb

@@ -84,7 +84,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
   end # def receive
 
   public
-  # To be overriden in implementations
+  # To be overridden in implementations
   def multi_receive(events)
     if @receives_encoded
       self.multi_receive_encoded(codec.multi_encode(events))

data/lib/logstash/patches.rb

@@ -3,3 +3,4 @@ require "logstash/patches/bugfix_jruby_2558"
 require "logstash/patches/cabin"
 require "logstash/patches/profile_require_calls"
 require "logstash/patches/stronger_openssl_defaults"
+require "logstash/patches/exception_to_json"

data/lib/logstash/patches/exception_to_json.rb

@@ -0,0 +1,5 @@
+class Exception
+  def to_json
+    {"exception_name" => self.class.name, "message" => message}
+  end
+end

data/lib/logstash/pipeline.rb

@@ -4,17 +4,20 @@ require "stud/interval"
 require "concurrent"
 require "logstash/namespace"
 require "logstash/errors"
+require "logstash-core/logstash-core"
+require "logstash/util/wrapped_acked_queue"
+require "logstash/util/wrapped_synchronous_queue"
 require "logstash/event"
 require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
 require "logstash/shutdown_watcher"
-require "logstash/util/wrapped_synchronous_queue"
 require "logstash/pipeline_reporter"
 require "logstash/instrument/metric"
 require "logstash/instrument/namespaced_metric"
 require "logstash/instrument/null_metric"
+require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
@@ -91,16 +94,17 @@ module LogStash; class Pipeline
     rescue => e
       raise
     end
-
-    queue = Util::WrappedSynchronousQueue.new
-    @input_queue_client = queue.write_client
-    @filter_queue_client = queue.read_client
-    # Note that @inflight_batches as a central mechanism for tracking inflight
+    @queue = build_queue_from_settings
+    @input_queue_client = @queue.write_client
+    @filter_queue_client = @queue.read_client
+    @signal_queue = Queue.new
+    # Note that @infilght_batches as a central mechanism for tracking inflight
     # batches will fail if we have multiple read clients here.
     @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
     @filter_queue_client.set_pipeline_metric(
         metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
     )
+
     @events_filtered = Concurrent::AtomicFixnum.new(0)
     @events_consumed = Concurrent::AtomicFixnum.new(0)
 
@@ -111,6 +115,32 @@ module LogStash; class Pipeline
     @flushing = Concurrent::AtomicReference.new(false)
   end # def initialize
 
+  def build_queue_from_settings
+    queue_type = settings.get("queue.type")
+    queue_page_capacity = settings.get("queue.page_capacity")
+    queue_max_bytes = settings.get("queue.max_bytes")
+    queue_max_events = settings.get("queue.max_events")
+    checkpoint_max_acks = settings.get("queue.checkpoint.acks")
+    checkpoint_max_writes = settings.get("queue.checkpoint.writes")
+    checkpoint_max_interval = settings.get("queue.checkpoint.interval")
+
+    if queue_type == "memory_acked"
+      # memory_acked is used in tests/specs
+      LogStash::Util::WrappedAckedQueue.create_memory_based("", queue_page_capacity, queue_max_events, queue_max_bytes)
+    elsif queue_type == "memory"
+      # memory is the legacy and default setting
+      LogStash::Util::WrappedSynchronousQueue.new()
+    elsif queue_type == "persisted"
+      # persisted is the disk based acked queue
+      queue_path = settings.get("path.queue")
+      LogStash::Util::WrappedAckedQueue.create_file_based(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
+    else
+      raise(ConfigurationError, "invalid queue.type setting")
+    end
+  end
+
+  private :build_queue_from_settings
+
   def ready?
     @ready.value
   end
@@ -167,6 +197,9 @@ module LogStash; class Pipeline
     shutdown_flusher
     shutdown_workers
 
+    @filter_queue_client.close
+    @queue.close
+
     @logger.debug("Pipeline #{@pipeline_id} has been shutdown")
 
     # exit code
@@ -241,12 +274,15 @@ module LogStash; class Pipeline
 
     while running
       batch = @filter_queue_client.take_batch
+      signal = @signal_queue.empty? ? NO_SIGNAL : @signal_queue.pop
+      running = !signal.shutdown?
+
       @events_consumed.increment(batch.size)
-      running = false if batch.shutdown_signal_received?
+
       filter_batch(batch)
 
-      if batch.shutdown_signal_received? || batch.flush_signal_received?
-        flush_filters_to_batch(batch)
+      if signal.flush? || signal.shutdown?
+        flush_filters_to_batch(batch, :final => signal.shutdown?)
       end
 
       output_batch(batch)
@@ -256,11 +292,9 @@ module LogStash; class Pipeline
 
   def filter_batch(batch)
     batch.each do |event|
-      if event.is_a?(Event)
-        filter_func(event).each do |e|
-          # these are both original and generated events
-          batch.merge(e) unless e.cancelled?
-        end
+      filter_func(event).each do |e|
+        #these are both original and generated events
+        batch.merge(e) unless e.cancelled?
       end
     end
     @filter_queue_client.add_filtered_metrics(batch)
@@ -382,7 +416,7 @@ module LogStash; class Pipeline
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
       @logger.debug("Pushing shutdown", :thread => t.inspect)
-      @input_queue_client.push(SHUTDOWN)
+      @signal_queue.push(SHUTDOWN)
     end
 
     @worker_threads.each do |t|
@@ -467,7 +501,7 @@ module LogStash; class Pipeline
   def flush
     if @flushing.compare_and_set(false, true)
       @logger.debug? && @logger.debug("Pushing flush onto pipeline")
-      @input_queue_client.push(FLUSH)
+      @signal_queue.push(FLUSH)
     end
   end
 
@@ -485,7 +519,6 @@ module LogStash; class Pipeline
   # @param batch [ReadClient::ReadBatch]
   # @param options [Hash]
   def flush_filters_to_batch(batch, options = {})
-    options[:final] = batch.shutdown_signal_received?
     flush_filters(options) do |event|
       unless event.cancelled?
         @logger.debug? and @logger.debug("Pushing flushed events", :event => event)