logstash-core 5.1.2-java → 5.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8de8152e9d47ea5ce222f2811e96ec13216542ec
-  data.tar.gz: da613e3b8bee64c8be84dbf804f45511fdb0fc07
+  metadata.gz: 50660404c5ec13b21a5dccc7fd62b0d282dea262
+  data.tar.gz: 5b5b44f759032fda3022a8d08f982cf35ef5179a
 SHA512:
-  metadata.gz: 5f5989dfd6273c39a46964798d8c4549f1ad9c196894afd0a5df53cb3b0d05ca35947c9d4032d4f3e41a5f12a50657465a249ec030ce1b2d6adc98bce5bf15b1
-  data.tar.gz: 8c5b2a3b25634f0f7c4d7543a1267a4ce536faa2eee61048c00519ad7d89cf68f165ea8fbe52c68dc5d6f0da121b2b8be0cbd79c90f0c47fee2a699442b83fc9
+  metadata.gz: 134ef54d0f7ef058c201420d997e702775bf4520e747a6c40ec76a6593597192a6448a537bad0e1df236214d636b469c6d95faa4894492cb05c23c826b0ce4e9
+  data.tar.gz: fa146b9c7ccd8891ed64dc892026313ff6356d87511613e23a270b8a30fa33a664a87d39b586b51b3e94961a3a18dd3640087746bdf965a5a603ff3bfd514b21

data/gemspec_jars.rb

@@ -2,7 +2,6 @@
 # runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec
 # for the jar-dependencies requirements.
 
-gem.requirements << "jar org.apache.logging.log4j:log4j-1.2-api, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
 gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.4"

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.1.2"
+LOGSTASH_CORE_VERSION = "5.2.0"

data/lib/logstash-core_jars.rb

@@ -5,7 +5,6 @@ rescue LoadError
   require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
   require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
   require 'com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar'
-  require 'org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar'
   require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar'
   require 'com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar'
 end
@@ -14,7 +13,6 @@ if defined? Jars
   require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
   require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.4' )
-  require_jar( 'org.apache.logging.log4j', 'log4j-1.2-api', '2.6.2' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.4' )
 end

data/lib/logstash/agent.rb

@@ -52,7 +52,8 @@ class LogStash::Agent
     # Create the collectors and configured it with the library
     configure_metrics_collectors
 
-    @reload_metric = metric.namespace([:stats, :pipelines])
+    @pipeline_reload_metric = metric.namespace([:stats, :pipelines])
+    @instance_reload_metric = metric.namespace([:stats, :reloads])
 
     @dispatcher = LogStash::EventDispatcher.new(self)
     LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, dispatcher)
@@ -109,7 +110,8 @@ class LogStash::Agent
         begin
           reload_pipeline!(pipeline_id)
         rescue => e
-          @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
+          @instance_reload_metric.increment(:failures)
+          @pipeline_reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
             n.increment(:failures)
             n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
             n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
@@ -128,7 +130,6 @@ class LogStash::Agent
   end
 
   def stop_collecting_metrics
-    @collector.stop
     @periodic_pollers.stop
   end
 
@@ -176,6 +177,12 @@ class LogStash::Agent
     @id_path ||= ::File.join(settings.get("path.data"), "uuid")
   end
 
+  def running_pipelines
+    @upgrade_mutex.synchronize do
+      @pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }
+    end
+  end
+
   def running_pipelines?
     @upgrade_mutex.synchronize do
       @pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }.any?
@@ -207,7 +214,9 @@ class LogStash::Agent
               end
 
 
-    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(@metric)
+    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(@metric,
+                                                                  settings.get("queue.type"),
+                                                                  self)
     @periodic_pollers.start
   end
 
@@ -236,7 +245,8 @@ class LogStash::Agent
     begin
       LogStash::Pipeline.new(config, settings, metric)
     rescue => e
-      @reload_metric.namespace([settings.get("pipeline.id").to_sym, :reloads]).tap do |n|
+      @instance_reload_metric.increment(:failures)
+      @pipeline_reload_metric.namespace([settings.get("pipeline.id").to_sym, :reloads]).tap do |n|
         n.increment(:failures)
         n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
         n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
@@ -291,7 +301,8 @@ class LogStash::Agent
       begin
         pipeline.run
       rescue => e
-        @reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
+        @instance_reload_metric.increment(:failures)
+        @pipeline_reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
           n.increment(:failures)
           n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
           n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
@@ -302,7 +313,7 @@ class LogStash::Agent
     while true do
       if !t.alive?
         return false
-      elsif pipeline.ready?
+      elsif pipeline.running?
         return true
       else
         sleep 0.01
@@ -319,8 +330,11 @@ class LogStash::Agent
   end
 
   def start_pipelines
-    @pipelines.each do |id, _|
+    @instance_reload_metric.increment(:successes, 0)
+    @instance_reload_metric.increment(:failures, 0)
+    @pipelines.each do |id, pipeline|
       start_pipeline(id)
+      pipeline.collect_stats
       # no reloads yet, initalize all the reload metrics
       init_pipeline_reload_metrics(id)
     end
@@ -340,10 +354,12 @@ class LogStash::Agent
     reset_pipeline_metrics(pipeline_id)
     @pipelines[pipeline_id] = new_pipeline
     if start_pipeline(pipeline_id) # pipeline started successfuly
-      @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
+      @instance_reload_metric.increment(:successes)
+      @pipeline_reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
         n.increment(:successes)
         n.gauge(:last_success_timestamp, LogStash::Timestamp.now)
       end
+      
     end
   end
 
@@ -356,7 +372,7 @@ class LogStash::Agent
   end
 
   def init_pipeline_reload_metrics(id)
-    @reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
+    @pipeline_reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
       n.increment(:successes, 0)
       n.increment(:failures, 0)
       n.gauge(:last_error, nil)

data/lib/logstash/api/commands/default_metadata.rb

@@ -20,7 +20,9 @@ module LogStash
         end
 
         def http_address
-          service.agent.webserver.address
+          @http_address ||= service.get_shallow(:http_address).value
+        rescue ::LogStash::Instrument::MetricStore::MetricNotFound, NoMethodError => e
+          nil
         end
       end
     end

data/lib/logstash/api/commands/stats.rb

@@ -3,6 +3,9 @@ require "logstash/api/commands/base"
 require 'logstash/util/thread_dump'
 require_relative "hot_threads_reporter"
 
+java_import java.nio.file.Files
+java_import java.nio.file.Paths
+
 module LogStash
   module Api
     module Commands
@@ -16,10 +19,14 @@ module LogStash
             ),
             :mem => memory,
             :gc => gc,
-            :uptime_in_millis => service.get_shallow(:jvm, :uptime_in_millis)
+            :uptime_in_millis => service.get_shallow(:jvm, :uptime_in_millis),
           }
         end
 
+        def reloads
+          service.get_shallow(:stats, :reloads)
+        end
+
         def process
           extract_metrics(
             [:jvm, :process],
@@ -61,6 +68,14 @@ module LogStash
           }
         end
 
+        def os
+          service.get_shallow(:os)
+        rescue
+          # The only currently fetch OS information is about the linux
+          # containers.
+          {}
+        end
+
         def gc
           service.get_shallow(:jvm, :gc)
         end
@@ -94,6 +109,7 @@ module LogStash
                 :outputs => plugin_stats(stats, :outputs)
               },
               :reloads => stats[:reloads],
+              :queue => stats[:queue]
             }
           end
         end # module PluginsStats

data/lib/logstash/api/modules/node_stats.rb

@@ -13,11 +13,16 @@ module LogStash
             :jvm => jvm_payload,
             :process => process_payload,
             :pipeline => pipeline_payload,
+            :reloads => reloads,
+            :os => os_payload
           }
           respond_with(payload, {:filter => params["filter"]})
         end
 
         private
+        def os_payload
+          @stats.os
+        end
 
         def events_payload
           @stats.events
@@ -27,6 +32,10 @@ module LogStash
           @stats.jvm
         end
 
+        def reloads
+          @stats.reloads
+        end
+
         def process_payload
           @stats.process
         end

data/lib/logstash/api/modules/stats.rb

@@ -30,8 +30,9 @@ module LogStash
             :jvm => {
               :timestamp => stats_command.started_at,
               :uptime_in_millis => stats_command.uptime,
-              :memory => stats_command.memory
-            }
+              :memory => stats_command.memory,
+            },
+            :os => stats_command.os
           }
           respond_with(payload, {:filter => params["filter"]})
         end

data/lib/logstash/config/mixin.rb

@@ -268,6 +268,7 @@ module LogStash::Config::Mixin
       return is_valid
     end # def validate
 
+    # TODO: Remove in 6.0
     def print_version_notice
       return if @@version_notice_given
 
@@ -288,14 +289,10 @@ module LogStash::Config::Mixin
           end
         end
       rescue LogStash::PluginNoVersionError
-        # If we cannot find a version in the currently installed gems we
-        # will display this message. This could happen in the test, if you 
-        # create an anonymous class to test a plugin.
-        self.logger.warn(I18n.t("logstash.plugin.no_version",
-                                :type => @plugin_type,
-                                :name => @config_name,
-                                :LOGSTASH_VERSION => LOGSTASH_VERSION))
-      ensure 
+        # This can happen because of one of the following:
+        # - The plugin is loaded from the plugins.path and contains no gemspec.
+        # - The plugin is defined in a universal plugin, so the loaded plugin doesn't correspond to an actual gemspec.
+      ensure
         @@version_notice_given = true
       end
     end

data/lib/logstash/instrument/collector.rb

@@ -11,12 +11,8 @@ module LogStash module Instrument
   # The Collector is the single point of reference for all
   # the metrics collection inside logstash, the metrics library will make
   # direct calls to this class.
-  #
-  # This class is an observable responsable of periodically emitting view of the system
-  # to other components like the internal metrics pipelines.
   class Collector
     include LogStash::Util::Loggable
-    include Observable
 
     SNAPSHOT_ROTATION_TIME_SECS = 1 # seconds
     SNAPSHOT_ROTATION_TIMEOUT_INTERVAL_SECS = 10 * 60 # seconds
@@ -26,7 +22,6 @@ module LogStash module Instrument
     def initialize
       @metric_store = MetricStore.new
       @agent = nil
-      start_periodic_snapshotting
     end
 
     # The metric library will call this unique interface
@@ -43,8 +38,6 @@ module LogStash module Instrument
         end
 
         metric.execute(*metric_type_params)
-
-        changed # we had changes coming in so we can notify the observers
       rescue MetricStore::NamespacesExpectedError => e
         logger.error("Collector: Cannot record metric", :exception => e)
       rescue NameError => e
@@ -58,51 +51,13 @@ module LogStash module Instrument
       end
     end
 
-    # Monitor the `Concurrent::TimerTask` this update is triggered on every successful or not
-    # run of the task, TimerTask implement Observable and the collector acts as
-    # the observer and will keep track if something went wrong in the execution.
-    #
-    # @param [Time] Time of execution
-    # @param [result] Result of the execution
-    # @param [Exception] Exception
-    def update(time_of_execution, result, exception)
-      return true if exception.nil?
-      logger.error("Collector: Something went wrong went sending data to the observers",
-                   :execution_time => time_of_execution,
-                   :result => result,
-                   :exception => exception.class.name)
-    end
-
     # Snapshot the current Metric Store and return it immediately,
     # This is useful if you want to get access to the current metric store without
     # waiting for a periodic call.
     #
     # @return [LogStash::Instrument::MetricStore]
     def snapshot_metric
-      Snapshot.new(@metric_store)
-    end
-
-    # Configure and start the periodic task for snapshotting the `MetricStore`
-    def start_periodic_snapshotting
-      @snapshot_task = Concurrent::TimerTask.new { publish_snapshot }
-      @snapshot_task.execution_interval = SNAPSHOT_ROTATION_TIME_SECS
-      @snapshot_task.timeout_interval = SNAPSHOT_ROTATION_TIMEOUT_INTERVAL_SECS
-      @snapshot_task.add_observer(self)
-      @snapshot_task.execute
-    end
-
-    def stop
-      @snapshot_task.shutdown
-    end
-
-    # Create a snapshot of the MetricStore and send it to to the registered observers
-    # The observer will receive the following signature in the update methode.
-    #
-    # `#update(created_at, metric_store)`
-    def publish_snapshot
-      created_at = Time.now
-      logger.debug("Collector: Sending snapshot to observers", :created_at => created_at) if logger.debug?
-      notify_observers(snapshot_metric)
+      Snapshot.new(@metric_store.dup)
     end
 
     def clear(keypath)

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -12,6 +12,8 @@ module LogStash module Instrument module PeriodicPoller
       :polling_timeout => 120
     }
 
+    attr_reader :metric
+
     public
     def initialize(metric, options = {})
       @metric = metric

data/lib/logstash/instrument/periodic_poller/cgroup.rb

@@ -0,0 +1,137 @@
+# encoding: utf-8
+require "pathname"
+require "logstash/util/loggable"
+
+# Logic from elasticsearch/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+# Move to ruby to remove any existing dependency
+module LogStash module Instrument module PeriodicPoller
+  class Cgroup
+    include LogStash::Util::Loggable
+
+    CONTROL_GROUP_RE = Regexp.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)");
+    CONTROLLER_SEPERATOR_RE = ","
+
+    PROC_SELF_CGROUP_FILE = Pathname.new("/proc/self/cgroup")
+    PROC_CGROUP_CPU_DIR = Pathname.new("/sys/fs/cgroup/cpu")
+    PROC_CGROUP_CPUACCT_DIR = Pathname.new("/sys/fs/cgroup/cpuacct")
+
+    GROUP_CPUACCT = "cpuacct"
+    CPUACCT_USAGE_FILE = "cpuacct.usage"
+
+    GROUP_CPU = "cpu"
+    CPU_FS_PERIOD_US_FILE = "cpu.cfs_period_us"
+    CPU_FS_QUOTA_US_FILE = "cpu.cfs_quota_us"
+
+    CPU_STATS_FILE = "cpu.stat"
+
+    class << self
+      def are_cgroup_available?
+        [::File.exist?(PROC_SELF_CGROUP_FILE),
+         Dir.exist?(PROC_CGROUP_CPU_DIR),
+         Dir.exist?(PROC_CGROUP_CPUACCT_DIR)].all?
+      end
+
+      def control_groups
+        response = {}
+
+        read_proc_self_cgroup_lines.each do |line|
+          matches = CONTROL_GROUP_RE.match(line)
+          # multiples controlles, same hierachy
+          controllers = matches[1].split(CONTROLLER_SEPERATOR_RE)
+          controllers.each_with_object(response) { |controller| response[controller] = matches[2] }
+        end
+
+        response
+      end
+
+      def read_first_line(path)
+        IO.readlines(path).first
+      end
+
+      def cgroup_cpuacct_usage_nanos(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group, CPUACCT_USAGE_FILE)).to_i
+      end
+
+      def cgroup_cpu_fs_period_micros(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group, CPU_FS_PERIOD_US_FILE)).to_i
+      end
+
+      def cgroup_cpu_fs_quota_micros(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group,  CPU_FS_QUOTA_US_FILE)).to_i
+      end
+
+      def read_proc_self_cgroup_lines
+        IO.readlines(PROC_SELF_CGROUP_FILE)
+      end
+
+      class CpuStats
+        attr_reader :number_of_elapsed_periods, :number_of_times_throttled, :time_throttled_nanos
+
+        def initialize(number_of_elapsed_periods, number_of_times_throttled, time_throttled_nanos)
+          @number_of_elapsed_periods = number_of_elapsed_periods
+          @number_of_times_throttled = number_of_times_throttled
+          @time_throttled_nanos = time_throttled_nanos
+        end
+      end
+
+      def read_sys_fs_cgroup_cpuacct_cpu_stat(control_group)
+        IO.readlines(::File.join(PROC_CGROUP_CPU_DIR, control_group, CPU_STATS_FILE))
+      end
+
+      def cgroup_cpuacct_cpu_stat(control_group)
+        lines = read_sys_fs_cgroup_cpuacct_cpu_stat(control_group);
+
+        number_of_elapsed_periods = -1;
+        number_of_times_throttled = -1;
+        time_throttled_nanos = -1;
+
+        lines.each do |line|
+          fields = line.split(/\s+/)
+          case fields.first
+          when "nr_periods" then number_of_elapsed_periods = fields[1].to_i
+          when "nr_throttled" then number_of_times_throttled= fields[1].to_i
+          when "throttled_time" then time_throttled_nanos = fields[1].to_i
+          end
+        end
+
+        CpuStats.new(number_of_elapsed_periods, number_of_times_throttled, time_throttled_nanos)
+      end
+
+      def get_all
+       groups = control_groups
+       return if groups.empty?
+
+       cgroups_stats = {
+         :cpuacct => {},
+         :cpu => {}
+       }
+
+       cpuacct_group = groups[GROUP_CPUACCT]
+       cgroups_stats[:cpuacct][:control_group] = cpuacct_group
+       cgroups_stats[:cpuacct][:usage_nanos] = cgroup_cpuacct_usage_nanos(cpuacct_group)
+
+       cpu_group = groups[GROUP_CPU]
+       cgroups_stats[:cpu][:control_group] = cpu_group
+       cgroups_stats[:cpu][:cfs_period_micros] = cgroup_cpu_fs_period_micros(cpu_group)
+       cgroups_stats[:cpu][:cfs_quota_micros] = cgroup_cpu_fs_quota_micros(cpu_group)
+
+       cpu_stats = cgroup_cpuacct_cpu_stat(cpu_group)
+
+       cgroups_stats[:cpu][:stat] = {
+         :number_of_elapsed_periods => cpu_stats.number_of_elapsed_periods,
+         :number_of_times_throttled => cpu_stats.number_of_times_throttled,
+         :time_throttled_nanos => cpu_stats.time_throttled_nanos
+       }
+
+       cgroups_stats
+      rescue => e
+        logger.debug("Error, cannot retrieve cgroups information", :exception => e.class.name, :message => e.message) if logger.debug?
+        nil
+      end
+
+      def get
+        are_cgroup_available? ? get_all : nil
+      end
+    end
+  end
+end end end