logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/instrument/metric_type/mean.rb

@@ -4,12 +4,7 @@ require "concurrent"
 
 module LogStash module Instrument module MetricType
   class Mean < Base
-    include ::LogStash::Util::Loggable
-
     def initialize(namespaces, key)
-      logger.warn("Deprecated 'mean' metric type used! This will be removed in Logstash 6.0!",
-                   :namespaces => namespaces,
-                   :key => key)
       super(namespaces, key)
 
       @counter = Concurrent::AtomicFixnum.new

data/lib/logstash/instrument/namespaced_metric.rb

@@ -43,10 +43,6 @@ module LogStash module Instrument
     def collector
       @metric.collector
     end
-    
-    def counter(key)
-      collector.get(@namespace_name, key, :counter)
-    end
 
     def namespace(name)
       NamespacedMetric.new(metric, namespace_name + Array(name))

data/lib/logstash/instrument/namespaced_null_metric.rb

@@ -44,10 +44,6 @@ module LogStash module Instrument
       @metric.collector
     end
 
-    def counter(_)
-      ::LogStash::Instrument::NullMetric::NullGauge
-    end
-
     def namespace(name)
       NamespacedNullMetric.new(metric, namespace_name + Array(name))
     end

data/lib/logstash/instrument/null_metric.rb

@@ -39,10 +39,6 @@ module LogStash module Instrument
       end
     end
 
-    def counter(_)
-      NullGauge
-    end
-
     def namespace(name)
       raise MetricNoNamespaceProvided if name.nil? || name.empty?
       NamespacedNullMetric.new(self, name)
@@ -53,12 +49,6 @@ module LogStash module Instrument
     end
 
     private
-
-    class NullGauge
-      def self.increment(_)
-      end
-    end
-
     # Null implementation of the internal timer class
     #
     # @see LogStash::Instrument::TimedExecution`

data/lib/logstash/instrument/periodic_poller/cgroup.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "pathname"
 require "logstash/util/loggable"
 
 # Logic from elasticsearch/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -6,214 +7,130 @@ require "logstash/util/loggable"
 module LogStash module Instrument module PeriodicPoller
   class Cgroup
     include LogStash::Util::Loggable
-    class Override
-      attr_reader :key, :value
-      def initialize(key)
-        @key = key
-        @value = java.lang.System.getProperty(@key)
-      end
 
-      def nil?
-        value.nil?
-      end
+    CONTROL_GROUP_RE = Regexp.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)");
+    CONTROLLER_SEPARATOR_RE = ","
 
-      def override(other)
-        nil? ? other : value
-      end
-    end
+    PROC_SELF_CGROUP_FILE = Pathname.new("/proc/self/cgroup")
+    PROC_CGROUP_CPU_DIR = Pathname.new("/sys/fs/cgroup/cpu")
+    PROC_CGROUP_CPUACCT_DIR = Pathname.new("/sys/fs/cgroup/cpuacct")
+
+    GROUP_CPUACCT = "cpuacct"
+    CPUACCT_USAGE_FILE = "cpuacct.usage"
+
+    GROUP_CPU = "cpu"
+    CPU_FS_PERIOD_US_FILE = "cpu.cfs_period_us"
+    CPU_FS_QUOTA_US_FILE = "cpu.cfs_quota_us"
 
-    ## `/proc/self/cgroup` contents look like this
-    # 5:cpu,cpuacct:/
-    # 4:cpuset:/
-    # 2:net_cls,net_prio:/
-    # 0::/user.slice/user-1000.slice/session-932.scope
-    ## e.g. N:controller:/path-to-info
-    # we find the controller and path
-    # we skip the line without a controller e.g. 0::/path
-    # we assume there are these symlinks:
-    # `/sys/fs/cgroup/cpu` -> `/sys/fs/cgroup/cpu,cpuacct
-    # `/sys/fs/cgroup/cpuacct` -> `/sys/fs/cgroup/cpu,cpuacct
-
-    CGROUP_FILE = "/proc/self/cgroup"
-    CPUACCT_DIR = "/sys/fs/cgroup/cpuacct"
-    CPU_DIR = "/sys/fs/cgroup/cpu"
-    CRITICAL_PATHS = [CGROUP_FILE, CPUACCT_DIR, CPU_DIR]
-
-    CONTROLLER_CPUACCT_LABEL = "cpuacct"
-    CONTROLLER_CPU_LABEL = "cpu"
-
-    class CGroupResources
-      CONTROL_GROUP_RE = Regexp.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)")
-      CONTROLLER_SEPARATOR_RE = ","
-
-      def cgroup_available?
-        # don't cache to ivar, in case the files are mounted after logstash starts??
-        CRITICAL_PATHS.all?{|path| ::File.exist?(path)}
+    CPU_STATS_FILE = "cpu.stat"
+
+    class << self
+      def are_cgroup_available?
+        [::File.exist?(PROC_SELF_CGROUP_FILE),
+         Dir.exist?(PROC_CGROUP_CPU_DIR),
+         Dir.exist?(PROC_CGROUP_CPUACCT_DIR)].all?
       end
 
-      def controller_groups
+      def control_groups
         response = {}
-        IO.readlines(CGROUP_FILE).each do |line|
+
+        read_proc_self_cgroup_lines.each do |line|
           matches = CONTROL_GROUP_RE.match(line)
-          next if matches.nil?
           # multiples controls, same hierarchy
           controllers = matches[1].split(CONTROLLER_SEPARATOR_RE)
-          controllers.each do |controller|
-            case controller
-            when CONTROLLER_CPU_LABEL
-              response[controller] = CpuResource.new(matches[2])
-            when CONTROLLER_CPUACCT_LABEL
-              response[controller] = CpuAcctResource.new(matches[2])
-            else
-              response[controller] = UnimplementedResource.new(controller, matches[2])
-            end
-          end
+          controllers.each_with_object(response) { |controller| response[controller] = matches[2] }
         end
+
         response
       end
-    end
 
-    module ControllerResource
-      attr_reader :base_path, :override, :offset_path
-      def implemented?
-        true
-      end
-      private
-      def common_initialize(base, override_key, original_path)
-        @base_path = base
-        # override is needed here for the logging statements
-        @override = Override.new(override_key)
-        @offset_path = @override.override(original_path)
-        @procs = {}
-        @procs[:read_int] = lambda {|path| IO.readlines(path).first.to_i }
-        @procs[:read_lines] = lambda {|path| IO.readlines(path) }
-      end
-      def call_if_file_exists(call_key, file, not_found_value)
-        path = ::File.join(@base_path, @offset_path, file)
-        if ::File.exist?(path)
-          @procs[call_key].call(path)
-        else
-          message = "File #{path} cannot be found, "
-          if override.nil?
-            message.concat("try providing an override '#{override.key}' in the Logstash JAVA_OPTS environment variable")
-          else
-            message.concat("even though the '#{override.key}' override is: '#{override.value}'")
-          end
-          logger.debug(message)
-          not_found_value
-        end
+      def read_first_line(path)
+        IO.readlines(path).first
       end
-    end
 
-    class CpuAcctResource
-      include LogStash::Util::Loggable
-      include ControllerResource
-      def initialize(original_path)
-        common_initialize(CPUACCT_DIR, "ls.cgroup.cpuacct.path.override", original_path)
+      def cgroup_cpuacct_usage_nanos(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group, CPUACCT_USAGE_FILE)).to_i
       end
-      def to_hash
-        {:control_group => offset_path, :usage_nanos => cpuacct_usage}
-      end
-      private
-      def cpuacct_usage
-        call_if_file_exists(:read_int, "cpuacct.usage", -1)
-      end
-    end
 
-    class CpuResource
-      include LogStash::Util::Loggable
-      include ControllerResource
-      def initialize(original_path)
-        common_initialize(CPU_DIR, "ls.cgroup.cpu.path.override", original_path)
-      end
-      def to_hash
-        {
-          :control_group => offset_path,
-          :cfs_period_micros => cfs_period_us,
-          :cfs_quota_micros => cfs_quota_us,
-          :stat => build_cpu_stats_hash
-        }
-      end
-      private
-      def cfs_period_us
-        call_if_file_exists(:read_int, "cpu.cfs_period_us", -1)
+      def cgroup_cpu_fs_period_micros(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group, CPU_FS_PERIOD_US_FILE)).to_i
       end
-      def cfs_quota_us
-        call_if_file_exists(:read_int, "cpu.cfs_quota_us", -1)
-      end
-      def build_cpu_stats_hash
-        stats = CpuStats.new
-        lines = call_if_file_exists(:read_lines, "cpu.stat", [])
-        stats.update(lines)
-        stats.to_hash
+
+      def cgroup_cpu_fs_quota_micros(control_group)
+        read_first_line(::File.join(PROC_CGROUP_CPUACCT_DIR, control_group,  CPU_FS_QUOTA_US_FILE)).to_i
       end
-    end
 
-    class UnimplementedResource
-      attr_reader :controller, :original_path
-      def initialize(controller, original_path)
-        @controller, @original_path = controller, original_path
+      def read_proc_self_cgroup_lines
+        IO.readlines(PROC_SELF_CGROUP_FILE)
       end
-      def implemented?
-        false
+
+      class CpuStats
+        attr_reader :number_of_elapsed_periods, :number_of_times_throttled, :time_throttled_nanos
+
+        def initialize(number_of_elapsed_periods, number_of_times_throttled, time_throttled_nanos)
+          @number_of_elapsed_periods = number_of_elapsed_periods
+          @number_of_times_throttled = number_of_times_throttled
+          @time_throttled_nanos = time_throttled_nanos
+        end
       end
-    end
 
-    class CpuStats
-      def initialize
-        @number_of_elapsed_periods = -1
-        @number_of_times_throttled = -1
-        @time_throttled_nanos = -1
+      def read_sys_fs_cgroup_cpuacct_cpu_stat(control_group)
+        IO.readlines(::File.join(PROC_CGROUP_CPU_DIR, control_group, CPU_STATS_FILE))
       end
-      def update(lines)
+
+      def cgroup_cpuacct_cpu_stat(control_group)
+        lines = read_sys_fs_cgroup_cpuacct_cpu_stat(control_group);
+
+        number_of_elapsed_periods = -1;
+        number_of_times_throttled = -1;
+        time_throttled_nanos = -1;
+
         lines.each do |line|
           fields = line.split(/\s+/)
-          next unless fields.size > 1
           case fields.first
-          when "nr_periods" then @number_of_elapsed_periods = fields[1].to_i
-          when "nr_throttled" then @number_of_times_throttled = fields[1].to_i
-          when "throttled_time" then @time_throttled_nanos = fields[1].to_i
+          when "nr_periods" then number_of_elapsed_periods = fields[1].to_i
+          when "nr_throttled" then number_of_times_throttled= fields[1].to_i
+          when "throttled_time" then time_throttled_nanos = fields[1].to_i
           end
         end
-      end
-      def to_hash
-        {
-          :number_of_elapsed_periods => @number_of_elapsed_periods,
-          :number_of_times_throttled => @number_of_times_throttled,
-          :time_throttled_nanos => @time_throttled_nanos
-        }
-      end
-    end
 
-    CGROUP_RESOURCES = CGroupResources.new
+        CpuStats.new(number_of_elapsed_periods, number_of_times_throttled, time_throttled_nanos)
+      end
 
-    class << self
       def get_all
-        unless CGROUP_RESOURCES.cgroup_available?
-          logger.debug("One or more required cgroup files or directories not found: #{CRITICAL_PATHS.join(', ')}")
-          return
-        end
+       groups = control_groups
+       return if groups.empty?
 
-        groups = CGROUP_RESOURCES.controller_groups
+       cgroups_stats = {
+         :cpuacct => {},
+         :cpu => {}
+       }
 
-        if groups.empty?
-          logger.debug("The main cgroup file did not have any controllers: #{CGROUP_FILE}")
-          return
-        end
+       cpuacct_group = groups[GROUP_CPUACCT]
+       cgroups_stats[:cpuacct][:control_group] = cpuacct_group
+       cgroups_stats[:cpuacct][:usage_nanos] = cgroup_cpuacct_usage_nanos(cpuacct_group)
 
-        cgroups_stats = {}
-        groups.each do |name, controller|
-          next unless controller.implemented?
-          cgroups_stats[name.to_sym] = controller.to_hash
-        end
-        cgroups_stats
+       cpu_group = groups[GROUP_CPU]
+       cgroups_stats[:cpu][:control_group] = cpu_group
+       cgroups_stats[:cpu][:cfs_period_micros] = cgroup_cpu_fs_period_micros(cpu_group)
+       cgroups_stats[:cpu][:cfs_quota_micros] = cgroup_cpu_fs_quota_micros(cpu_group)
+
+       cpu_stats = cgroup_cpuacct_cpu_stat(cpu_group)
+
+       cgroups_stats[:cpu][:stat] = {
+         :number_of_elapsed_periods => cpu_stats.number_of_elapsed_periods,
+         :number_of_times_throttled => cpu_stats.number_of_times_throttled,
+         :time_throttled_nanos => cpu_stats.time_throttled_nanos
+       }
+
+       cgroups_stats
       rescue => e
-        logger.debug("Error, cannot retrieve cgroups information", :exception => e.class.name, :message => e.message, :backtrace => e.backtrace.take(4)) if logger.debug?
+        logger.debug("Error, cannot retrieve cgroups information", :exception => e.class.name, :message => e.message) if logger.debug?
         nil
       end
 
       def get
-        get_all
+        are_cgroup_available? ? get_all : nil
       end
     end
   end

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -2,19 +2,18 @@
 require "logstash/instrument/periodic_poller/base"
 require "logstash/instrument/periodic_poller/load_average"
 require "logstash/environment"
+require "jrmonitor"
 require "set"
 
-java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'java.lang.management.ManagementFactory'
 java_import 'java.lang.management.OperatingSystemMXBean'
 java_import 'java.lang.management.GarbageCollectorMXBean'
 java_import 'java.lang.management.RuntimeMXBean'
+java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'javax.management.MBeanServer'
 java_import 'javax.management.ObjectName'
 java_import 'javax.management.AttributeList'
 java_import 'javax.naming.directory.Attribute'
-java_import 'org.logstash.instrument.reports.MemoryReport'
-java_import 'org.logstash.instrument.reports.ProcessReport'
 
 
 module LogStash module Instrument module PeriodicPoller
@@ -51,7 +50,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = MemoryReport.generate
+      raw = JRMonitor.memory.generate
       collect_jvm_metrics(raw)
       collect_pools_metrics(raw)
       collect_threads_metrics
@@ -82,10 +81,11 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect_process_metrics
-      process_metrics = ProcessReport.generate
+      process_metrics = JRMonitor.process.generate
 
       path = [:jvm, :process]
 
+
       open_fds = process_metrics["open_file_descriptors"]
       if @peak_open_fds.nil? || open_fds > @peak_open_fds
         @peak_open_fds = open_fds

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,13 +11,9 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.get_running_user_defined_pipelines
-      unless pipelines.nil?
-        pipelines.each {|_, pipeline|
-          unless pipeline.nil?
-            pipeline.collect_stats
-          end
-        }
+      pipeline_id, pipeline = @agent.running_pipelines.first
+      unless pipeline.nil?
+        pipeline.collect_stats
       end
     end
   end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/instrument/periodic_poller/dlq"
 require "logstash/instrument/periodic_poller/os"
 require "logstash/instrument/periodic_poller/jvm"
 require "logstash/instrument/periodic_poller/pq"
@@ -15,8 +14,7 @@ module LogStash module Instrument
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
-                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines)]
     end
 
     def start

data/lib/logstash/instrument/wrapped_write_client.rb

@@ -10,12 +10,7 @@ module LogStash module Instrument
       @events_metrics = metric.namespace([:stats, :events])
       @pipeline_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :events])
       @plugin_events_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :plugins, plugin_type, plugin.id.to_sym, :events])
-      @events_metrics_counter = @events_metrics.counter(:in)
-      @events_metrics_time = @events_metrics.counter(:queue_push_duration_in_millis)
-      @pipeline_metrics_counter = @pipeline_metrics.counter(:in)
-      @pipeline_metrics_time = @pipeline_metrics.counter(:queue_push_duration_in_millis)
-      @plugin_events_metrics_counter = @plugin_events_metrics.counter(:out)
-      @plugin_events_metrics_time = @plugin_events_metrics.counter(:queue_push_duration_in_millis)
+
       define_initial_metrics_values
     end
 
@@ -24,45 +19,41 @@ module LogStash module Instrument
     end
 
     def push(event)
-      increment_counters(1)
-      start_time = java.lang.System.current_time_millis
-      result = @write_client.push(event)
-      report_execution_time(start_time)
-      result
+      record_metric { @write_client.push(event) }
     end
-
     alias_method(:<<, :push)
 
     def push_batch(batch)
-      increment_counters(batch.size)
-      start_time = java.lang.System.current_time_millis
-      result = @write_client.push_batch(batch)
-      report_execution_time(start_time)
-      result
+      record_metric(batch.size) { @write_client.push_batch(batch) }
     end
 
     private
+    def record_metric(size = 1)
+      @events_metrics.increment(:in, size)
+      @pipeline_metrics.increment(:in, size)
+      @plugin_events_metrics.increment(:out, size)
 
-    def increment_counters(size)
-      @events_metrics_counter.increment(size)
-      @pipeline_metrics_counter.increment(size)
-      @plugin_events_metrics_counter.increment(size)
-    end
+      clock = @events_metrics.time(:queue_push_duration_in_millis)
 
-    def report_execution_time(start_time)
-      execution_time = java.lang.System.current_time_millis - start_time
-      @events_metrics_time.increment(execution_time)
-      @pipeline_metrics_time.increment(execution_time)
-      @plugin_events_metrics_time.increment(execution_time)
+      result = yield
+
+      # Reuse the same values for all the endpoints to make sure we don't have skew in times.
+      execution_time = clock.stop
+
+      @pipeline_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+
+      result
     end
 
     def define_initial_metrics_values
-      @events_metrics_counter.increment(0)
-      @pipeline_metrics_counter.increment(0)
-      @plugin_events_metrics_counter.increment(0)
-      @events_metrics_time.increment(0)
-      @pipeline_metrics_time.increment(0)
-      @plugin_events_metrics_time.increment(0)
+      @events_metrics.increment(:in, 0)
+      @pipeline_metrics.increment(:in, 0)
+      @plugin_events_metrics.increment(:out, 0)
+
+      @events_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @pipeline_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, 0)
     end
   end
 end end