logstash-core 5.1.2-java → 5.2.0-java

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -39,7 +39,6 @@ module LogStash module Instrument module PeriodicPoller
 
     def initialize(metric, options = {})
       super(metric, options)
-      @metric = metric
       @load_average = LoadAverage.create
     end
 
@@ -114,7 +113,7 @@ module LogStash module Instrument module PeriodicPoller
 
       metric.gauge([:jvm, :process, :cpu], :load_average, load_average) unless load_average.nil?
     end
-    
+
     def collect_jvm_metrics(data)
       runtime_mx_bean = ManagementFactory.getRuntimeMXBean()
       metric.gauge([:jvm], :uptime_in_millis, runtime_mx_bean.getUptime())

data/lib/logstash/instrument/periodic_poller/os.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
+require "logstash/instrument/periodic_poller/cgroup"
 
 module LogStash module Instrument module PeriodicPoller
   class Os < Base
@@ -8,6 +9,26 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
+      collect_cgroup
+    end
+
+    def collect_cgroup
+      if stats = Cgroup.get
+        save_metric([:os], :cgroup, stats)
+      end
+    end
+
+    # Recursive function to create the Cgroups values form the created hash
+    def save_metric(namespace, k, v)
+      if v.is_a?(Hash)
+        v.each do |new_key, new_value|
+          n = namespace.dup
+          n << k.to_sym
+          save_metric(n, new_key, new_value)
+        end
+      else
+        metric.gauge(namespace, k.to_sym, v)
+      end
     end
   end
 end; end; end

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/base"
+
+module LogStash module Instrument module PeriodicPoller
+  class PersistentQueue < Base
+    def initialize(metric, queue_type, agent, options = {})
+      super(metric, options)
+      @metric = metric
+      @queue_type = queue_type
+      @agent = agent
+    end
+
+    def collect
+      pipeline_id, pipeline = @agent.running_pipelines.first
+      unless pipeline.nil?
+        pipeline.collect_stats
+      end
+    end
+  end
+end; end; end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/os"
 require "logstash/instrument/periodic_poller/jvm"
+require "logstash/instrument/periodic_poller/pq"
 
 module LogStash module Instrument
   # Each PeriodPoller manager his own thread to do the poller
@@ -9,10 +10,11 @@ module LogStash module Instrument
   class PeriodicPollers
     attr_reader :metric
 
-    def initialize(metric)
+    def initialize(metric, queue_type, pipelines)
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
-                          PeriodicPoller::JVM.new(metric)]
+                           PeriodicPoller::JVM.new(metric),
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines)]
     end
 
     def start

data/lib/logstash/output_delegator.rb

@@ -39,7 +39,9 @@ module LogStash class OutputDelegator
 
   def multi_receive(events)
     @metric_events.increment(:in, events.length)
+    clock = @metric_events.time(:duration_in_millis)
     @strategy.multi_receive(events)
+    clock.stop
     @metric_events.increment(:out, events.length)
   end
 

data/lib/logstash/pipeline.rb

@@ -40,7 +40,8 @@ module LogStash; class Pipeline
     :settings,
     :metric,
     :filter_queue_client,
-    :input_queue_client
+    :input_queue_client,
+    :queue
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
@@ -69,7 +70,11 @@ module LogStash; class Pipeline
 
     # This needs to be configured before we evaluate the code to make
     # sure the metric instance is correctly send to the plugins to make the namespace scoping work
-    @metric = namespaced_metric.nil? ? Instrument::NullMetric.new : namespaced_metric
+    @metric = if namespaced_metric
+                settings.get("metric.collect") ? namespaced_metric : Instrument::NullMetric.new(namespaced_metric.collector)
+              else
+                Instrument::NullMetric.new
+              end
 
     grammar = LogStashConfigParser.new
     @config = grammar.parse(config_str)
@@ -549,6 +554,30 @@ module LogStash; class Pipeline
     end
   end
 
+  def collect_stats
+    pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
+    pipeline_metric.gauge(:type, settings.get("queue.type"))
+
+    if @queue.is_a?(LogStash::Util::WrappedAckedQueue) && @queue.queue.is_a?(LogStash::AckedQueue)
+      queue = @queue.queue
+      dir_path = queue.dir_path
+      file_store = Files.get_file_store(Paths.get(dir_path))
+
+      pipeline_metric.namespace([:capacity]).tap do |n|
+        n.gauge(:page_capacity_in_bytes, queue.page_capacity)
+        n.gauge(:max_queue_size_in_bytes, queue.max_size_in_bytes)
+        n.gauge(:max_unread_events, queue.max_unread_events)
+      end
+      pipeline_metric.namespace([:data]).tap do |n|
+        n.gauge(:free_space_in_bytes, file_store.get_unallocated_space)
+        n.gauge(:storage_type, file_store.type)
+        n.gauge(:path, dir_path)
+      end
+
+      pipeline_metric.gauge(:events, queue.unread_count)
+    end
+  end
+
   # Sometimes we log stuff that will dump the pipeline which may contain
   # sensitive information (like the raw syntax tree which can contain passwords)
   # We want to hide most of what's in here

data/lib/logstash/runner.rb

@@ -170,7 +170,12 @@ class LogStash::Runner < Clamp::StrictCommand
     rescue => e
       # abort unless we're just looking for the help
       unless cli_help?(args)
-        $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
+        if e.kind_of?(Psych::Exception)
+          yaml_file_path = ::File.join(LogStash::SETTINGS.get("path.settings"), "logstash.yml")
+          $stderr.puts "ERROR: Failed to parse YAML file \"#{yaml_file_path}\". Please confirm if the YAML structure is valid (e.g. look for incorrect usage of whitespace or indentation). Aborting... parser_error=>#{e.message}"
+        else
+          $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
+        end
         return 1
       end
     end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -33,6 +33,8 @@ module LogStash; module Util
 
     private_class_method :new
 
+    attr_reader :queue
+
     def with_queue(queue)
       @queue = queue
       @queue.open
@@ -130,10 +132,19 @@ module LogStash; module Util
 
       def set_events_metric(metric)
         @event_metric = metric
+        define_initial_metrics_values(@event_metric)
       end
 
       def set_pipeline_metric(metric)
         @pipeline_metric = metric
+        define_initial_metrics_values(@pipeline_metric)
+      end
+
+      def define_initial_metrics_values(namespaced_metric)
+        namespaced_metric.report_time(:duration_in_millis, 0)
+        namespaced_metric.increment(:filtered, 0)
+        namespaced_metric.increment(:in, 0)
+        namespaced_metric.increment(:out, 0)
       end
 
       def inflight_batches

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -79,10 +79,19 @@ module LogStash; module Util
 
       def set_events_metric(metric)
         @event_metric = metric
+        define_initial_metrics_values(@event_metric)
       end
 
       def set_pipeline_metric(metric)
         @pipeline_metric = metric
+        define_initial_metrics_values(@pipeline_metric)
+      end
+
+      def define_initial_metrics_values(namespaced_metric)
+        namespaced_metric.report_time(:duration_in_millis, 0)
+        namespaced_metric.increment(:filtered, 0)
+        namespaced_metric.increment(:in, 0)
+        namespaced_metric.increment(:out, 0)
       end
 
       def inflight_batches

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.1.2"
+LOGSTASH_VERSION = "5.2.0"

data/lib/logstash/webserver.rb

@@ -87,9 +87,17 @@ module LogStash
       @server = ::Puma::Server.new(app, events)
       @server.add_tcp_listener(http_host, port)
 
-      logger.info("Successfully started Logstash API endpoint", :port => @port)
+      logger.info("Successfully started Logstash API endpoint", :port => port)
+
+      set_http_address_metric("#{http_host}:#{port}")
 
       @server.run.join
     end
+
+    private
+    def set_http_address_metric(value)
+      return unless @agent.metric
+      @agent.metric.gauge([], :http_address, value)
+    end
   end
 end

data/locales/en.yml

@@ -44,9 +44,6 @@ en:
         %{plugin} plugin is using the 'milestone' method to declare the version
         of the plugin this method is deprecated in favor of declaring the
         version inside the gemspec.
-      no_version: >-
-        %{name} plugin doesn't have a version. This plugin isn't well
-         supported by the community and likely has no maintainer.
       version:
         0-9-x:
          Using version 0.9.x %{type} plugin '%{name}'. This plugin should work but

data/spec/api/lib/api/node_stats_spec.rb

@@ -80,7 +80,11 @@ describe LogStash::Api::Modules::NodeStats do
         "filtered" => Numeric,
         "out" => Numeric
      }
-    }
+   },
+   "reloads" => {
+     "successes" => Numeric,
+     "failures" => Numeric
+   }
   }
 
   test_api_and_resources(root_structure)

data/spec/api/spec_helper.rb

@@ -20,7 +20,9 @@ end
 module LogStash
   class DummyAgent < Agent
     def start_webserver
-      @webserver = Struct.new(:address).new("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}")
+      http_address = "#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}"
+      @webserver = Struct.new(:address).new(http_address)
+      self.metric.gauge([], :http_address, http_address)
     end
     def stop_webserver; end
   end

data/spec/logstash/agent_spec.rb

@@ -479,7 +479,9 @@ describe LogStash::Agent do
       it "increases the successful reload count" do
         snapshot = subject.metric.collector.snapshot_metric
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
+        instance_value = snapshot.metric_store.get_with_path("/stats")[:stats][:reloads][:successes].value
         expect(value).to eq(1)
+        expect(instance_value).to eq(1)
       end
 
       it "does not set the failure reload timestamp" do

data/spec/logstash/instrument/collector_spec.rb

@@ -45,5 +45,9 @@ describe LogStash::Instrument::Collector do
     it "return a `LogStash::Instrument::MetricStore`" do
       expect(subject.snapshot_metric).to be_kind_of(LogStash::Instrument::Snapshot)
     end
+
+    it "returns a clone of the metric store" do
+      expect(subject.snapshot_metric).not_to eq(subject.snapshot_metric)
+    end
   end
 end

data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb

@@ -0,0 +1,148 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/cgroup"
+require "spec_helper"
+
+describe LogStash::Instrument::PeriodicPoller::Cgroup do
+  subject { described_class }
+
+  context ".are_cgroup_available?" do
+    context "all the file exist" do
+      before do
+        allow(::File).to receive(:exist?).with(subject::PROC_SELF_CGROUP_FILE).and_return(true)
+        allow(::Dir).to receive(:exist?).with(subject::PROC_CGROUP_CPU_DIR).and_return(true)
+        allow(::Dir).to receive(:exist?).with(subject::PROC_CGROUP_CPUACCT_DIR).and_return(true)
+      end
+
+      it "returns true" do
+        expect(subject.are_cgroup_available?).to be_truthy
+      end
+    end
+
+    context "not all the file exist" do
+      before do
+        allow(::File).to receive(:exist?).with(subject::PROC_SELF_CGROUP_FILE).and_return(true)
+        allow(::Dir).to receive(:exist?).with(subject::PROC_CGROUP_CPU_DIR).and_return(false)
+        allow(::Dir).to receive(:exist?).with(subject::PROC_CGROUP_CPUACCT_DIR).and_return(true)
+      end
+
+      it "returns false" do
+        expect(subject.are_cgroup_available?).to be_falsey
+      end
+    end
+  end
+
+  context ".control_groups" do
+    let(:proc_self_cgroup_content) {
+      %w(14:name=systemd,holaunlimited:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+13:pids:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+12:hugetlb:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+11:net_prio:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+10:perf_event:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+9:net_cls:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+8:freezer:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+7:devices:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+6:memory:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+5:blkio:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+4:cpuacct:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+3:cpu:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+2:cpuset:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+1:name=openrc:/docker) }
+
+    before do
+      allow(subject).to receive(:read_proc_self_cgroup_lines).and_return(proc_self_cgroup_content)
+    end
+
+    it "returns the control groups" do
+      expect(subject.control_groups).to match({
+        "name=systemd" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "holaunlimited" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "pids" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "hugetlb" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "net_prio" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "perf_event" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "net_cls" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "freezer" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "devices" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "memory" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "blkio" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "cpuacct" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "cpu" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "cpuset" => "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61",
+        "name=openrc" => "/docker"
+      })
+    end
+  end
+
+  context ".get_all" do
+    context "when we can retreive the stats" do
+      let(:cpuacct_control_group) { "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61" }
+      let(:cpuacct_usage) { 1982 }
+      let(:cpu_control_group) { "/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61" }
+      let(:cfs_period_micros) { 500 }
+      let(:cfs_quota_micros) { 98 }
+      let(:cpu_stats_number_of_periods) { 1 }
+      let(:cpu_stats_number_of_time_throttled) { 2 }
+      let(:cpu_stats_time_throttled_nanos) { 3 }
+      let(:proc_self_cgroup_content) {
+        %W(14:name=systemd,holaunlimited:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+13:pids:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+12:hugetlb:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+11:net_prio:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+10:perf_event:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+9:net_cls:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+8:freezer:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+7:devices:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+6:memory:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+5:blkio:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+4:cpuacct:#{cpuacct_control_group}
+3:cpu:#{cpu_control_group}
+2:cpuset:/docker/a10687343f90e97bbb1f7181bd065a42de96c40c4aa91764a9d526ea30475f61
+1:name=openrc:/docker) }
+      let(:cpu_stat_file_content) {
+        [
+          "nr_periods #{cpu_stats_number_of_periods}",
+          "nr_throttled #{cpu_stats_number_of_time_throttled}",
+          "throttled_time #{cpu_stats_time_throttled_nanos}"
+        ]
+      }
+
+      before do
+        allow(subject).to receive(:read_proc_self_cgroup_lines).and_return(proc_self_cgroup_content)
+        allow(subject).to receive(:read_sys_fs_cgroup_cpuacct_cpu_stat).and_return(cpu_stat_file_content)
+
+        allow(subject).to receive(:cgroup_cpuacct_usage_nanos).with(cpuacct_control_group).and_return(cpuacct_usage)
+        allow(subject).to receive(:cgroup_cpu_fs_period_micros).with(cpu_control_group).and_return(cfs_period_micros)
+        allow(subject).to receive(:cgroup_cpu_fs_quota_micros).with(cpu_control_group).and_return(cfs_quota_micros)
+      end
+
+      it "returns all the stats" do
+        expect(subject.get_all).to match(
+          :cpuacct => {
+            :control_group => cpuacct_control_group,
+            :usage_nanos => cpuacct_usage,
+          },
+          :cpu => {
+            :control_group => cpu_control_group,
+            :cfs_period_micros => cfs_period_micros,
+            :cfs_quota_micros => cfs_quota_micros,
+            :stat => {
+                :number_of_elapsed_periods => cpu_stats_number_of_periods,
+                :number_of_times_throttled => cpu_stats_number_of_time_throttled,
+                :time_throttled_nanos => cpu_stats_time_throttled_nanos
+            }
+          }
+        )
+      end
+    end
+
+    context "when an exception is raised" do
+      before do
+        allow(subject).to receive(:control_groups).and_raise("Something went wrong")
+      end
+
+      it "returns nil" do
+        expect(subject.get_all).to be_nil
+      end
+    end
+  end
+end