logstash-core 7.2.1-java → 7.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '049f2a5cf05f1811e4393f09bbf0dfd07b63fcfc27e760fefafdb7d78c340659'
-  data.tar.gz: a22c780783a2d190f4e97877e5e41191921a49568184b59c1d12ce276f55e567
+  metadata.gz: 4d89cdfc9aa7abd577325a9452457eabce0c675f0e279c83e1d412717597113c
+  data.tar.gz: 8bbe82e61e6a0f8066b47869eacae5e70d3d9f9d5c982509b550ef0476cae934
 SHA512:
-  metadata.gz: ce56af4e9f40c706cd08f7ef6fb00dcab449e64b72968312feff85aa0c8460a28b48b41fa5b059eb75b610a679ccfef38653cf5774160eb3878eb5fded78ec1d
-  data.tar.gz: 6d480b9c63e22f25b2b51521e6da69013bac56f875b803c94e173a8153ab2618bae4d366d90904286ecde7bdb4f0584030aad05fab1bb2103d251835058691ee
+  metadata.gz: e8804c92c588d2acd081318b94af4c92d1cdb310c578ff3e2915a1b4d40754450d9b6a62e550a101e33b88b61dfea7f9b4605fcaa0dc77a269e72f76dac7343e
+  data.tar.gz: ffcdf7710fece27cddc70601bd6650c6b6efba007fa75486e93e884754ac84828429cc7b8bdb3c5658599072c6544a720b02f7a1ee1d27931125f5af07d605ec

data/lib/logstash/api/commands/default_metadata.rb

@@ -15,6 +15,11 @@ module LogStash
            :ephemeral_id => service.agent.ephemeral_id,
            :status => "green",  # This is hard-coded to mirror x-pack behavior
            :snapshot => ::BUILD_INFO["build_snapshot"],
+           :pipeline => {
+             :workers => LogStash::SETTINGS.get("pipeline.workers"),
+             :batch_size => LogStash::SETTINGS.get("pipeline.batch.size"),
+             :batch_delay => LogStash::SETTINGS.get("pipeline.batch.delay"),
+           }
            }
         end
 

data/lib/logstash/api/commands/node.rb

@@ -36,10 +36,12 @@ module LogStash
             :config_reload_interval,
             :dead_letter_queue_enabled,
             :dead_letter_queue_path,
-            :cluster_uuids
           ).reject{|_, v|v.nil?}
           if options.fetch(:graph, false)
-            metrics.merge!(extract_metrics([:stats, :pipelines, pipeline_id.to_sym, :config], :graph))
+            extended_stats = extract_metrics([:stats, :pipelines, pipeline_id.to_sym, :config], :graph)
+            decorated_vertices = extended_stats[:graph]["graph"]["vertices"].map { |vertex| decorate_with_cluster_uuids(vertex)  }
+            extended_stats[:graph]["graph"]["vertices"] = decorated_vertices
+            metrics.merge!(extended_stats)
           end
           metrics
         rescue
@@ -78,6 +80,24 @@ module LogStash
         def hot_threads(options={})
           HotThreadsReport.new(self, options)
         end
+
+        private
+        ##
+        # Returns a vertex, decorated with the cluster UUID metadata retrieved from ES
+        # Does not mutate the passed `vertex` object.
+        # @api private
+        # @param vertex [Hash{String=>Object}]
+        # @return [Hash{String=>Object}]
+        def decorate_with_cluster_uuids(vertex)
+          plugin_id = vertex["id"]&.to_s
+          return vertex unless plugin_id && LogStash::PluginMetadata.exists?(plugin_id)
+
+          plugin_metadata = LogStash::PluginMetadata.for_plugin(plugin_id)
+          cluster_uuid = plugin_metadata&.get(:cluster_uuid)
+          vertex = vertex.merge("cluster_uuid" => cluster_uuid) unless cluster_uuid.nil?
+
+          vertex
+        end
       end
     end
   end

data/lib/logstash/api/commands/stats.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/api/commands/base"
 require 'logstash/util/thread_dump'
+require 'logstash/config/pipelines_info'
 require_relative "hot_threads_reporter"
 
 java_import java.nio.file.Files
@@ -10,6 +11,20 @@ module LogStash
   module Api
     module Commands
       class Stats < Commands::Base
+        def queue
+          pipeline_ids = service.get_shallow(:stats, :pipelines).keys
+          total_queued_events = 0
+          pipeline_ids.each do |pipeline_id|
+            p_stats = service.get_shallow(:stats, :pipelines, pipeline_id.to_sym)
+            type = p_stats[:queue] && p_stats[:queue][:type].value
+            pipeline = service.agent.get_pipeline(pipeline_id)
+            next if pipeline.nil? || pipeline.system? || type != 'persisted'
+            total_queued_events += p_stats[:queue][:events].value
+          end
+
+          {:events_count => total_queued_events}
+        end
+
         def jvm
           {
             :threads => extract_metrics(
@@ -45,14 +60,24 @@ module LogStash
           )
         end
 
-        def pipeline(pipeline_id = nil)
+        def pipeline(pipeline_id = nil, opts={})
+          extended_stats = LogStash::Config::PipelinesInfo.format_pipelines_info(
+            service.agent,
+            service.snapshot.metric_store,
+            true).each_with_object({}) do |pipeline_stats, memo|
+              pipeline_id = pipeline_stats["id"].to_s
+              memo[pipeline_id] = pipeline_stats
+            end
+
           if pipeline_id.nil?
             pipeline_ids = service.get_shallow(:stats, :pipelines).keys
             pipeline_ids.each_with_object({}) do |pipeline_id, result|
-              result[pipeline_id] = plugins_stats_report(pipeline_id)
+              extended_pipeline = extended_stats[pipeline_id.to_s]
+              result[pipeline_id] = plugins_stats_report(pipeline_id, extended_pipeline, opts)
             end
           else
-            { pipeline_id => plugins_stats_report(pipeline_id) }
+            extended_pipeline = extended_stats[pipeline_id.to_s]
+            { pipeline_id => plugins_stats_report(pipeline_id, extended_pipeline, opts) }
           end
         rescue # failed to find pipeline
           {}
@@ -92,9 +117,9 @@ module LogStash
         end
 
         private
-        def plugins_stats_report(pipeline_id)
+        def plugins_stats_report(pipeline_id, extended_pipeline, opts={})
           stats = service.get_shallow(:stats, :pipelines, pipeline_id.to_sym)
-          PluginsStats.report(stats)
+          PluginsStats.report(stats, extended_pipeline, opts)
         end
 
         module PluginsStats
@@ -110,8 +135,8 @@ module LogStash
             end
           end
 
-          def report(stats)
-            {
+          def report(stats, extended_stats=nil, opts={})
+            ret = {
               :events => stats[:events],
               :plugins => {
                 :inputs => plugin_stats(stats, :inputs),
@@ -121,8 +146,38 @@ module LogStash
               },
               :reloads => stats[:reloads],
               :queue => stats[:queue]
-            }.merge(stats[:dlq] ? {:dead_letter_queue => stats[:dlq]} : {})
+            }
+            ret[:dead_letter_queue] = stats[:dlq] if stats.include?(:dlq)
+
+            # if extended_stats were provided, enrich the return value
+            if extended_stats
+              ret[:queue]    = extended_stats["queue"] if extended_stats.include?("queue")
+              ret[:hash] = extended_stats["hash"]
+              ret[:ephemeral_id] = extended_stats["ephemeral_id"]
+              if opts[:vertices] && extended_stats.include?("vertices")
+                ret[:vertices] = extended_stats["vertices"].map { |vertex| decorate_vertex(vertex) }
+              end
             end
+
+            ret
+          end
+
+          ##
+          # Returns a vertex, decorated with additional metadata if available.
+          # Does not mutate the passed `vertex` object.
+          # @api private
+          # @param vertex [Hash{String=>Object}]
+          # @return [Hash{String=>Object}]
+          def decorate_vertex(vertex)
+            plugin_id = vertex["id"]&.to_s
+            return vertex unless plugin_id && LogStash::PluginMetadata.exists?(plugin_id)
+
+            plugin_metadata = LogStash::PluginMetadata.for_plugin(plugin_id)
+            cluster_uuid = plugin_metadata&.get(:cluster_uuid)
+            vertex = vertex.merge("cluster_uuid" => cluster_uuid) unless cluster_uuid.nil?
+
+            vertex
+          end
         end # module PluginsStats
       end
     end

data/lib/logstash/api/modules/node.rb

@@ -29,14 +29,16 @@ module LogStash
 
         get "/pipelines/:id" do
           pipeline_id = params["id"]
-          opts = {:graph => as_boolean(params.fetch("graph", false))}
+          opts = {:graph => as_boolean(params.fetch("graph", false)),
+                  :vertices => as_boolean(params.fetch("vertices", false))}
           payload = node.pipeline(pipeline_id, opts)
           halt(404) if payload.empty?
           respond_with(:pipelines => { pipeline_id => payload } )
         end
 
         get "/pipelines" do
-          opts = {:graph => as_boolean(params.fetch("graph", false))}
+          opts = {:graph => as_boolean(params.fetch("graph", false)),
+                  :vertices => as_boolean(params.fetch("vertices", false))}
           payload = node.pipelines(opts)
           halt(404) if payload.empty?
           respond_with(:pipelines => payload )

data/lib/logstash/api/modules/node_stats.rb

@@ -21,11 +21,17 @@ module LogStash
             :events => events_payload,
             :pipelines => pipeline_payload,
             :reloads => reloads_payload,
-            :os => os_payload
+            :os => os_payload,
+            :queue => queue
           }
           respond_with(payload, {:filter => params["filter"]})
         end
 
+        private
+        def queue
+          @stats.queue
+        end
+
         private
         def os_payload
           @stats.os
@@ -52,7 +58,8 @@ module LogStash
         end
 
         def pipeline_payload(val = nil)
-          @stats.pipeline(val)
+          opts = {:vertices => as_boolean(params.fetch("vertices", false))}
+          @stats.pipeline(val, opts)
         end
       end
     end

data/lib/logstash/config/lir_serializer.rb

@@ -124,3 +124,4 @@ module LogStash;
   end
   end
 end
+

data/lib/logstash/config/pipelines_info.rb

@@ -0,0 +1,152 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+module LogStash; module Config;
+  class PipelinesInfo
+    def self.format_pipelines_info(agent, metric_store, extended_performance_collection)
+      # It is important that we iterate via the agent's pipelines vs. the
+      # metrics pipelines. This prevents race conditions as pipeline stats may be
+      # populated before the agent has it in its own pipelines state
+      stats = metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines]
+      agent.running_pipelines.map do |pipeline_id, pipeline|
+        p_stats = stats[pipeline_id]
+        # Don't record stats for system pipelines
+        next nil if pipeline.system?
+        res = {
+          "id" => pipeline_id.to_s,
+          "hash" => pipeline.lir.unique_hash,
+          "ephemeral_id" => pipeline.ephemeral_id,
+          "events" => format_pipeline_events(p_stats[:events]),
+          "queue" => format_queue_stats(pipeline_id, metric_store),
+          "reloads" => {
+            "successes" => p_stats[:reloads][:successes].value,
+            "failures" => p_stats[:reloads][:failures].value
+          }
+        }
+        if extended_performance_collection
+          res["vertices"] = format_pipeline_vertex_stats(p_stats[:plugins], pipeline)
+        end
+        res
+      end.compact
+    end
+
+    def self.format_pipeline_events(stats)
+      result = {}
+      (stats || {}).each { |stage, counter| result[stage.to_s] = counter.value }
+      result
+    end
+
+    def self.format_pipeline_vertex_stats(stats, pipeline)
+      return nil unless stats
+      
+      [:inputs, :filters, :outputs].flat_map do |section|
+        format_pipeline_vertex_section_stats(stats[section], pipeline)
+      end.select {|stats| !stats.nil?} # Ignore empty sections
+    end
+
+    ROOT_METRIC_MAPPINGS = {
+      'events.in' => 'events_in',
+      'events.out' => 'events_out',
+      'events.queue_push_duration_in_millis' => 'queue_push_duration_in_millis',
+      'events.duration_in_millis' => 'duration_in_millis',
+      'name' => :discard # we don't need this, pipeline_state has this already
+    }
+
+    def self.format_pipeline_vertex_section_stats(stats, pipeline)
+      return nil unless stats
+      
+       (stats || {}).reduce([]) do |acc,kv|
+        plugin_id, plugin_stats = kv
+        
+        props = Hash.new {|h,k| h[k] = []}
+        next unless plugin_stats
+        
+        flattened = flatten_metrics(plugin_stats)
+
+        segmented = flattened.reduce(Hash.new {|h,k| h[k] = []}) do |acc,kv|
+          k,v = kv
+          metric_value = v.value
+          root_metric_field = ROOT_METRIC_MAPPINGS[k]
+
+          if root_metric_field
+            if root_metric_field != :discard
+              acc[root_metric_field] = metric_value
+            end
+          else
+            type_sym = v.type.to_sym
+
+            nested_type = if type_sym == :"counter/long"
+                            :long_counters
+                          elsif type_sym == :"gauge/numeric"
+                            :double_gauges
+                          else
+                            nil
+                          end
+
+            if nested_type
+              acc[nested_type] << { :name => k, :value => metric_value }
+            end
+         end
+
+          acc
+        end
+        segment = {
+          :id => plugin_id,
+          :pipeline_ephemeral_id => pipeline.ephemeral_id
+        }
+
+        if LogStash::PluginMetadata.exists?(plugin_id.to_s)
+          plugin_metadata = LogStash::PluginMetadata.for_plugin(plugin_id.to_s)
+          cluster_uuid = plugin_metadata&.get(:cluster_uuid)
+          segment[:cluster_uuid] = cluster_uuid unless cluster_uuid.nil?
+        end
+
+        acc << segment.merge(segmented)
+        acc
+      end
+    end
+
+    def self.flatten_metrics(hash_or_value, namespaces = [])
+      if hash_or_value.is_a?(Hash)
+        return hash_or_value.reduce({}) do |acc,kv| 
+          k,v = kv
+          # We must concat the arrays, creating a copy instead of mutation
+          # to handle the case where there are multiple sibling metrics in a namespace
+          new_namespaces = namespaces.clone
+          new_namespaces << k
+          acc.merge(flatten_metrics(v, new_namespaces))
+        end
+      else
+        { namespaces.join('.') => hash_or_value }
+      end
+    end
+
+    def self.format_queue_stats(pipeline_id, metric_store)
+      path = [:stats, :pipelines, pipeline_id, :queue, :type]
+      if metric_store.has_metric?(*path)
+        queue_type = metric_store.get_shallow(*path).value
+      else
+        queue_type = 'memory'
+      end
+
+      events = 0
+      queue_size_in_bytes = 0
+      max_queue_size_in_bytes = 0
+
+      if queue_type == "persisted"
+        queue_path = [:stats, :pipelines, pipeline_id, :queue]
+        events = metric_store.get_shallow(*queue_path, :events).value
+        queue_size_in_bytes = metric_store.get_shallow(*queue_path, :capacity, :queue_size_in_bytes).value
+        max_queue_size_in_bytes = metric_store.get_shallow(*queue_path, :capacity, :max_queue_size_in_bytes).value
+      end
+
+      {
+          :type => queue_type,
+          :events_count => events,
+          :queue_size_in_bytes => queue_size_in_bytes,
+          :max_queue_size_in_bytes => max_queue_size_in_bytes,
+      }
+    end
+  end
+end; end;

data/logstash-core.gemspec

@@ -72,5 +72,4 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "elasticsearch", "~> 5"
   gem.add_runtime_dependency "manticore", '~> 0.6'
-  gem.add_runtime_dependency "faraday", '~> 0.9.0'
 end

data/versions-gem-copy.yml

@@ -1,7 +1,7 @@
 ---
 # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var
-logstash: 7.2.1
-logstash-core: 7.2.1
+logstash: 7.3.0
+logstash-core: 7.3.0
 logstash-core-plugin-api: 2.1.16
 
 # jruby must reference a *released* version of jruby which can be downloaded from the official download url

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 7.2.1
+  version: 7.3.0
 platform: java
 authors:
 - Elastic
@@ -288,20 +288,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.0
-  name: faraday
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.0
 description: The core components of logstash, the scalable log and event management
   tool
 email:
@@ -352,6 +338,7 @@ files:
 - lib/logstash/config/mixin.rb
 - lib/logstash/config/modules_common.rb
 - lib/logstash/config/pipeline_config.rb
+- lib/logstash/config/pipelines_info.rb
 - lib/logstash/config/source/base.rb
 - lib/logstash/config/source/local.rb
 - lib/logstash/config/source/modules.rb