logstash-core 7.1.1-java → 7.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64360418777f1955caf42e8b14ac7c9201a641bd668612c55f9382cdf0f94b9e
-  data.tar.gz: 96661365e73cb4eeaaf943a7cc25e87f9e5639f81a39657f77578ca9c2097c2e
+  metadata.gz: f34b96ed7e4139bc1eb3d820d6f325e3bd579be9381a369be73aa08dc231d4e1
+  data.tar.gz: e8d3c2aa6e2d59b44e7f99f27af966ed962608f90e7c9af0f68b5a459d4804cf
 SHA512:
-  metadata.gz: 9343c65ed931f58f9350972d9fffccbab77674cce2c7f4001a9d7a9b5b37f30461a5e5b687c61ec6efca6093517d1ce8a67ac34bce0d38d95e9f8493a19ab841
-  data.tar.gz: 73c06d0f1400e194db5373300cb0464dec9094dfa822147e0f3b911571aeab464d3bebded47435318b0360aa38753fd39010fd8a2dacdc1db67e883829f14de0
+  metadata.gz: 0c13a53ea439c0d4293ab3e6bcb4f91bd93e98c98184d2cadc1c1466372bd2af7ad469522802f4e82fa5d66cae9b486c8b3b51111ebdd683c50e5203b4975cae
+  data.tar.gz: c7dbd2b483cc41c47fd3036dc4bf7223578de3cdb4ef7ea5d75cfa3c4bf7572808756bfb5dc85ac394a9e74cb238b2c19707ca6361edb871de8c2dd403f0aa15

data/lib/logstash/api/commands/default_metadata.rb

@@ -7,8 +7,15 @@ module LogStash
     module Commands
       class DefaultMetadata < Commands::Base
         def all
-          {:host => host, :version => version, :http_address => http_address,
-           :id => service.agent.id, :name => service.agent.name}
+          {:host => host,
+           :version => version,
+           :http_address => http_address,
+           :id => service.agent.id,
+           :name => service.agent.name,
+           :ephemeral_id => service.agent.ephemeral_id,
+           :status => "green",  # This is hard-coded to mirror x-pack behavior
+           :snapshot => ::BUILD_INFO["build_snapshot"],
+           }
         end
 
         def host

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -3,11 +3,11 @@ java_import 'org.logstash.instrument.reports.ThreadsReport'
 
 class HotThreadsReport
   STRING_SEPARATOR_LENGTH = 80.freeze
-  HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
+  HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 50.freeze
 
   def initialize(cmd, options)
     @cmd = cmd
-    filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
+    filter = { 'stacktrace_size' => "#{options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT)}" }
     @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => ThreadsReport.generate(filter)))
   end
 

data/lib/logstash/api/commands/node.rb

@@ -17,18 +17,31 @@ module LogStash
           payload
         end
 
-        def pipelines
+        def pipelines(options={})
           pipeline_ids = service.get_shallow(:stats, :pipelines).keys
           pipeline_ids.each_with_object({}) do |pipeline_id, result|
-            result[pipeline_id] = pipeline(pipeline_id)
+            result[pipeline_id] = pipeline(pipeline_id, options)
           end
         end
 
-        def pipeline(pipeline_id)
-          extract_metrics(
+        def pipeline(pipeline_id, options={})
+          metrics = extract_metrics(
             [:stats, :pipelines, pipeline_id.to_sym, :config],
-            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval, :dead_letter_queue_enabled, :dead_letter_queue_path
+            :ephemeral_id,
+            :hash,
+            :workers,
+            :batch_size,
+            :batch_delay,
+            :config_reload_automatic,
+            :config_reload_interval,
+            :dead_letter_queue_enabled,
+            :dead_letter_queue_path,
+            :cluster_uuids
           ).reject{|_, v|v.nil?}
+          if options.fetch(:graph, false)
+            metrics.merge!(extract_metrics([:stats, :pipelines, pipeline_id.to_sym, :config], :graph))
+          end
+          metrics
         rescue
           {}
         end

data/lib/logstash/api/commands/stats.rb

@@ -115,6 +115,7 @@ module LogStash
               :events => stats[:events],
               :plugins => {
                 :inputs => plugin_stats(stats, :inputs),
+                :codecs => plugin_stats(stats, :codecs),
                 :filters => plugin_stats(stats, :filters),
                 :outputs => plugin_stats(stats, :outputs)
               },

data/lib/logstash/api/modules/node.rb

@@ -13,9 +13,10 @@ module LogStash
         get "/hot_threads" do
           begin
             ignore_idle_threads = params["ignore_idle_threads"] || true
-
             options = {:ignore_idle_threads => as_boolean(ignore_idle_threads)}
             options[:threads] = params["threads"].to_i if params.has_key?("threads")
+            options[:ordered_by] = params["ordered_by"] if params.has_key?("ordered_by")
+            options[:stacktrace_size] = params["stacktrace_size"] if params.has_key?("stacktrace_size")
 
             as = human? ? :string : :json
             respond_with(node.hot_threads(options), {:as => as})
@@ -28,11 +29,19 @@ module LogStash
 
         get "/pipelines/:id" do
           pipeline_id = params["id"]
-          payload = node.pipeline(pipeline_id)
+          opts = {:graph => as_boolean(params.fetch("graph", false))}
+          payload = node.pipeline(pipeline_id, opts)
           halt(404) if payload.empty?
           respond_with(:pipelines => { pipeline_id => payload } )
         end
 
+        get "/pipelines" do
+          opts = {:graph => as_boolean(params.fetch("graph", false))}
+          payload = node.pipelines(opts)
+          halt(404) if payload.empty?
+          respond_with(:pipelines => payload )
+        end
+
          get "/?:filter?" do
            selected_fields = extract_fields(params["filter"].to_s.strip)
            values = node.all(selected_fields)

data/lib/logstash/codecs/delegator.rb

@@ -0,0 +1,52 @@
+module LogStash::Codecs
+  class Delegator < SimpleDelegator
+    def initialize(obj)
+      super(obj)
+      @encode_metric = LogStash::Instrument::NamespacedNullMetric.new
+      @decode_metric = LogStash::Instrument::NamespacedNullMetric.new
+    end
+
+    def class
+      __getobj__.class
+    end
+
+    def metric=(metric)
+      __getobj__.metric = metric
+
+      __getobj__.metric.gauge(:name, __getobj__.class.config_name)
+
+      @encode_metric = __getobj__.metric.namespace(:encode)
+      @encode_metric.counter(:writes_in)
+      @encode_metric.report_time(:duration_in_millis, 0)
+
+      @decode_metric = __getobj__.metric.namespace(:decode)
+      @decode_metric.counter(:writes_in)
+      @decode_metric.counter(:out)
+      @decode_metric.report_time(:duration_in_millis, 0)
+    end
+
+    def encode(event)
+      @encode_metric.increment(:writes_in)
+      @encode_metric.time(:duration_in_millis) do
+        __getobj__.encode(event)
+      end
+    end
+
+    def multi_encode(events)
+      @encode_metric.increment(:writes_in, events.length)
+      @encode_metric.time(:duration_in_millis) do
+        __getobj__.multi_encode(events)
+      end
+    end
+
+    def decode(data)
+      @decode_metric.increment(:writes_in)
+      @decode_metric.time(:duration_in_millis) do
+        __getobj__.decode(data) do |event|
+          @decode_metric.increment(:out)
+          yield event
+        end
+      end
+    end
+  end
+end

data/lib/logstash/compiler/lscl.rb

@@ -3,6 +3,7 @@ require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
 require "logstash/compiler/lscl/helpers"
 require "logstash/config/string_escape"
+require "logstash/util"
 
 java_import org.logstash.config.ir.DSL
 java_import org.logstash.common.SourceWithMetadata
@@ -111,7 +112,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
           # hash value; e.g., `{"match" => {"baz" => "bar"}, "match" => {"foo" => "bulb"}}` is
           # interpreted as `{"match" => {"baz" => "bar", "foo" => "blub"}}`.
           # (NOTE: this bypasses `AST::Hash`'s ability to detect duplicate keys)
-          hash[k] = existing.merge(v)
+          hash[k] = ::LogStash::Util.hash_merge_many(existing, v)
         elsif existing.kind_of?(::Array)
           hash[k] = existing.push(*v)
         else
@@ -164,8 +165,8 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
   class Number < Value
     def expr
       jdsl.eValue(source_meta, text_value.include?(".") ?
-        text_value.to_f :
-        text_value.to_i)
+                                   Float(text_value) :
+                                   Integer(text_value))
     end
   end
 

data/lib/logstash/config/lir_serializer.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require 'logstash-core'
+require 'logstash/compiler'
+
+module LogStash;
+  module Config;
+  class LIRSerializer
+    attr_reader :lir_pipeline
+    
+    def self.serialize(lir_pipeline)
+      self.new(lir_pipeline).serialize
+    end
+    
+    def initialize(lir_pipeline)
+      @lir_pipeline = lir_pipeline
+    end
+    
+    def serialize
+      {
+        "hash" => lir_pipeline.unique_hash,
+        "type" => "lir",
+        "version" => "0.0.0",
+        "graph" => {
+          "vertices" => vertices,
+          "edges" => edges
+        }
+      }
+    end
+    
+    def vertices
+      graph.getVertices.map {|v| vertex(v) }
+    end
+    
+    def edges
+      graph.getEdges.map {|e| edge(e) }
+    end
+    
+    def graph
+      lir_pipeline.graph
+    end
+    
+    def vertex(v)
+      hashified_vertex = case vertex_type(v) 
+                         when :plugin 
+                           plugin_vertex(v)
+                         when :if 
+                           if_vertex(v)
+                         when :queue
+                           queue_vertex(v)
+                         end
+
+      decorate_vertex(v, hashified_vertex)
+    end
+    
+    def vertex_type(v)
+      if v.java_kind_of?(org.logstash.config.ir.graph.PluginVertex)
+        :plugin
+      elsif v.java_kind_of?(org.logstash.config.ir.graph.IfVertex)
+        :if
+      elsif v.java_kind_of?(org.logstash.config.ir.graph.QueueVertex)
+        :queue
+      else
+        raise "Unexpected vertex type! #{v}"
+      end
+    end
+    
+    def decorate_vertex(v, v_json)
+      v_json["meta"] = format_swm(v.source_with_metadata)
+      v_json["id"] = v.id
+      v_json["explicit_id"] = !!v.explicit_id
+      v_json["type"] = vertex_type(v).to_s
+      v_json
+    end
+    
+    def plugin_vertex(v)
+      pd = v.plugin_definition
+      {
+        "config_name" => pd.name,
+        "plugin_type" => pd.getType.to_s.downcase
+      }
+    end
+    
+    def if_vertex(v)
+      {
+        "condition" => v.humanReadableExpression
+      }
+    end
+    
+    def queue_vertex(v)
+      {}
+    end
+    
+    def edge(e)
+      e_json = {
+        "from" => e.from.id,
+        "to" => e.to.id,
+        "id" => e.id
+      }
+      
+      if e.java_kind_of?(org.logstash.config.ir.graph.BooleanEdge)
+        e_json["when"] = e.edge_type
+        e_json["type"] = "boolean"
+      else
+        e_json["type"] = "plain"
+      end
+      
+      e_json
+    end
+    
+    def format_swm(source_with_metadata)
+      return nil unless source_with_metadata
+      {
+        "source" => {
+          "protocol" => source_with_metadata.protocol,
+          "id" => source_with_metadata.id,
+          "line" =>  source_with_metadata.line,
+          "column" => source_with_metadata.column
+          # We omit the text of the source code for security reasons
+          # raw text may contain passwords
+        }
+      }
+    end
+    
+  end
+  end
+end

data/lib/logstash/config/mixin.rb

@@ -4,6 +4,7 @@ require "logstash/util/safe_uri"
 require "logstash/version"
 require "logstash/environment"
 require "logstash/util/plugin_version"
+require "logstash/codecs/delegator"
 require "filesize"
 
 LogStash::Environment.load_locale!
@@ -410,7 +411,7 @@ module LogStash::Config::Mixin
         case validator
           when :codec
             if value.first.is_a?(String)
-              value = LogStash::Plugin.lookup("codec", value.first).new
+              value = LogStash::Codecs::Delegator.new LogStash::Plugin.lookup("codec", value.first).new
               return true, value
             else
               value = value.first

data/lib/logstash/environment.rb

@@ -43,6 +43,7 @@ module LogStash
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
            Setting::Boolean.new("pipeline.java_execution", true),
            Setting::Boolean.new("pipeline.reloadable", true),
+           Setting::Boolean.new("pipeline.plugin_classloaders", false),
                     Setting.new("path.plugins", Array, []),
     Setting::NullableString.new("interactive", nil, false),
            Setting::Boolean.new("config.debug", false),
@@ -163,11 +164,15 @@ module LogStash
     end
 
     def windows?
-      RbConfig::CONFIG['host_os'] =~ WINDOW_OS_RE
+      host_os =~ WINDOW_OS_RE
     end
 
     def linux?
-      RbConfig::CONFIG['host_os'] =~ LINUX_OS_RE
+      host_os =~ LINUX_OS_RE
+    end
+
+    def host_os
+      RbConfig::CONFIG['host_os']
     end
 
     def locales_path(path)

data/lib/logstash/inputs/base.rb

@@ -99,6 +99,13 @@ class LogStash::Inputs::Base < LogStash::Plugin
     cloned
   end
 
+  def metric=(metric)
+    super
+    # Hack to create a new metric namespace using 'plugins' as the root
+    @codec.metric = metric.root.namespace(metric.namespace_name[0...-2].push(:codecs, codec.id))
+    metric
+  end
+
   def execution_context=(context)
     super
     # There is no easy way to propage an instance variable into the codec, because the codec

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -153,12 +153,15 @@ module LogStash module Instrument module PeriodicPoller
       old  = {}
       old = old.merge!(heap["CMS Old Gen"]) if heap.has_key?("CMS Old Gen")
       old = old.merge!(heap["PS Old Gen"])  if heap.has_key?("PS Old Gen")
+      old = old.merge!(heap["G1 Old Gen"])  if heap.has_key?("G1 Old Gen")
       young = {}
       young = young.merge!(heap["Par Eden Space"]) if heap.has_key?("Par Eden Space")
       young = young.merge!(heap["PS Eden Space"])  if heap.has_key?("PS Eden Space")
+      young = young.merge!(heap["G1 Eden Space"])  if heap.has_key?("G1 Eden Space")
       survivor = {}
       survivor = survivor.merge!(heap["Par Survivor Space"]) if heap.has_key?("Par Survivor Space")
       survivor = survivor.merge!(heap["PS Survivor Space"])  if heap.has_key?("PS Survivor Space")
+      survivor = survivor.merge!(heap["G1 Survivor Space"])  if heap.has_key?("G1 Survivor Space")
       {
         "young"    => aggregate_information_for(young),
         "old"      => aggregate_information_for(old),

data/lib/logstash/java_pipeline.rb

@@ -6,6 +6,7 @@ require "logstash/inputs/base"
 require "logstash/outputs/base"
 require "logstash/instrument/collector"
 require "logstash/compiler"
+require "logstash/config/lir_serializer"
 
 module LogStash; class JavaPipeline < JavaBasePipeline
   include LogStash::Util::Loggable
@@ -216,7 +217,10 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       config_metric.gauge(:config_reload_interval, settings.get("config.reload.interval"))
       config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
       config_metric.gauge(:dead_letter_queue_path, dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
-
+      config_metric.gauge(:ephemeral_id, ephemeral_id)
+      config_metric.gauge(:hash, lir.unique_hash)
+      config_metric.gauge(:graph, ::LogStash::Config::LIRSerializer.serialize(lir))
+      config_metric.gauge(:cluster_uuids, resolve_cluster_uuids)
 
       @logger.info("Starting pipeline", default_logging_keys(
         "pipeline.workers" => pipeline_workers,
@@ -255,6 +259,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     end
   end
 
+  def resolve_cluster_uuids
+    outputs.each_with_object(Set.new) do |output, cluster_uuids|
+      if LogStash::PluginMetadata.exists?(output.id)
+        cluster_uuids << LogStash::PluginMetadata.for_plugin(output.id).get(:cluster_uuid)
+      end
+    end.to_a.compact
+  end
+
   def wait_inputs
     @input_threads.each do |thread|
       if thread.class == Java::JavaObject
@@ -313,9 +325,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       # Assuming the failure that caused this exception is transient,
       # let's sleep for a bit and execute #run again
       sleep(1)
+      begin
+        plugin.do_close
+      rescue => close_exception
+        @logger.debug("Input plugin raised exception while closing, ignoring",
+                      default_logging_keys(:plugin => plugin.class.config_name, :exception => close_exception.message,
+                                           :backtrace => close_exception.backtrace))
+      end
       retry
-    ensure
-      plugin.do_close
     end
   end # def inputworker