logstash-core 5.5.3-java → 5.6.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35460f220b8242afa8c61bcd9f1ed826cd67d94b
-  data.tar.gz: 06f3bf3f2408e650320fb5a01e7dd124112411ce
+  metadata.gz: c0f19e6dd83b6582139637f263865a2249965e84
+  data.tar.gz: d80ae0275d1526e640468e236b7b809fd90f9577
 SHA512:
-  metadata.gz: a0e8fb932fac79ae5e867f337a0ad1e6c3e305128eb14ee9270b3580c7086b7c4dab3aa0466388a6b5d5c546266d18546d13c8c2ad9a59d34272e7a7f292d603
-  data.tar.gz: 1c7022d6dcf395d708fd90cd71a0bb680af59bcbc894b1fde3b7dee5ccdd03fdefd55ca88760e55e6ff8983fdd10017c63752a4e456d7a6d208fa70e0f48e250
+  metadata.gz: cbdbb0a428571a5409bdadb6cb100f2b494d513a2a8a6b0b8d40a90bd3f046956a5feeee71f64eea7ea0f5a1f63584e599e9cc8325839baa541e7309de3c1def
+  data.tar.gz: d1c50059807f9a8fc6b87a65d713c659261356106a7097523561688e86fb9d592c069a35ae89fda59a66d0ce082f91030811296d8ac779e9557ead6c64821093

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.5.3"
+LOGSTASH_CORE_VERSION = "5.6.0"

data/lib/logstash/api/commands/node.rb

@@ -20,8 +20,8 @@ module LogStash
         def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
           stats = extract_metrics(
             [:stats, :pipelines, pipeline_id, :config],
-            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
-          )
+            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval, :dead_letter_queue_enabled, :dead_letter_queue_path
+          ).reject{|_, v|v.nil?}
           stats.merge(:id => pipeline_id)
         end
 

data/lib/logstash/api/commands/stats.rb

@@ -107,8 +107,8 @@ module LogStash
               },
               :reloads => stats[:reloads],
               :queue => stats[:queue]
-            }
-          end
+            }.merge(stats[:dlq] ? {:dead_letter_queue => stats[:dlq]} : {})
+            end
         end # module PluginsStats
       end
     end

data/lib/logstash/config/config_ast.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'logstash/errors'
+require "logstash/config/string_escape"
 require "treetop"
 
 class Treetop::Runtime::SyntaxNode
@@ -9,6 +10,18 @@ class Treetop::Runtime::SyntaxNode
     return elements.collect(&:compile).reject(&:empty?).join("")
   end
 
+  def get_meta(key)
+    @ast_metadata ||= {}
+    return @ast_metadata[key] if @ast_metadata[key]
+    return self.parent.get_meta(key) if self.parent
+    nil
+  end
+
+  def set_meta(key, value)
+    @ast_metadata ||= {}
+    @ast_metadata[key] = value
+  end
+
   # Traverse the syntax tree recursively.
   # The order should respect the order of the configuration file as it is read
   # and written by humans (and the order in which it is parsed).
@@ -59,6 +72,7 @@ end
 
 
 module LogStash; module Config; module AST
+  PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
 
   def self.deferred_conditionals=(val)
     @deferred_conditionals = val
@@ -91,6 +105,11 @@ module LogStash; module Config; module AST
   end
 
   class Config < Node
+    def process_escape_sequences=(val)
+      set_meta(PROCESS_ESCAPE_SEQUENCES, val)
+    end
+
+
     def compile
       LogStash::Config::AST.deferred_conditionals = []
       LogStash::Config::AST.deferred_conditionals_index = 0
@@ -333,7 +352,11 @@ module LogStash; module Config; module AST
   end
   class String < Value
     def compile
-      return Unicode.wrap(text_value[1...-1])
+      if get_meta(PROCESS_ESCAPE_SEQUENCES)
+        Unicode.wrap(LogStash::Config::StringEscape.process_escapes(text_value[1...-1]))
+      else
+        Unicode.wrap(text_value[1...-1])
+      end
     end
   end
   class RegExp < Value

data/lib/logstash/config/modules_common.rb

@@ -1,18 +1,33 @@
 # encoding: utf-8
 require "logstash/util/loggable"
 require "logstash/elasticsearch_client"
-require "logstash/modules/importer"
+require "logstash/modules/kibana_client"
+require "logstash/modules/elasticsearch_importer"
+require "logstash/modules/kibana_importer"
+require "logstash/modules/settings_merger"
 require "logstash/errors"
 
 module LogStash module Config
   class ModulesCommon # extracted here for bwc with 5.x
     include LogStash::Util::Loggable
 
+    MODULES_MAX_PIPELINES = 1
+
     def self.pipeline_configs(settings)
       pipelines = []
       plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
 
-      modules_array = settings.get("modules.cli").empty? ? settings.get("modules") : settings.get("modules.cli")
+      cli_settings = settings.get("modules.cli")
+      yml_settings = settings.get("modules")
+
+      modules_array = if !(cli_settings.empty? && yml_settings.empty?)
+            LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
+          elsif cli_settings.empty?
+             yml_settings
+          else
+            cli_settings
+          end
+
       if modules_array.empty?
         # no specifed modules
         return pipelines
@@ -20,6 +35,11 @@ module LogStash module Config
       logger.debug("Specified modules", :modules_array => modules_array.to_s)
 
       module_names = modules_array.collect {|module_hash| module_hash["name"]}
+      if module_names.size > MODULES_MAX_PIPELINES
+        error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', '))
+        raise LogStash::ConfigLoadingError, error_message
+      end
+
       if module_names.length > module_names.uniq.length
         duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
         raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
@@ -36,29 +56,41 @@ module LogStash module Config
       specified_and_available_names.each do |module_name|
         connect_fail_args = {}
         begin
+          module_settings = settings.clone
+
           module_hash = modules_array.find {|m| m["name"] == module_name}
           current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
 
           alt_name = "module-#{module_name}"
           pipeline_id = alt_name
-
+          module_settings.set("pipeline.id", pipeline_id)
           current_module.with_settings(module_hash)
-          esclient = LogStash::ElasticsearchClient.build(module_hash)
           config_test = settings.get("config.test_and_exit")
-          if esclient.can_connect? || config_test
-            if !config_test
-              current_module.import(LogStash::Modules::Importer.new(esclient))
+          modul_setup = settings.get("modules_setup")
+          # Only import data if it's not a config test and --setup is true
+          if !config_test && modul_setup
+            esclient = LogStash::ElasticsearchClient.build(module_hash)
+            kbnclient = LogStash::Modules::KibanaClient.new(module_hash)
+            esconnected = esclient.can_connect?
+            kbnconnected = kbnclient.can_connect?
+            if esconnected && kbnconnected
+              current_module.add_kibana_version(kbnclient.version_parts)
+              current_module.import(
+                  LogStash::Modules::ElasticsearchImporter.new(esclient),
+                  LogStash::Modules::KibanaImporter.new(kbnclient)
+                )
+            else
+              connect_fail_args[:module_name] = module_name
+              connect_fail_args[:elasticsearch_hosts] = esclient.host_settings
+              connect_fail_args[:kibana_hosts] = kbnclient.host_settings
             end
-
-            config_string = current_module.config_string
-
-            pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => settings}
-          else
-            connect_fail_args[:module_name] = module_name
-            connect_fail_args[:hosts] = esclient.host_settings
           end
+          config_string = current_module.config_string
+          pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => module_settings}
         rescue => e
-          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.parse-failed", :error => e.message)
+          new_error = LogStash::ConfigLoadingError.new(I18n.t("logstash.modules.configuration.parse-failed", :error => e.message))
+          new_error.set_backtrace(e.backtrace)
+          raise new_error
         end
 
         if !connect_fail_args.empty?

data/lib/logstash/config/source/modules.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "logstash/config/source/base"
+require "logstash/config/modules_common"
+require "logstash/config/pipeline_config"
+require "logstash/util/loggable"
+require "logstash/errors"
+
+module LogStash module Config module Source
+  class Modules < Base
+    include LogStash::Util::Loggable
+    def pipeline_configs
+      if config_conflict? # double check
+        raise ConfigurationError, @conflict_messages.join(", ")
+      end
+
+      pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
+      pipelines.map do |hash|
+        PipelineConfig.new(self, hash["pipeline_id"].to_sym,
+          org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]),
+          hash["settings"])
+      end
+    end
+
+    def match?
+      # see basic settings predicates and getters defined in the base class
+      (modules_cli? || modules?) && !(config_string? || config_path?) && !automatic_reload_with_modules?
+    end
+
+    def config_conflict?
+      @conflict_messages.clear
+      # Make note that if modules are configured in both cli and logstash.yml that cli module
+      # settings will overwrite the logstash.yml modules settings
+      if modules_cli? && modules?
+        logger.info(I18n.t("logstash.runner.cli-module-override"))
+      end
+
+      if automatic_reload_with_modules?
+        @conflict_messages << I18n.t("logstash.runner.reload-with-modules")
+      end
+
+      # Check if config (-f or -e) and modules are configured
+      if (modules_cli? || modules?) && (config_string? || config_path?)
+        @conflict_messages << I18n.t("logstash.runner.config-module-exclusive")
+      end
+
+      @conflict_messages.any?
+    end
+
+    private
+
+    def automatic_reload_with_modules?
+      (modules_cli? || modules?) && config_reload_automatic?
+    end
+  end
+end end end

data/lib/logstash/config/string_escape.rb

@@ -0,0 +1,27 @@
+
+
+module LogStash; module Config; module StringEscape
+  class << self
+    def process_escapes(input)
+      input.gsub(/\\./) do |value|
+        process(value)
+      end
+    end
+
+    private
+    def process(value)
+      case value[1]
+      when '"', "'", "\\"
+        value[1]
+      when "n"
+        "\n"
+      when "r"
+        "\r"
+      when "t"
+        "\t"
+      else
+        value
+      end
+    end
+  end
+end end end

data/lib/logstash/elasticsearch_client.rb

@@ -26,6 +26,24 @@ module LogStash class ElasticsearchClient
       @settings = settings
       @logger = logger
       @client_args = client_args
+
+      ssl_options = {}
+
+      if @settings["var.elasticsearch.ssl.enabled"] == "true"
+        ssl_options[:verify] = @settings.fetch("var.elasticsearch.ssl.verification_mode", true)
+        ssl_options[:ca_file] = @settings.fetch("var.elasticsearch.ssl.certificate_authority", nil)
+        ssl_options[:client_cert] = @settings.fetch("var.elasticsearch.ssl.certificate", nil)
+        ssl_options[:client_key] = @settings.fetch("var.elasticsearch.ssl.key", nil)
+      end
+
+      @client_args[:ssl] = ssl_options
+
+      username = @settings["var.elasticsearch.username"]
+      password = @settings["var.elasticsearch.password"]
+      if username
+        @client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp } }
+      end
+
       @client = Elasticsearch::Client.new(@client_args)
     end
 
@@ -90,7 +108,11 @@ module LogStash class ElasticsearchClient
     end
 
     def unpack_hosts
-      @settings.fetch("var.output.elasticsearch.hosts", "localhost:9200").split(',').map(&:strip)
+      setting = @settings.fetch("var.elasticsearch.hosts", "localhost:9200")
+      if setting.is_a?(String)
+        return setting.split(',').map(&:strip)
+      end
+      setting
     end
   end
 
@@ -117,4 +139,4 @@ module LogStash class ElasticsearchClient
   def host_settings
     @client.host_settings
   end
-end end # class LogStash::ModulesImporter
+end end

data/lib/logstash/environment.rb

@@ -22,8 +22,10 @@ module LogStash
     Setting::NullableString.new("config.string", nil, false),
                     Setting.new("modules.cli", Array, []),
                     Setting.new("modules", Array, []),
+           Setting::Boolean.new("modules_setup", false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
+           Setting::Boolean.new("config.support_escapes", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),

data/lib/logstash/filter_delegator.rb

@@ -26,6 +26,9 @@ module LogStash
       @filter.execution_context = execution_context
 
       @metric_events = namespaced_metric.namespace(:events)
+      @metric_events_in = @metric_events.counter(:in)
+      @metric_events_out = @metric_events.counter(:out)
+      @metric_events_time = @metric_events.counter(:duration_in_millis)
       namespaced_metric.gauge(:name, config_name)
 
       # Not all the filters will do bufferings
@@ -37,19 +40,19 @@ module LogStash
     end
 
     def multi_filter(events)
-      @metric_events.increment(:in, events.size)
+      @metric_events_in.increment(events.size)
 
-      clock = @metric_events.time(:duration_in_millis)
+      start_time = java.lang.System.current_time_millis
       new_events = @filter.multi_filter(events)
-      clock.stop
+      @metric_events_time.increment(java.lang.System.current_time_millis - start_time)
 
       # There is no guarantee in the context of filter
       # that EVENTS_INT == EVENTS_OUT, see the aggregates and
       # the split filter
       c = new_events.count { |event| !event.cancelled? }
-      @metric_events.increment(:out, c) if c > 0
 
-      return new_events
+      @metric_events_out.increment(c) if c > 0
+      new_events
     end
 
     private
@@ -61,7 +64,7 @@ module LogStash
 
         # Filter plugins that does buffering or spooling of events like the
         # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
-        @metric_events.increment(:out, new_events.size) if new_events && new_events.size > 0
+        @metric_events_out.increment(new_events.size) if new_events && new_events.size > 0
         new_events
       end
     end

data/lib/logstash/instrument/collector.rb

@@ -33,11 +33,7 @@ module LogStash module Instrument
     #
     def push(namespaces_path, key, type, *metric_type_params)
       begin
-        metric = @metric_store.fetch_or_store(namespaces_path, key) do
-          LogStash::Instrument::MetricType.create(type, namespaces_path, key)
-        end
-
-        metric.execute(*metric_type_params)
+        get(namespaces_path, key, type).execute(*metric_type_params)
       rescue MetricStore::NamespacesExpectedError => e
         logger.error("Collector: Cannot record metric", :exception => e)
       rescue NameError => e
@@ -51,6 +47,12 @@ module LogStash module Instrument
       end
     end
 
+    def get(namespaces_path, key, type)
+      @metric_store.fetch_or_store(namespaces_path, key) do
+        LogStash::Instrument::MetricType.create(type, namespaces_path, key)
+      end
+    end
+
     # Snapshot the current Metric Store and return it immediately,
     # This is useful if you want to get access to the current metric store without
     # waiting for a periodic call.

data/lib/logstash/instrument/metric_store.rb

@@ -41,26 +41,26 @@ module LogStash module Instrument
     # @param [Symbol] The metric key
     # @return [Object] Return the new_value of the retrieve object in the tree
     def fetch_or_store(namespaces, key, default_value = nil)
-      provided_value =  block_given? ? yield(key) : default_value
 
       # We first check in the `@fast_lookup` store to see if we have already see that metrics before,
       # This give us a `o(1)` access, which is faster than searching through the structured
       # data store (Which is a `o(n)` operation where `n` is the number of element in the namespace and
-      # the value of the key). If the metric is already present in the `@fast_lookup`, the call to
-      # `#put_if_absent` will return the value. This value is send back directly to the caller.
+      # the value of the key). If the metric is already present in the `@fast_lookup`, then that value is sent
+      # back directly to the caller.
       #
-      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
-      # `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
+      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and we assume that we don't
       # have it in the `@metric_store` for structured search so we add it there too.
-      if found_value = @fast_lookup.put_if_absent(namespaces.dup << key, provided_value)
-        return found_value
-      else
+
+      value = @fast_lookup.get(namespaces.dup << key)
+      if value.nil?
+        value = block_given? ? yield(key) : default_value
+        @fast_lookup.put(namespaces.dup << key, value)
         @structured_lookup_mutex.synchronize do
-          # If we cannot find the value this mean we need to save it in the store.
-          fetch_or_store_namespaces(namespaces).fetch_or_store(key, provided_value)
+            # If we cannot find the value this mean we need to save it in the store.
+          fetch_or_store_namespaces(namespaces).fetch_or_store(key, value)
         end
-        return provided_value
       end
+      return value;
     end
 
     # This method allow to retrieve values for a specific path,