logstash-core 5.4.3-java → 5.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18cd186c56a7177512bce816947c1bf23e6aaa29
-  data.tar.gz: bfd08b7dd4db8136614ccd74deb77bf90d7a1c65
+  metadata.gz: 4b98be9ab7006918b14eec0ad319a2faa11e2461
+  data.tar.gz: d94019aa11fa085d61ed700b8eab297975449648
 SHA512:
-  metadata.gz: 1d85248bae95148415ff465d6ee4ef154db59909a1713f4169a9877db4be46e636a4c8df63c5f3243788e0203612f72e4ae53e1a0d4dcf7f4b0214f6e573020f
-  data.tar.gz: f863dfd3f9a227bb02dfdf0ebe47624c0f1be6cfd9c5438b80ef4d319123485e8090e2ccf5d97ea956bcf9b36bb7ce2e6048c61a5924fdd169d8ad14713b84e3
+  metadata.gz: 3fe7898ed25893c3b47c06762dbf411613a29a07a00d15eddc496f25f6be3d7fe017a60d8e1e0016814be2755a6e3a81cdbc71958c6c56ec5c149cbce5d1be59
+  data.tar.gz: 4b06ab8cfbc1bdf914977a57a9dbf99f429c1b6df6da703fd6bb9a2a51b5fb7f1fb7343c364c66577c34a234769aaad5f2ab2c92d362b1ce25b1fc2ce5f7a3a0

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.4.3"
+LOGSTASH_CORE_VERSION = "5.5.0"

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+java_import 'org.logstash.instrument.reports.ThreadsReport'
 
 class HotThreadsReport
   STRING_SEPARATOR_LENGTH = 80.freeze
@@ -7,8 +8,7 @@ class HotThreadsReport
   def initialize(cmd, options)
     @cmd = cmd
     filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-    jr_dump = JRMonitor.threads.generate(filter)
-    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => ThreadsReport.generate(filter)))
   end
 
   def to_s

data/lib/logstash/api/commands/node.rb

@@ -40,7 +40,6 @@ module LogStash
           {
             :pid =>  ManagementFactory.getRuntimeMXBean().getName().split("@").first.to_i,
             :version => java.lang.System.getProperty("java.version"),
-            :vm_name => java.lang.System.getProperty("java.vm.name"),
             :vm_version => java.lang.System.getProperty("java.version"),
             :vm_vendor => java.lang.System.getProperty("java.vendor"),
             :vm_name => java.lang.System.getProperty("java.vm.name"),

data/lib/logstash/api/commands/stats.rb

@@ -54,7 +54,6 @@ module LogStash
         def memory
           memory = service.get_shallow(:jvm, :memory)
           {
-            :heap_used_in_bytes => memory[:heap][:used_in_bytes],
             :heap_used_percent => memory[:heap][:used_percent],
             :heap_committed_in_bytes => memory[:heap][:committed_in_bytes],
             :heap_max_in_bytes => memory[:heap][:max_in_bytes],

data/lib/logstash/config/mixin.rb

@@ -6,7 +6,6 @@ require "logstash/util/password"
 require "logstash/util/safe_uri"
 require "logstash/version"
 require "logstash/environment"
-require "logstash/util/environment_variables"
 require "logstash/util/plugin_version"
 require "filesize"
 
@@ -34,37 +33,21 @@ LogStash::Environment.load_locale!
 # }
 #
 module LogStash::Config::Mixin
+  
+  include LogStash::Util::EnvironmentVariables
+  
   attr_accessor :config
   attr_accessor :original_params
 
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
-
-  ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
-
+  
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
     # Add the DSL methods to the 'base' given.
     base.extend(LogStash::Config::Mixin::DSL)
   end
-
-  # Recursive method to replace environment variable references in parameters
-  def deep_replace(value)
-    if (value.is_a?(Hash))
-      value.each do |valueHashKey, valueHashValue|
-        value[valueHashKey.to_s] = deep_replace(valueHashValue)
-      end
-    else
-      if (value.is_a?(Array))
-        value.each_index do | valueArrayIndex|
-          value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
-        end
-      else
-        return replace_env_placeholders(value)
-      end
-    end
-  end
-
+  
   def config_init(params)
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
@@ -159,27 +142,6 @@ module LogStash::Config::Mixin
     @config = params
   end # def config_init
 
-  # Replace all environment variable references in 'value' param by environment variable value and return updated value
-  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
-  def replace_env_placeholders(value)
-    return value unless value.is_a?(String)
-
-    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
-      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
-      # the call, so this should be thread-safe.
-      #
-      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
-      name = Regexp.last_match(:name)
-      default = Regexp.last_match(:default)
-
-      replacement = ENV.fetch(name, default)
-      if replacement.nil?
-        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
-      end
-      replacement
-    end
-  end # def replace_env_placeholders
-
   module DSL
 
     include LogStash::Util::EnvironmentVariables

data/lib/logstash/config/modules_common.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require "logstash/elasticsearch_client"
+require "logstash/modules/importer"
+require "logstash/errors"
+
+module LogStash module Config
+  class ModulesCommon # extracted here for bwc with 5.x
+    include LogStash::Util::Loggable
+
+    def self.pipeline_configs(settings)
+      pipelines = []
+      plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
+
+      modules_array = settings.get("modules.cli").empty? ? settings.get("modules") : settings.get("modules.cli")
+      if modules_array.empty?
+        # no specifed modules
+        return pipelines
+      end
+      logger.debug("Specified modules", :modules_array => modules_array.to_s)
+
+      module_names = modules_array.collect {|module_hash| module_hash["name"]}
+      if module_names.length > module_names.uniq.length
+        duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
+      end
+
+      available_module_names = plugin_modules.map(&:module_name)
+      specified_and_available_names = module_names & available_module_names
+
+      if (specified_and_available_names).empty?
+        i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names}
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", i18n_opts)
+      end
+
+      specified_and_available_names.each do |module_name|
+        connect_fail_args = {}
+        begin
+          module_hash = modules_array.find {|m| m["name"] == module_name}
+          current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
+
+          alt_name = "module-#{module_name}"
+          pipeline_id = alt_name
+
+          current_module.with_settings(module_hash)
+          esclient = LogStash::ElasticsearchClient.build(module_hash)
+          config_test = settings.get("config.test_and_exit")
+          if esclient.can_connect? || config_test
+            if !config_test
+              current_module.import(LogStash::Modules::Importer.new(esclient))
+            end
+
+            config_string = current_module.config_string
+
+            pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => settings}
+          else
+            connect_fail_args[:module_name] = module_name
+            connect_fail_args[:hosts] = esclient.host_settings
+          end
+        rescue => e
+          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.parse-failed", :error => e.message)
+        end
+
+        if !connect_fail_args.empty?
+          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", connect_fail_args)
+        end
+      end
+      pipelines
+    end
+  end
+end end

data/lib/logstash/elasticsearch_client.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+module LogStash class ElasticsearchClient
+  include LogStash::Util::Loggable
+
+  class Response
+    # duplicated here from Elasticsearch::Transport::Transport::Response
+    # to create a normalised response across different client IMPL
+    attr_reader :status, :body, :headers
+    def initialize(status, body, headers={})
+      @status, @body, @headers = status, body, headers
+      @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
+    end
+  end
+
+  def self.build(settings)
+    new(RubyClient.new(settings, logger))
+  end
+
+  class RubyClient
+    def initialize(settings, logger)
+      @settings = settings
+      @logger = logger
+      @client_args = client_args
+      @client = Elasticsearch::Client.new(@client_args)
+    end
+
+    def can_connect?
+      begin
+        head(SecureRandom.hex(32).prepend('_'))
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        true
+      rescue Manticore::SocketException
+        false
+      end
+    end
+
+    def host_settings
+      @client_args[:hosts]
+    end
+
+    def delete(path)
+      begin
+        normalize_response(@client.perform_request('DELETE', path, {}, nil))
+      rescue Exception => e
+        if is_404_error?(e)
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    def put(path, content)
+      normalize_response(@client.perform_request('PUT', path, {}, content))
+    end
+
+    def head(path)
+      begin
+        normalize_response(@client.perform_request('HEAD', path, {}, nil))
+      rescue Exception => e
+        if is_404_error?(e)
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    private
+
+    def is_404_error?(error)
+      error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i
+    end
+
+    def normalize_response(response)
+      Response.new(response.status, response.body, response.headers)
+    end
+
+    def client_args
+      {
+        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
+        :hosts => [*unpack_hosts],
+        # :logger => @logger, # silence the client logging
+      }
+    end
+
+    def unpack_hosts
+      @settings.fetch("var.output.elasticsearch.hosts", "localhost:9200").split(',').map(&:strip)
+    end
+  end
+
+  def initialize(client)
+    @client = client
+  end
+
+  def delete(path)
+    @client.delete(path)
+  end
+
+  def put(path, content)
+    @client.put(path, content)
+  end
+
+  def head(path)
+    @client.head(path)
+  end
+
+  def can_connect?
+    @client.can_connect?
+  end
+
+  def host_settings
+    @client.host_settings
+  end
+end end # class LogStash::ModulesImporter

data/lib/logstash/environment.rb

@@ -20,6 +20,8 @@ module LogStash
     Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
     Setting::NullableString.new("config.string", nil, false),
+                    Setting.new("modules.cli", Array, []),
+                    Setting.new("modules", Array, []),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
@@ -49,6 +51,7 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::Boolean.new("dead_letter_queue.enable", false),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
             Setting::TimeValue.new("slowlog.threshold.info", "-1"),
             Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
@@ -58,13 +61,21 @@ module LogStash
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
-  
+  # Compute the default dead_letter_queue path based on `path.data`
+  default_dlq_file_path = ::File.join(SETTINGS.get("path.data"), "dead_letter_queue")
+  SETTINGS.register Setting::WritableDirectory.new("path.dead_letter_queue", default_dlq_file_path)
+
   SETTINGS.on_post_process do |settings|
     # If the data path is overridden but the queue path isn't recompute the queue path
     # We need to do this at this stage because of the weird execution order
     # our monkey-patched Clamp follows
-    if settings.set?("path.data") && !settings.set?("path.queue")
-      settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+    if settings.set?("path.data")
+      if !settings.set?("path.queue")
+        settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+      end
+      if !settings.set?("path.dead_letter_queue")
+        settings.set_value("path.dead_letter_queue", ::File.join(settings.get("path.data"), "dead_letter_queue"))
+      end
     end
   end
 

data/lib/logstash/errors.rb

@@ -9,4 +9,5 @@ module LogStash
 
   class Bug < Error; end
   class ThisMethodWasRemoved < Bug; end
+  class ConfigLoadingError < Error; end
 end

data/lib/logstash/execution_context.rb

@@ -1,10 +1,18 @@
 # encoding: utf-8
+require "logstash/util/dead_letter_queue_manager"
 module LogStash
   class ExecutionContext
-    attr_reader :pipeline_id
+    attr_reader :pipeline, :dlq_writer
 
-    def initialize(pipeline_id)
-      @pipeline_id = pipeline_id
+    def initialize(pipeline, plugin_id, plugin_type, dlq_writer)
+      @pipeline = pipeline
+      @plugin_id = plugin_id
+      @plugin_type = plugin_type
+      @dlq_writer = LogStash::Util::PluginDeadLetterQueueWriter.new(dlq_writer, @plugin_id, @plugin_type)
+    end
+
+    def pipeline_id
+      @pipeline.pipeline_id
     end
   end
 end

data/lib/logstash/inputs/base.rb

@@ -105,6 +105,8 @@ class LogStash::Inputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/instrument/global_metrics.rb

@@ -0,0 +1,13 @@
+class GlobalMetrics
+  class Stats(metric)
+    @metric = metric
+  end
+
+  def initialize(metric)
+    @metric = metric
+
+    @pipeline_reloads = metric.namespace([:stats, :pipelines])
+  end
+
+
+end

data/lib/logstash/instrument/metric_type/mean.rb

@@ -4,7 +4,12 @@ require "concurrent"
 
 module LogStash module Instrument module MetricType
   class Mean < Base
+    include ::LogStash::Util::Loggable
+
     def initialize(namespaces, key)
+      logger.warn("Deprecated 'mean' metric type used! This will be removed in Logstash 6.0!",
+                   :namespaces => namespaces,
+                   :key => key)
       super(namespaces, key)
 
       @counter = Concurrent::AtomicFixnum.new

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -2,18 +2,19 @@
 require "logstash/instrument/periodic_poller/base"
 require "logstash/instrument/periodic_poller/load_average"
 require "logstash/environment"
-require "jrmonitor"
 require "set"
 
+java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'java.lang.management.ManagementFactory'
 java_import 'java.lang.management.OperatingSystemMXBean'
 java_import 'java.lang.management.GarbageCollectorMXBean'
 java_import 'java.lang.management.RuntimeMXBean'
-java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'javax.management.MBeanServer'
 java_import 'javax.management.ObjectName'
 java_import 'javax.management.AttributeList'
 java_import 'javax.naming.directory.Attribute'
+java_import 'org.logstash.instrument.reports.MemoryReport'
+java_import 'org.logstash.instrument.reports.ProcessReport'
 
 
 module LogStash module Instrument module PeriodicPoller
@@ -50,7 +51,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = JRMonitor.memory.generate
+      raw = MemoryReport.generate
       collect_jvm_metrics(raw)
       collect_pools_metrics(raw)
       collect_threads_metrics
@@ -81,11 +82,10 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect_process_metrics
-      process_metrics = JRMonitor.process.generate
+      process_metrics = ProcessReport.generate
 
       path = [:jvm, :process]
 
-
       open_fds = process_metrics["open_file_descriptors"]
       if @peak_open_fds.nil? || open_fds > @peak_open_fds
         @peak_open_fds = open_fds