logstash-core 5.3.3-java → 5.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44014812c539884509e0aa9cabc90cd910aad106
-  data.tar.gz: 85abbce0cbd3d42b9d4e50132870e31865f5c4c4
+  metadata.gz: b75f61585cb54f56f39ba3df892c501842fe618b
+  data.tar.gz: 1a2c44621cdc55841d2d6a9f03e869cc4b58af81
 SHA512:
-  metadata.gz: df7f86070224396d2f6af942e6dbc066f744a6340701f97f419eaf2da6f0184d59c7109fe8d2546eb0823f04cfae0b7748b4ddb294813d13bd77ca780ed4af8b
-  data.tar.gz: 207373ff34b7ec956c775ad9509e0abb9ed133c6c4d3e276827afb81974f2f2e6e4368c17ec66ceac4a047ebd22bd9dd6547f45fe7c09d17120a71b126ca3ac2
+  metadata.gz: 62549f54412120fafee19c383bd22885ea7e53fc091b65291aeda0d2b42b2ee885e36776385ff88a5f8996c326d8607b44f6a8430315e4940e4c92f1c446a7f8
+  data.tar.gz: be4c9ab4a85b0e86c33d74a0013d9cd49f44cb45aeba386b92d3489e330ce4bcb41568fdc0d8efa0dca19907d505a15d5c6e34acbba87592a32e75c2acc6a717

data/gemspec_jars.rb

@@ -6,3 +6,5 @@ gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
 gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.4"
 gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.7.4"
+gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.7.4"
+gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.7.4"

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.3.3"
+LOGSTASH_CORE_VERSION = "5.4.0"

data/lib/logstash-core_jars.rb

@@ -3,16 +3,20 @@ begin
   require 'jar_dependencies'
 rescue LoadError
   require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
+  require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.7.4/jackson-module-afterburner-2.7.4.jar'
   require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
   require 'com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar'
   require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar'
+  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.7.4/jackson-dataformat-cbor-2.7.4.jar'
   require 'com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar'
 end
 
 if defined? Jars
   require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
+  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.4' )
   require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.4' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
+  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.7.4' )
   require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.4' )
 end

data/lib/logstash/agent.rb

@@ -61,7 +61,7 @@ class LogStash::Agent
   end
 
   def execute
-    @thread = Thread.current # this var is implicilty used by Stud.stop?
+    @thread = Thread.current # this var is implicitly used by Stud.stop?
     @logger.debug("starting agent")
 
     start_pipelines
@@ -75,8 +75,8 @@ class LogStash::Agent
       Stud.interval(@reload_interval) { reload_state! }
     else
       while !Stud.stop?
-        if clean_state? || running_pipelines?
-          sleep 0.5
+        if clean_state? || running_user_defined_pipelines?
+          sleep(0.5)
         else
           break
         end
@@ -189,6 +189,15 @@ class LogStash::Agent
     end
   end
 
+  def running_user_defined_pipelines?
+    @upgrade_mutex.synchronize do
+      @pipelines.select do |pipeline_id, _|
+        pipeline = @pipelines[pipeline_id]
+        pipeline.running? && !pipeline.system?
+      end.any?
+    end
+  end
+
   def close_pipeline(id)
     pipeline = @pipelines[id]
     if pipeline
@@ -298,7 +307,7 @@ class LogStash::Agent
     end
 
     # check if this pipeline is not reloadable. it should not happen as per the check below
-    # but keep it here as a safety net if a reloadable pipeline was releoaded with a non reloadable pipeline
+    # but keep it here as a safety net if a reloadable pipeline was reloaded with a non reloadable pipeline
     if !old_pipeline.reloadable?
       @logger.error("pipeline is not reloadable", :pipeline => id)
       return
@@ -370,7 +379,7 @@ class LogStash::Agent
       return
     end
 
-    # pipeline started successfuly, update reload success metrics
+    # pipeline started successfully, update reload success metrics
     @instance_reload_metric.increment(:successes)
     @pipeline_reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
       n.increment(:successes)
@@ -424,7 +433,7 @@ class LogStash::Agent
     @pipelines.each do |id, pipeline|
       start_pipeline(id)
       pipeline.collect_stats
-      # no reloads yet, initalize all the reload metrics
+      # no reloads yet, initialize all the reload metrics
       init_pipeline_reload_metrics(id)
     end
   end

data/lib/logstash/api/modules/base.rb

@@ -31,7 +31,7 @@ module LogStash
         end
 
         not_found do
-          # We cannot raise here because it wont be catched by the `error` handler.
+          # We cannot raise here because it won't be catched by the `error` handler.
           # So we manually create a new instance of NotFound and just pass it down.
           respond_with(NotFoundError.new)
         end

data/lib/logstash/api/rack_app.rb

@@ -80,7 +80,7 @@ module LogStash
           # Custom logger object. Rack CommonLogger does not work with cabin
           use ApiLogger, logger
 
-          # In test env we want errors to propogate up the chain
+          # In test env we want errors to propagate up the chain
           # so we get easy to understand test failures.
           # In production / dev we don't want a bad API endpoint
           # to crash the process

data/lib/logstash/config/config_ast.rb

@@ -60,20 +60,20 @@ end
 
 module LogStash; module Config; module AST
 
-  def self.defered_conditionals=(val)
-    @defered_conditionals = val
+  def self.deferred_conditionals=(val)
+    @deferred_conditionals = val
   end
 
-  def self.defered_conditionals
-    @defered_conditionals
+  def self.deferred_conditionals
+    @deferred_conditionals
   end
 
-  def self.defered_conditionals_index
-    @defered_conditionals_index
+  def self.deferred_conditionals_index
+    @deferred_conditionals_index
   end
 
-  def self.defered_conditionals_index=(val)
-    @defered_conditionals_index = val
+  def self.deferred_conditionals_index=(val)
+    @deferred_conditionals_index = val
   end
 
   def self.plugin_instance_index
@@ -92,8 +92,8 @@ module LogStash; module Config; module AST
 
   class Config < Node
     def compile
-      LogStash::Config::AST.defered_conditionals = []
-      LogStash::Config::AST.defered_conditionals_index = 0
+      LogStash::Config::AST.deferred_conditionals = []
+      LogStash::Config::AST.deferred_conditionals_index = 0
       LogStash::Config::AST.plugin_instance_index = 0
       code = []
 
@@ -136,7 +136,7 @@ module LogStash; module Config; module AST
 
       code += definitions.join("\n").split("\n", -1).collect { |l| "  #{l}" }
 
-      code += LogStash::Config::AST.defered_conditionals
+      code += LogStash::Config::AST.deferred_conditionals
 
       return code.join("\n")
     end
@@ -402,7 +402,7 @@ module LogStash; module Config; module AST
       type = recursive_select_parent(PluginSection).first.plugin_type.text_value
 
       if type == "filter"
-        i = LogStash::Config::AST.defered_conditionals_index += 1
+        i = LogStash::Config::AST.deferred_conditionals_index += 1
         source = <<-CODE
           @generated_objects[:cond_func_#{i}] = lambda do |input_events|
             result = []
@@ -415,7 +415,7 @@ module LogStash; module Config; module AST
             result
           end
         CODE
-        LogStash::Config::AST.defered_conditionals << source
+        LogStash::Config::AST.deferred_conditionals << source
 
         <<-CODE
           events = @generated_objects[:cond_func_#{i}].call(events)

data/lib/logstash/config/mixin.rb

@@ -47,15 +47,32 @@ module LogStash::Config::Mixin
     base.extend(LogStash::Config::Mixin::DSL)
   end
 
+  # Recursive method to replace environment variable references in parameters
+  def deep_replace(value)
+    if (value.is_a?(Hash))
+      value.each do |valueHashKey, valueHashValue|
+        value[valueHashKey.to_s] = deep_replace(valueHashValue)
+      end
+    else
+      if (value.is_a?(Array))
+        value.each_index do | valueArrayIndex|
+          value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
+        end
+      else
+        return replace_env_placeholders(value)
+      end
+    end
+  end
+
   def config_init(params)
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
-    
+
     # Keep a copy of the original config params so that we can later
     # differentiate between explicit configuration and implicit (default)
     # configuration.
     original_params = params.clone
-    
+
     # store the plugin type, turns LogStash::Inputs::Base into 'input'
     @plugin_type = self.class.ancestors.find { |a| a.name =~ /::Base$/ }.config_name
 
@@ -88,7 +105,7 @@ module LogStash::Config::Mixin
       next if params.include?(name.to_s)
       if opts.include?(:default) and (name.is_a?(Symbol) or name.is_a?(String))
         # default values should be cloned if possible
-        # cloning prevents 
+        # cloning prevents
         case opts[:default]
           when FalseClass, TrueClass, NilClass, Numeric
             params[name.to_s] = opts[:default]
@@ -105,19 +122,7 @@ module LogStash::Config::Mixin
 
     # Resolve environment variables references
     params.each do |name, value|
-      if (value.is_a?(Hash))
-        value.each do |valueHashKey, valueHashValue|
-          value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
-        end
-      else
-        if (value.is_a?(Array))
-          value.each_index do |valueArrayIndex|
-            value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
-          end
-        else
-          params[name.to_s] = replace_env_placeholders(value)
-        end
-      end
+      params[name.to_s] = deep_replace(value)
     end
 
 
@@ -204,7 +209,7 @@ module LogStash::Config::Mixin
 
       name = name.to_s if name.is_a?(Symbol)
       @config[name] = opts  # ok if this is empty
-      
+
       if name.is_a?(String)
         define_method(name) { instance_variable_get("@#{name}") }
         define_method("#{name}=") { |v| instance_variable_set("@#{name}", v) }
@@ -343,7 +348,7 @@ module LogStash::Config::Mixin
                                :setting => config_key, :plugin => @plugin_name,
                                :type => @plugin_type))
           is_valid = false
-        end        
+        end
       end
 
       return is_valid
@@ -351,26 +356,26 @@ module LogStash::Config::Mixin
 
     def process_parameter_value(value, config_settings)
       config_val = config_settings[:validate]
-      
+
       if config_settings[:list]
         value = Array(value) # coerce scalars to lists
         # Empty lists are converted to nils
         return true, nil if value.empty?
-          
+
         validated_items = value.map {|v| validate_value(v, config_val)}
         is_valid = validated_items.all? {|sr| sr[0] }
         processed_value = validated_items.map {|sr| sr[1]}
       else
         is_valid, processed_value = validate_value(value, config_val)
       end
-      
+
       return [is_valid, processed_value]
     end
 
     def validate_check_parameter_values(params)
-      # Filter out parametrs that match regexp keys.
+      # Filter out parameters that match regexp keys.
       # These are defined in plugins like this:
-      #   config /foo.*/ => ... 
+      #   config /foo.*/ => ...
       all_params_valid = true
 
       params.each do |key, value|
@@ -378,10 +383,10 @@ module LogStash::Config::Mixin
           next unless (config_key.is_a?(Regexp) && key =~ config_key) \
                       || (config_key.is_a?(String) && key == config_key)
 
-          config_settings = @config[config_key]          
+          config_settings = @config[config_key]
 
           is_valid, processed_value = process_parameter_value(value, config_settings)
-          
+
           if is_valid
             # Accept coerced value if valid
             # Used for converting values in the config to proper objects.
@@ -393,7 +398,7 @@ module LogStash::Config::Mixin
                                  :value_type => config_settings[:validate],
                                  :note => processed_value))
           end
-          
+
           all_params_valid &&= is_valid
 
           break # done with this param key
@@ -433,7 +438,7 @@ module LogStash::Config::Mixin
         end
         result = value.first
       elsif validator.is_a?(Symbol)
-        # TODO(sissel): Factor this out into a coersion method?
+        # TODO(sissel): Factor this out into a coercion method?
         # TODO(sissel): Document this stuff.
         value = hash_or_array(value)
 
@@ -539,7 +544,7 @@ module LogStash::Config::Mixin
             if value.size > 1
               return false, "Expected uri (one value), got #{value.size} values?"
             end
-            
+
             result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
           when :path
             if value.size > 1 # Only 1 value wanted

data/lib/logstash/environment.rb

@@ -25,6 +25,7 @@ module LogStash
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
+            Setting::Boolean.new("pipeline.system", false),
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
    Setting::PositiveInteger.new("pipeline.output.workers", 1),
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
@@ -41,6 +42,7 @@ module LogStash
             Setting::PortRange.new("http.port", 9600..9700),
             Setting::String.new("http.environment", "production"),
             Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
+            Setting::Boolean.new("queue.drain", false),
             Setting::Bytes.new("queue.page_capacity", "250mb"),
             Setting::Bytes.new("queue.max_bytes", "1024mb"),
             Setting::Numeric.new("queue.max_events", 0), # 0 is unlimited
@@ -56,6 +58,15 @@ module LogStash
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
+  
+  SETTINGS.on_post_process do |settings|
+    # If the data path is overridden but the queue path isn't recompute the queue path
+    # We need to do this at this stage because of the weird execution order
+    # our monkey-patched Clamp follows
+    if settings.set?("path.data") && !settings.set?("path.queue")
+      settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+    end
+  end
 
   module Environment
     extend self

data/lib/logstash/event.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+require "logstash/json"
+require "jruby_event_ext"
+require "jruby_timestamp_ext"
+require "logstash/timestamp"
+require "logstash/string_interpolation"
+
+# transient pipeline events for normal in-flow signaling as opposed to
+# flow altering exceptions. for now having base classes is adequate and
+# in the future it might be necessary to refactor using like a BaseEvent
+# class to have a common interface for all pipeline events to support
+# eventual queueing persistence for example, TBD.
+module LogStash
+  class SignalEvent
+    def flush?; raise "abstract method"; end;
+    def shutdown?; raise "abstract method"; end;
+  end
+
+  class ShutdownEvent < SignalEvent
+    def flush?; false; end;
+    def shutdown?; true; end;
+  end
+
+  class FlushEvent < SignalEvent
+    def flush?; true; end;
+    def shutdown?; false; end;
+  end
+
+  class NoSignal < SignalEvent
+    def flush?; false; end;
+    def shutdown?; false; end;
+  end
+
+  FLUSH = FlushEvent.new
+  SHUTDOWN = ShutdownEvent.new
+  NO_SIGNAL = NoSignal.new
+
+  class Event
+    MSG_BRACKETS_METHOD_MISSING = "Direct event field references (i.e. event['field']) have been disabled in favor of using event get and set methods (e.g. event.get('field')). Please consult the Logstash 5.0 breaking changes documentation for more details.".freeze
+    MSG_BRACKETS_EQUALS_METHOD_MISSING = "Direct event field references (i.e. event['field'] = 'value') have been disabled in favor of using event get and set methods (e.g. event.set('field', 'value')). Please consult the Logstash 5.0 breaking changes documentation for more details.".freeze
+    RE_BRACKETS_METHOD = /^\[\]$/.freeze
+    RE_BRACKETS_EQUALS_METHOD = /^\[\]=$/.freeze
+
+    def method_missing(method_name, *arguments, &block)
+      if RE_BRACKETS_METHOD.match(method_name.to_s)
+        raise NoMethodError.new(MSG_BRACKETS_METHOD_MISSING)
+      end
+      if RE_BRACKETS_EQUALS_METHOD.match(method_name.to_s)
+        raise NoMethodError.new(MSG_BRACKETS_EQUALS_METHOD_MISSING)
+      end
+      super
+    end
+  end
+end

data/lib/logstash/event_dispatcher.rb

@@ -1,13 +1,13 @@
 # encoding: utf-8
 module LogStash
   class EventDispatcher
-    java_import "java.util.concurrent.CopyOnWriteArrayList"
+    java_import "java.util.concurrent.CopyOnWriteArraySet"
 
     attr_reader :emitter
 
     def initialize(emitter)
       @emitter = emitter
-      @listeners = CopyOnWriteArrayList.new
+      @listeners = CopyOnWriteArraySet.new
     end
 
     # This operation is slow because we use a CopyOnWriteArrayList