logstash-core 5.3.3-java → 5.4.0-java

data/lib/logstash/plugin.rb

@@ -8,7 +8,8 @@ require "securerandom"
 
 class LogStash::Plugin
   include LogStash::Util::Loggable
-  attr_accessor :params
+
+  attr_accessor :params, :execution_context
 
   NL = "\n"
 
@@ -122,6 +123,7 @@ class LogStash::Plugin
                          LogStash::Instrument::NamespacedNullMetric.new(@metric, :null)
                        end
   end
+
   # return the configured name of this plugin
   # @return [String] The name of the plugin defined by `config_name`
   def config_name

data/lib/logstash/plugins/hooks_registry.rb

@@ -6,17 +6,17 @@ module LogStash module Plugins
     java_import "java.util.concurrent.CopyOnWriteArrayList"
 
     def initialize
-      @registered_emmitters = ConcurrentHashMap.new
+      @registered_emitters = ConcurrentHashMap.new
       @registered_hooks = ConcurrentHashMap.new
     end
 
     def register_emitter(emitter_scope, dispatcher)
-      @registered_emmitters.put(emitter_scope, dispatcher)
+      @registered_emitters.put(emitter_scope, dispatcher)
       sync_hooks
     end
 
     def remove_emitter(emitter_scope)
-      @registered_emmitters.remove(emitter_scope)
+      @registered_emitters.remove(emitter_scope)
     end
 
     def register_hooks(emitter_scope, callback)
@@ -28,8 +28,8 @@ module LogStash module Plugins
       sync_hooks
     end
 
-    def emmitters_count
-      @registered_emmitters.size
+    def emitters_count
+      @registered_emitters.size
     end
 
     def hooks_count(emitter_scope = nil)
@@ -43,7 +43,7 @@ module LogStash module Plugins
 
     private
     def sync_hooks
-      @registered_emmitters.each do |emitter, dispatcher|
+      @registered_emitters.each do |emitter, dispatcher|
         listeners = @registered_hooks.get(emitter)
 
         unless listeners.nil?

data/lib/logstash/plugins/registry.rb

@@ -111,7 +111,7 @@ module LogStash module Plugins
 
     def load_available_plugins
       GemRegistry.logstash_plugins.each do |plugin_context|
-        # When a plugin has a HOOK_FILE defined, its the responsability of the plugin
+        # When a plugin has a HOOK_FILE defined, its the responsibility of the plugin
         # to register itself to the registry of available plugins.
         #
         # Legacy plugin will lazy register themselves
@@ -205,7 +205,7 @@ module LogStash module Plugins
     private
     # lookup a plugin by type and name in the existing LogStash module namespace
     # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
-    # @param type [String] plugin type, "input", "ouput", "filter"
+    # @param type [String] plugin type, "input", "output", "filter"
     # @param name [String] plugin name, ex.: "grok"
     # @return [Class] the plugin class or raises NameError
     # @raise NameError if plugin class does not exist or is invalid

data/lib/logstash/queue_factory.rb

@@ -16,18 +16,20 @@ module LogStash
       checkpoint_max_writes = settings.get("queue.checkpoint.writes")
       checkpoint_max_interval = settings.get("queue.checkpoint.interval")
 
+      queue_path = ::File.join(settings.get("path.queue"), settings.get("pipeline.id"))
+
       case queue_type
       when "memory_acked"
         # memory_acked is used in tests/specs
-        LogStash::Util::WrappedAckedQueue.create_memory_based("", queue_page_capacity, queue_max_events, queue_max_bytes)
-      when "memory"
-        # memory is the legacy and default setting
-        LogStash::Util::WrappedSynchronousQueue.new
+        FileUtils.mkdir_p(queue_path)
+        LogStash::Util::WrappedAckedQueue.create_memory_based(queue_path, queue_page_capacity, queue_max_events, queue_max_bytes)
       when "persisted"
         # persisted is the disk based acked queue
-        queue_path = ::File.join(settings.get("path.queue"), settings.get("pipeline.id"))
         FileUtils.mkdir_p(queue_path)
         LogStash::Util::WrappedAckedQueue.create_file_based(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
+      when "memory"
+        # memory is the legacy and default setting
+        LogStash::Util::WrappedSynchronousQueue.new
       else
         raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are: 'memory_acked', 'memory', 'persisted'"
       end

data/lib/logstash/runner.rb

@@ -20,6 +20,8 @@ require "logstash/settings"
 require "logstash/version"
 require "logstash/plugins/registry"
 
+java_import 'org.logstash.FileLockFactory'
+
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
   # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
@@ -184,6 +186,11 @@ class LogStash::Runner < Clamp::StrictCommand
   end
 
   def execute
+    # Only when execute is have the CLI options been added to the @settings
+    # We invoke post_process to apply extra logic to them.
+    # The post_process callbacks have been added in environment.rb
+    @settings.post_process
+    
     require "logstash/util"
     require "logstash/util/java_version"
     require "stud/task"
@@ -195,7 +202,7 @@ class LogStash::Runner < Clamp::StrictCommand
     java.lang.System.setProperty("ls.log.level", setting("log.level"))
     unless java.lang.System.getProperty("log4j.configurationFile")
       log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
-      LogStash::Logging::Logger::initialize("file:///" + log4j_config_location)
+      LogStash::Logging::Logger::initialize("file://" + log4j_config_location)
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
@@ -259,6 +266,9 @@ class LogStash::Runner < Clamp::StrictCommand
       end
     end
 
+    # lock path.data before starting the agent
+    @data_path_lock = FileLockFactory.getDefault().obtainLock(setting("path.data"), ".lock");
+
     @agent = create_agent(@settings)
 
     @agent.register_pipeline(@settings)
@@ -283,6 +293,9 @@ class LogStash::Runner < Clamp::StrictCommand
 
     agent_return
 
+  rescue org.logstash.LockException => e
+    logger.fatal(I18n.t("logstash.runner.locked-data-path", :path => setting("path.data")))
+    return 1
   rescue Clamp::UsageError => e
     $stderr.puts "ERROR: #{e.message}"
     show_short_help
@@ -299,6 +312,7 @@ class LogStash::Runner < Clamp::StrictCommand
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
     Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
     Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
+    FileLockFactory.getDefault().releaseLock(@data_path_lock) if @data_path_lock
     @log_fd.close if @log_fd
   end # def self.main
 

data/lib/logstash/settings.rb

@@ -108,6 +108,20 @@ module LogStash
     def from_yaml(yaml_path)
       settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
       self.merge(flatten_hash(settings), true)
+      self
+    end
+    
+    def post_process
+      if @post_process_callbacks
+        @post_process_callbacks.each do |callback|
+          callback.call(self)
+        end
+      end
+    end
+    
+    def on_post_process(&block)
+      @post_process_callbacks ||= []
+      @post_process_callbacks << block
     end
 
     def validate_all
@@ -232,7 +246,6 @@ module LogStash
           @default = default
         end
       end
-
       def set(value)
         coerced_value = coerce(value)
         validate(coerced_value)
@@ -520,4 +533,3 @@ module LogStash
 
   SETTINGS = Settings.new
 end
-

data/lib/logstash/string_interpolation.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module LogStash
+  module StringInterpolation
+    extend self
+
+    # clear the global compiled templates cache
+    def clear_cache
+      Java::OrgLogstash::StringInterpolation.get_instance.clear_cache;
+    end
+
+    # @return [Fixnum] the compiled templates cache size
+    def cache_size
+      Java::OrgLogstash::StringInterpolation.get_instance.cache_size;
+    end
+  end
+end
+

data/lib/logstash/timestamp.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+
+module LogStash
+  class TimestampParserError < StandardError; end
+
+  class Timestamp
+    include Comparable
+
+    # TODO (colin) implement in Java
+    def <=>(other)
+      self.time <=> other.time
+    end
+
+    # TODO (colin) implement in Java
+    def +(other)
+      self.time + other
+    end
+
+    # TODO (colin) implement in Java
+    def -(value)
+      self.time - (value.is_a?(Timestamp) ? value.time : value)
+    end
+
+  end
+end

data/lib/logstash/util.rb

@@ -141,7 +141,7 @@ module LogStash::Util
   end # def hash_merge_many
 
 
-  # nomalize method definition based on platform.
+  # normalize method definition based on platform.
   # normalize is used to convert an object create through
   # json deserialization from JrJackson in :raw mode to pure Ruby
   # to support these pure Ruby object monkey patches.

data/lib/logstash/util/prctl.rb

@@ -4,7 +4,7 @@ module LibC
   extend FFI::Library
   ffi_lib 'c'
 
-  # Ok so the 2nd arg isn't really a string... but whaatever
+  # Ok so the 2nd arg isn't really a string... but whatever
   attach_function :prctl, [:int, :string, :long, :long, :long], :int
 end
 

data/lib/logstash/util/retryable.rb

@@ -3,7 +3,7 @@ module LogStash
   module Retryable
     # execute retryable code block
     # @param [Hash] options retryable options
-    # @option options [Fixnum] :tries retries to perform, default 1, set to 0 for infite retries. 1 means that upon exception the block will be retried once
+    # @option options [Fixnum] :tries retries to perform, default 1, set to 0 for infinite retries. 1 means that upon exception the block will be retried once
     # @option options [Fixnum] :base_sleep seconds to sleep on first retry, default 1
     # @option options [Fixnum] :max_sleep max seconds to sleep upon exponential backoff, default 1
     # @option options [Exception] :rescue exception class list to retry on, defaults is Exception, which retries on any Exception.

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
-require "logstash-core-queue-jruby/logstash-core-queue-jruby"
+require "jruby_acked_queue_ext"
+require "jruby_acked_batch_ext"
 require "concurrent"
 # This is an adapted copy of the wrapped_synchronous_queue file
 # ideally this should be moved to Java/JRuby
@@ -62,7 +63,7 @@ module LogStash; module Util
     #
     # @param [Object] Object to add to the queue
     # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successfull if not it return false
+    # @return [Boolean] True if adding was successful if not it return false
     def offer(obj, timeout_ms)
       raise NotImplementedError.new("The offer method is not implemented. There is no non blocking write operation yet.")
     end
@@ -125,6 +126,10 @@ module LogStash; module Util
         @queue.close
       end
 
+      def empty?
+        @mutex.synchronize { @queue.is_fully_acked? }
+      end
+
       def set_batch_dimensions(batch_size, wait_for)
         @batch_size = batch_size
         @wait_for = wait_for
@@ -143,7 +148,6 @@ module LogStash; module Util
       def define_initial_metrics_values(namespaced_metric)
         namespaced_metric.report_time(:duration_in_millis, 0)
         namespaced_metric.increment(:filtered, 0)
-        namespaced_metric.increment(:in, 0)
         namespaced_metric.increment(:out, 0)
       end
 
@@ -157,16 +161,28 @@ module LogStash; module Util
         @inflight_batches.fetch(Thread.current, [])
       end
 
-      def take_batch
+      # create a new empty batch
+      # @return [ReadBatch] a new empty read batch
+      def new_batch
+        ReadBatch.new(@queue, @batch_size, @wait_for)
+      end
+
+      def read_batch
         if @queue.closed?
           raise QueueClosedError.new("Attempt to take a batch from a closed AckedQueue")
         end
+
+        batch = new_batch
+        @mutex.synchronize { batch.read_next }
+        start_metrics(batch)
+        batch
+      end
+
+      def start_metrics(batch)
         @mutex.synchronize do
-          batch = ReadBatch.new(@queue, @batch_size, @wait_for)
-          add_starting_metrics(batch)
+          # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
           start_clock
-          batch
         end
       end
 
@@ -177,21 +193,30 @@ module LogStash; module Util
       def close_batch(batch)
         @mutex.synchronize do
           batch.close
+
+          # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
-          stop_clock
+          stop_clock(batch)
         end
       end
 
       def start_clock
         @inflight_clocks[Thread.current] = [
-        @event_metric.time(:duration_in_millis),
-        @pipeline_metric.time(:duration_in_millis)
+          @event_metric.time(:duration_in_millis),
+          @pipeline_metric.time(:duration_in_millis)
         ]
       end
 
-      def stop_clock
-        @inflight_clocks[Thread.current].each(&:stop)
-        @inflight_clocks.delete(Thread.current)
+      def stop_clock(batch)
+        unless @inflight_clocks[Thread.current].nil?
+          if batch.size > 0
+            # onl/y stop (which also records) the metrics if the batch is non-empty.
+            # start_clock is now called at empty batch creation and an empty batch could
+            # stay empty all the way down to the close_batch call.
+            @inflight_clocks[Thread.current].each(&:stop)
+          end
+          @inflight_clocks.delete(Thread.current)
+        end
       end
 
       def add_starting_metrics(batch)
@@ -213,6 +238,10 @@ module LogStash; module Util
 
     class ReadBatch
       def initialize(queue, size, wait)
+        @queue = queue
+        @size = size
+        @wait = wait
+
         @originals = Hash.new
 
         # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
@@ -221,7 +250,13 @@ module LogStash; module Util
         @generated = Hash.new
         @iterating_temp = Hash.new
         @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
-        take_originals_from_queue(queue, size, wait) # this sets a reference to @acked_batch
+        @acked_batch = nil
+      end
+
+      def read_next
+        @acked_batch = @queue.read_batch(@size, @wait)
+        return if @acked_batch.nil?
+        @acked_batch.get_elements.each { |e| @originals[e] = true }
       end
 
       def close
@@ -301,14 +336,6 @@ module LogStash; module Util
         @generated.update(@iterating_temp)
         @iterating_temp.clear
       end
-
-      def take_originals_from_queue(queue, size, wait)
-        @acked_batch = queue.read_batch(size, wait)
-        return if @acked_batch.nil?
-        @acked_batch.get_elements.each do |e|
-          @originals[e] = true
-        end
-      end
     end
 
     class WriteClient
@@ -343,6 +370,10 @@ module LogStash; module Util
         @events = []
       end
 
+      def size
+        @events.size
+      end
+
       def push(event)
         @events.push(event)
       end

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -18,12 +18,12 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
-    # Offer an object to the queue, wait for the specified amout of time.
-    # If adding to the queue was successfull it wil return true, false otherwise.
+    # Offer an object to the queue, wait for the specified amount of time.
+    # If adding to the queue was successful it wil return true, false otherwise.
     #
     # @param [Object] Object to add to the queue
     # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successfull if not it return false
+    # @return [Boolean] True if adding was successful if not it return false
     def offer(obj, timeout_ms)
       @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
     end
@@ -58,7 +58,7 @@ module LogStash; module Util
       def initialize(queue, batch_size = 125, wait_for = 250)
         @queue = queue
         @mutex = Mutex.new
-        # Note that @infilght_batches as a central mechanism for tracking inflight
+        # Note that @inflight_batches as a central mechanism for tracking inflight
         # batches will fail if we have multiple read clients in the pipeline.
         @inflight_batches = {}
 
@@ -72,6 +72,10 @@ module LogStash; module Util
         # noop, compat with acked queue read client
       end
 
+      def empty?
+        true # synchronous queue is alway empty
+      end
+
       def set_batch_dimensions(batch_size, wait_for)
         @batch_size = batch_size
         @wait_for = wait_for
@@ -90,7 +94,6 @@ module LogStash; module Util
       def define_initial_metrics_values(namespaced_metric)
         namespaced_metric.report_time(:duration_in_millis, 0)
         namespaced_metric.increment(:filtered, 0)
-        namespaced_metric.increment(:in, 0)
         namespaced_metric.increment(:out, 0)
       end
 
@@ -104,18 +107,24 @@ module LogStash; module Util
         @inflight_batches.fetch(Thread.current, [])
       end
 
-      def take_batch
+      # create a new empty batch
+      # @return [ReadBatch] a new empty read batch
+      def new_batch
+        ReadBatch.new(@queue, @batch_size, @wait_for)
+      end
+
+      def read_batch
+        batch = new_batch
+        @mutex.synchronize { batch.read_next }
+        start_metrics(batch)
+        batch
+      end
+
+      def start_metrics(batch)
         @mutex.synchronize do
-          batch = ReadBatch.new(@queue, @batch_size, @wait_for)
+          # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
-
-          # We dont actually have any events to work on so lets
-          # not bother with recording metrics for them
-          if batch.size > 0
-            add_starting_metrics(batch)
-            start_clock
-          end
-          batch
+          start_clock
         end
       end
 
@@ -125,8 +134,9 @@ module LogStash; module Util
 
       def close_batch(batch)
         @mutex.synchronize do
+          # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
-          stop_clock
+          stop_clock(batch)
         end
       end
 
@@ -137,18 +147,18 @@ module LogStash; module Util
         ]
       end
 
-      def stop_clock
+      def stop_clock(batch)
         unless @inflight_clocks[Thread.current].nil?
-          @inflight_clocks[Thread.current].each(&:stop)
+          if batch.size > 0
+            # only stop (which also records) the metrics if the batch is non-empty.
+            # start_clock is now called at empty batch creation and an empty batch could
+            # stay empty all the way down to the close_batch call.
+            @inflight_clocks[Thread.current].each(&:stop)
+          end
           @inflight_clocks.delete(Thread.current)
         end
       end
 
-      def add_starting_metrics(batch)
-        @event_metric.increment(:in, batch.starting_size)
-        @pipeline_metric.increment(:in, batch.starting_size)
-      end
-
       def add_filtered_metrics(batch)
         @event_metric.increment(:filtered, batch.filtered_size)
         @pipeline_metric.increment(:filtered, batch.filtered_size)
@@ -162,6 +172,10 @@ module LogStash; module Util
 
     class ReadBatch
       def initialize(queue, size, wait)
+        @queue = queue
+        @size = size
+        @wait = wait
+
         @originals = Hash.new
 
         # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
@@ -170,7 +184,16 @@ module LogStash; module Util
         @generated = Hash.new
         @iterating_temp = Hash.new
         @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
-        take_originals_from_queue(queue, size, wait)
+        @acked_batch = nil
+      end
+
+      def read_next
+        @size.times do |t|
+          event = @queue.poll(@wait)
+          return if event.nil? # queue poll timed out
+
+          @originals[event] = true
+        end
       end
 
       def merge(event)
@@ -235,15 +258,6 @@ module LogStash; module Util
         @generated.update(@iterating_temp)
         @iterating_temp.clear
       end
-
-      def take_originals_from_queue(queue, size, wait)
-        size.times do |t|
-          event = queue.poll(wait)
-          return if event.nil? # queue poll timed out
-
-          @originals[event] = true
-        end
-      end
     end
 
     class WriteClient
@@ -272,6 +286,10 @@ module LogStash; module Util
         @events = []
       end
 
+      def size
+        @events.size
+      end
+
       def push(event)
         @events.push(event)
       end