logstash-core 6.0.1-java → 6.1.0-java

data/lib/logstash/plugins/registry.rb

@@ -9,6 +9,8 @@ module LogStash module Plugins
   class Registry
     include LogStash::Util::Loggable
 
+    class UnknownPlugin < NameError; end
+
     # Add a bit more sanity with when interacting with the rubygems'
     # specifications database, most of out code interact directly with really low level
     # components of bundler/rubygems we need to encapsulate that and this is a start.
@@ -152,14 +154,19 @@ module LogStash module Plugins
       begin
         path = "logstash/#{type}s/#{plugin_name}"
 
-        begin
-          require path
-        rescue LoadError
-          # Plugin might be already defined in the current scope
-          # This scenario often happen in test when we write an adhoc class
+        klass = begin
+          namespace_lookup(type, plugin_name)
+        rescue UnknownPlugin => e
+          # Plugin not registered. Try to load it.
+          begin
+            require path
+            namespace_lookup(type, plugin_name)
+          rescue LoadError => e
+            logger.error("Tried to load a plugin's code, but failed.", :exception => e, :path => path, :type => type, :name => plugin_name)
+            raise
+          end
         end
 
-        klass = namespace_lookup(type, plugin_name)
         plugin = lazy_add(type, plugin_name, klass)
       rescue => e
         logger.error("Problems loading a plugin with",
@@ -195,6 +202,10 @@ module LogStash module Plugins
       add_plugin(type, name, klass)
     end
 
+    def remove(type, plugin_name)
+      @registry.delete(key_for(type, plugin_name))
+    end
+
     def get(type, plugin_name)
       @registry[key_for(type, plugin_name)]
     end
@@ -223,7 +234,7 @@ module LogStash module Plugins
       klass_sym = namespace.constants.find { |c| is_a_plugin?(namespace.const_get(c), name) }
       klass = klass_sym && namespace.const_get(klass_sym)
 
-      raise(NameError) unless klass
+      raise(UnknownPlugin) unless klass
       klass
     end
 

data/lib/logstash/queue_factory.rb

@@ -29,7 +29,9 @@ module LogStash
         LogStash::Util::WrappedAckedQueue.create_file_based(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
       when "memory"
         # memory is the legacy and default setting
-        LogStash::Util::WrappedSynchronousQueue.new
+        LogStash::Util::WrappedSynchronousQueue.new(
+          settings.get("pipeline.batch.size") * settings.get("pipeline.workers") * 2
+        )
       else
         raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are: 'memory_acked', 'memory', 'persisted'"
       end

data/lib/logstash/runner.rb

@@ -10,6 +10,7 @@ require "logstash/namespace"
 require "logstash-core/logstash-core"
 require "logstash/environment"
 require "logstash/modules/cli_parser"
+require "logstash/util/settings_helper"
 
 LogStash::Environment.load_locale!
 
@@ -30,12 +31,8 @@ java_import 'org.logstash.FileLockFactory'
 
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
-  # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
-  # because the `Environment::LOGSTASH_HOME` doesn't exist in the context of the `logstash-core` gem.
-  #
-  # See issue https://github.com/elastic/logstash/issues/5361
-  LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
-  LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
+
+  LogStash::Util::SettingsHelper.pre_process
 
   # Ordered list of check to run before starting logstash
   # theses checks can be changed by a plugin loaded into memory.
@@ -93,6 +90,11 @@ class LogStash::Runner < Clamp::StrictCommand
     :attribute_name => "pipeline.workers",
     :default => LogStash::SETTINGS.get_default("pipeline.workers")
 
+  option ["--experimental-java-execution"], :flag,
+         I18n.t("logstash.runner.flag.experimental-java-execution"),
+         :attribute_name => "pipeline.java_execution",
+         :default => LogStash::SETTINGS.get_default("pipeline.java_execution")
+
   option ["-b", "--pipeline.batch.size"], "SIZE",
     I18n.t("logstash.runner.flag.pipeline-batch-size"),
     :attribute_name => "pipeline.batch.size",
@@ -208,35 +210,12 @@ class LogStash::Runner < Clamp::StrictCommand
   end
 
   def run(args)
-    settings_path = fetch_settings_path(args)
-
-    @settings.set("path.settings", settings_path) if settings_path
-
-    begin
-      LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
-    rescue Errno::ENOENT
-      $stderr.puts "WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults"
-    rescue => e
-      # abort unless we're just looking for the help
-      unless cli_help?(args)
-        if e.kind_of?(Psych::Exception)
-          yaml_file_path = ::File.join(LogStash::SETTINGS.get("path.settings"), "logstash.yml")
-          $stderr.puts "ERROR: Failed to parse YAML file \"#{yaml_file_path}\". Please confirm if the YAML structure is valid (e.g. look for incorrect usage of whitespace or indentation). Aborting... parser_error=>#{e.message}"
-        else
-          $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
-        end
-        return 1
-      end
-    end
-
+    return 1 unless LogStash::Util::SettingsHelper.from_yaml(args)
     super(*[args])
   end
 
   def execute
-    # Only when execute is have the CLI options been added to the @settings
-    # We invoke post_process to apply extra logic to them.
-    # The post_process callbacks have been added in environment.rb
-    @settings.post_process
+    LogStash::Util::SettingsHelper.post_process
 
     require "logstash/util"
     require "logstash/util/java_version"
@@ -359,6 +338,8 @@ class LogStash::Runner < Clamp::StrictCommand
     sigint_id = trap_sigint()
     sigterm_id = trap_sigterm()
 
+    logger.info("Starting Logstash", "logstash.version" => LOGSTASH_VERSION)
+
     @agent_task = Stud::Task.new { @agent.execute }
 
     # no point in enabling config reloading before the agent starts
@@ -402,7 +383,7 @@ class LogStash::Runner < Clamp::StrictCommand
 
     if logger.info?
       show_version_ruby
-      show_version_java if LogStash::Environment.jruby?
+      show_version_java
       show_gems if logger.debug?
     end
   end # def show_version
@@ -501,28 +482,4 @@ class LogStash::Runner < Clamp::StrictCommand
     @settings.get_value(key)
   end
 
-  # where can I find the logstash.yml file?
-  # 1. look for a "--path.settings path"
-  # 2. look for a "--path.settings=path"
-  # 3. check if the LS_SETTINGS_DIR environment variable is set
-  # 4. return nil if not found
-  def fetch_settings_path(cli_args)
-    if i=cli_args.find_index("--path.settings")
-      cli_args[i+1]
-    elsif settings_arg = cli_args.find {|v| v.match(/--path.settings=/) }
-      match = settings_arg.match(/--path.settings=(.*)/)
-      match[1]
-    elsif ENV['LS_SETTINGS_DIR']
-      ENV['LS_SETTINGS_DIR']
-    else
-      nil
-    end
-  end
-
-  # is the user asking for CLI help subcommand?
-  def cli_help?(args)
-    # I know, double negative
-    !(["--help", "-h"] & args).empty?
-  end
-
 end

data/lib/logstash/settings.rb

@@ -2,13 +2,13 @@
 require "logstash/util/loggable"
 require "fileutils"
 require "logstash/util/byte_value"
-require "logstash/util/environment_variables"
+require "logstash/util/substitution_variables"
 require "logstash/util/time_value"
 
 module LogStash
   class Settings
 
-    include LogStash::Util::EnvironmentVariables
+    include LogStash::Util::SubstitutionVariables
     
     def initialize
       @settings = {}

data/lib/logstash/timestamp.rb

@@ -3,7 +3,6 @@
 require "logstash/namespace"
 
 module LogStash
-  class TimestampParserError < StandardError; end
 
   class Timestamp
     include Comparable

data/lib/logstash/util.rb

@@ -147,28 +147,20 @@ module LogStash::Util
   # to support these pure Ruby object monkey patches.
   # see logstash/json.rb and logstash/java_integration.rb
 
-  if LogStash::Environment.jruby?
-    require "java"
-
-    # recursively convert any Java LinkedHashMap and ArrayList to pure Ruby.
-    # will not recurse into pure Ruby objects. Pure Ruby object should never
-    # contain LinkedHashMap and ArrayList since these are only created at
-    # initial deserialization, anything after (deeper) will be pure Ruby.
-    def self.normalize(o)
-      case o
-      when Java::JavaUtil::LinkedHashMap
-        o.inject({}){|r, (k, v)| r[k] = normalize(v); r}
-      when Java::JavaUtil::ArrayList
-        o.map{|i| normalize(i)}
-      else
-        o
-      end
+  require "java"
+  # recursively convert any Java LinkedHashMap and ArrayList to pure Ruby.
+  # will not recurse into pure Ruby objects. Pure Ruby object should never
+  # contain LinkedHashMap and ArrayList since these are only created at
+  # initial deserialization, anything after (deeper) will be pure Ruby.
+  def self.normalize(o)
+    case o
+    when Java::JavaUtil::LinkedHashMap
+      o.inject({}){|r, (k, v)| r[k] = normalize(v); r}
+    when Java::JavaUtil::ArrayList
+      o.map{|i| normalize(i)}
+    else
+      o
     end
-
-  else
-
-    # identity function, pure Ruby object don't need normalization.
-    def self.normalize(o); o; end
   end
 
   def self.stringify_symbols(o)

data/lib/logstash/util/java_version.rb

@@ -3,7 +3,6 @@
 module LogStash::Util::JavaVersion
   # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
   def self.version
-    return nil unless LogStash::Environment.jruby?
     java.lang.System.getProperty("java.runtime.version")
   end
 

data/lib/logstash/util/settings_helper.rb

@@ -0,0 +1,79 @@
+require "logstash/settings"
+require "logstash-core/logstash-core"
+require "logstash/environment"
+
+##########################
+# Settings have an implicit initialization life-cycle, this helper exists to aid with the reading the settings.
+#
+# Defaults are read from "logstash/environment"
+# SettingsHelper.pre_process - registers defaults unavailable from "logstash/environment"
+# SettingsHelper.from_yaml - finds yaml based on "--path.settings" and updates defaults if needed.
+# SettingsHelper.post_process - callback to re-calculate derived settings
+#
+# The methods should be called in the above order before the settings are ready to be used.
+########################
+module LogStash::Util::SettingsHelper
+
+  # The `path.settings` and `path.logs` can not be defined in "logstash/environment" since the `Environment::LOGSTASH_HOME` doesn't
+  # exist unless launched via "bootstrap/environment"
+  def self.pre_process
+    LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
+    LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
+  end
+
+  # Ensure that any settings are re-calculated after loading yaml
+  def self.post_process
+    LogStash::SETTINGS.post_process
+  end
+
+  # param args: The array of all the command line arguments. Used to find the "path.settings" to read the settings yaml.
+  # Return true if successful, false otherwise
+  def self.from_yaml(args)
+    settings_path = fetch_settings_path(args)
+
+    LogStash::SETTINGS.set("path.settings", settings_path) if settings_path
+
+    begin
+      LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
+    rescue Errno::ENOENT
+      $stderr.puts "WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults"
+    rescue => e
+      # abort unless we're just looking for the help
+      unless cli_help?(args)
+        if e.kind_of?(Psych::Exception)
+          yaml_file_path = ::File.join(LogStash::SETTINGS.get("path.settings"), "logstash.yml")
+          $stderr.puts "ERROR: Failed to parse YAML file \"#{yaml_file_path}\". Please confirm if the YAML structure is valid (e.g. look for incorrect usage of whitespace or indentation). Aborting... parser_error=>#{e.message}"
+        else
+          $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
+        end
+        return false
+      end
+    end
+    return true
+  end
+
+  # where can I find the logstash.yml file?
+  # 1. look for a "--path.settings path"
+  # 2. look for a "--path.settings=path"
+  # 3. check if the LS_SETTINGS_DIR environment variable is set
+  # 4. return nil if not found
+  def self.fetch_settings_path(cli_args)
+    if i=cli_args.find_index("--path.settings")
+      cli_args[i+1]
+    elsif settings_arg = cli_args.find {|v| v.match(/--path.settings=/)}
+      match = settings_arg.match(/--path.settings=(.*)/)
+      match[1]
+    elsif ENV['LS_SETTINGS_DIR']
+      ENV['LS_SETTINGS_DIR']
+    else
+      nil
+    end
+  end
+
+  # is the user asking for CLI help subcommand?
+  def self.cli_help?(args)
+    # I know, double negative
+    !(["--help", "-h"] & args).empty?
+  end
+
+end

data/lib/logstash/util/{environment_variables.rb → substitution_variables.rb}

@@ -1,7 +1,7 @@
 # encoding: utf-8
-module ::LogStash::Util::EnvironmentVariables
+module ::LogStash::Util::SubstitutionVariables
 
-  ENV_PLACEHOLDER_REGEX = /\${(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?}/
+  SUBSTITUTION_PLACEHOLDER_REGEX = /\${(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?}/
 
   # Recursive method to replace environment variable references in parameters
   def deep_replace(value)
@@ -15,17 +15,19 @@ module ::LogStash::Util::EnvironmentVariables
           value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
         end
       else
-        return replace_env_placeholders(value)
+        return replace_placeholders(value)
       end
     end
   end
 
-  # Replace all environment variable references in 'value' param by environment variable value and return updated value
-  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
-  def replace_env_placeholders(value)
+  # Replace all substitution variable references in the 'value' param and returns the substituted value, or the original value if a substitution can not be made
+  # Process following patterns : ${VAR}, ${VAR:defaultValue}
+  # If value matches the pattern, returns the following precedence : Environment entry value, default value as provided in the pattern
+  # If the value does not match the pattern, the 'value' param returns as-is
+  def replace_placeholders(value)
     return value unless value.is_a?(String)
 
-    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
+    value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder|
       # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
       # the call, so this should be thread-safe.
       #
@@ -39,5 +41,5 @@ module ::LogStash::Util::EnvironmentVariables
       end
       replacement
     end
-  end # def replace_env_placeholders
+  end # def replace_placeholders
 end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 
-require "jruby_acked_queue_ext"
-require "jruby_acked_batch_ext"
 require "concurrent"
 # This is an adapted copy of the wrapped_synchronous_queue file
 # ideally this should be moved to Java/JRuby
@@ -103,7 +101,7 @@ module LogStash; module Util
         # batches will fail if we have multiple read clients in the pipeline.
         @inflight_batches = {}
         # allow the worker thread to report the execution time of the filter + output
-        @inflight_clocks = {}
+        @inflight_clocks = Concurrent::Map.new
         @batch_size = batch_size
         @wait_for = wait_for
       end
@@ -151,10 +149,6 @@ module LogStash; module Util
         end
       end
 
-      def current_inflight_batch
-        @inflight_batches.fetch(Thread.current, [])
-      end
-
       # create a new empty batch
       # @return [ReadBatch] a new empty read batch
       def new_batch
@@ -167,75 +161,52 @@ module LogStash; module Util
         end
 
         batch = new_batch
-        @mutex.lock
-        begin
-          batch.read_next
-        ensure
-          @mutex.unlock
-        end
+        batch.read_next
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
+        thread = Thread.current
         @mutex.lock
         begin
-          # there seems to be concurrency issues with metrics, keep it in the mutex
-          set_current_thread_inflight_batch(batch)
-          start_clock
+          @inflight_batches[thread] = batch
         ensure
           @mutex.unlock
         end
-      end
-
-      def set_current_thread_inflight_batch(batch)
-        @inflight_batches[Thread.current] = batch
+        @inflight_clocks[thread] = java.lang.System.nano_time
       end
 
       def close_batch(batch)
+        thread = Thread.current
         @mutex.lock
         begin
           batch.close
-          # there seems to be concurrency issues with metrics, keep it in the mutex
-          @inflight_batches.delete(Thread.current)
-          stop_clock(batch)
+          @inflight_batches.delete(thread)
         ensure
           @mutex.unlock
         end
-      end
-
-      def start_clock
-        @inflight_clocks[Thread.current] = java.lang.System.nano_time
-      end
-
-      def stop_clock(batch)
-        unless @inflight_clocks[Thread.current].nil?
+        start_time = @inflight_clocks.get_and_set(thread, nil)
+        unless start_time.nil?
           if batch.size > 0
             # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            time_taken = (java.lang.System.nano_time - @inflight_clocks[Thread.current]) / 1_000_000
+            time_taken = (java.lang.System.nano_time - start_time) / 1_000_000
             @event_metric.report_time(:duration_in_millis, time_taken)
             @pipeline_metric.report_time(:duration_in_millis, time_taken)
           end
-          @inflight_clocks.delete(Thread.current)
         end
       end
 
-      def add_starting_metrics(batch)
-        return if @event_metric.nil? || @pipeline_metric.nil?
-        @event_metric.increment(:in, batch.starting_size)
-        @pipeline_metric.increment(:in, batch.starting_size)
-      end
-
-      def add_filtered_metrics(batch)
-        @event_metric.increment(:filtered, batch.filtered_size)
-        @pipeline_metric.increment(:filtered, batch.filtered_size)
+      def add_filtered_metrics(filtered_size)
+        @event_metric.increment(:filtered, filtered_size)
+        @pipeline_metric.increment(:filtered, filtered_size)
       end
 
-      def add_output_metrics(batch)
-        @event_metric.increment(:out, batch.filtered_size)
-        @pipeline_metric.increment(:out, batch.filtered_size)
+      def add_output_metrics(filtered_size)
+        @event_metric.increment(:out, filtered_size)
+        @pipeline_metric.increment(:out, filtered_size)
       end
     end
 
@@ -251,8 +222,6 @@ module LogStash; module Util
         # @cancelled = Hash.new
 
         @generated = Hash.new
-        @iterating_temp = Hash.new
-        @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
         @acked_batch = nil
       end
 
@@ -271,20 +240,7 @@ module LogStash; module Util
 
       def merge(event)
         return if event.nil? || @originals.key?(event)
-        # take care not to cause @generated to change during iteration
-        # @iterating_temp is merged after the iteration
-        if iterating?
-          @iterating_temp[event] = true
-        else
-          # the periodic flush could generate events outside of an each iteration
-          @generated[event] = true
-        end
-      end
-
-      def cancel(event)
-        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
-        raise("cancel is unsupported")
-        # @cancelled[event] = true
+        @generated[event] = true
       end
 
       def to_a
@@ -294,19 +250,14 @@ module LogStash; module Util
       end
 
       def each(&blk)
-        # take care not to cause @originals or @generated to change during iteration
-
         # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
         # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
-        @iterating = true
         @originals.each do |e, _|
           blk.call(e) unless e.cancelled?
         end
         @generated.each do |e, _|
           blk.call(e) unless e.cancelled?
         end
-        @iterating = false
-        update_generated
       end
 
       def size
@@ -321,12 +272,6 @@ module LogStash; module Util
         @originals.size + @generated.size
       end
 
-      def cancelled_size
-        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
-        raise("cancelled_size is unsupported ")
-        # @cancelled.size
-      end
-
       def shutdown_signal_received?
         false
       end
@@ -334,17 +279,6 @@ module LogStash; module Util
       def flush_signal_received?
         false
       end
-
-      private
-
-      def iterating?
-        @iterating
-      end
-
-      def update_generated
-        @generated.update(@iterating_temp)
-        @iterating_temp.clear
-      end
     end
 
     class WriteClient
@@ -352,10 +286,6 @@ module LogStash; module Util
         @queue = queue
       end
 
-      def get_new_batch
-        WriteBatch.new
-      end
-
       def push(event)
         if @queue.closed?
           raise QueueClosedError.new("Attempted to write an event to a closed AckedQueue")
@@ -373,26 +303,5 @@ module LogStash; module Util
         end
       end
     end
-
-    class WriteBatch
-      def initialize
-        @events = []
-      end
-
-      def size
-        @events.size
-      end
-
-      def push(event)
-        @events.push(event)
-      end
-      alias_method(:<<, :push)
-
-      def each(&blk)
-        @events.each do |e|
-          blk.call(e)
-        end
-      end
-    end
   end
 end end