logstash-core 6.3.2-java → 6.4.0-java

data/lib/logstash/pipeline_action/create.rb

@@ -2,8 +2,6 @@
 require "logstash/pipeline_action/base"
 require "logstash/pipeline"
 require "logstash/java_pipeline"
-require "logstash/converge_result"
-require "logstash/util/loggable"
 
 module LogStash module PipelineAction
   class Create < Base

data/lib/logstash/pipeline_action/reload.rb

@@ -2,9 +2,6 @@
 require "logstash/pipeline_action/base"
 require "logstash/pipeline_action/create"
 require "logstash/pipeline_action/stop"
-require "logstash/errors"
-require "logstash/util/loggable"
-require "logstash/converge_result"
 
 module LogStash module PipelineAction
   class Reload < Base
@@ -33,7 +30,7 @@ module LogStash module PipelineAction
       begin
         pipeline_validator =
           if @pipeline_config.settings.get_value("pipeline.java_execution")
-            LogStash::JavaBasePipeline.new(@pipeline_config)
+            LogStash::JavaBasePipeline.new(@pipeline_config, nil, logger, nil)
           else
             LogStash::BasePipeline.new(@pipeline_config)
           end

data/lib/logstash/pipeline_action/stop.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 require "logstash/pipeline_action/base"
-require "logstash/shutdown_watcher"
-require "logstash/converge_result"
 
 module LogStash module PipelineAction
   class Stop < Base

data/lib/logstash/pipeline_reporter.rb

@@ -1,108 +1,2 @@
-# encoding: utf-8
-require 'ostruct'
-
-module LogStash; class PipelineReporter
-  attr_reader :logger, :pipeline
-
-  # This is an immutable copy of the pipeline state,
-  # It is a proxy to a hash to allow us to add methods dynamically to the hash
-  class Snapshot
-    def initialize(data)
-      @data = data
-    end
-
-    def to_hash
-      @data
-    end
-
-    def to_simple_hash
-      {"inflight_count" => inflight_count, "stalling_thread_info" => format_threads_by_plugin}
-    end
-
-    def to_str
-      to_simple_hash.to_s
-    end
-    alias_method :to_s, :to_str
-
-    def method_missing(meth)
-      @data[meth]
-    end
-
-    def format_threads_by_plugin
-      stalled_plugins = {}
-      stalling_threads_info.each do |thr|
-        key = (thr.delete("plugin") || "other")
-        stalled_plugins[key] ||= []
-        stalled_plugins[key] << thr
-      end
-      stalled_plugins
-    end
-  end
-
-  def initialize(logger, pipeline)
-    @logger = logger
-    @pipeline = pipeline
-  end
-
-  # The main way of accessing data from the reporter,,
-  # this provides a (more or less) consistent snapshot of what's going on in the
-  # pipeline with some extra decoration
-  def snapshot
-    Snapshot.new(self.to_hash)
-  end
-
-  def to_hash
-    # pipeline.filter_queue_client.inflight_batches is synchronized
-    batch_map = pipeline.filter_queue_client.inflight_batches
-    worker_states_snap = worker_states(batch_map) # We only want to run this once
-    inflight_count = worker_states_snap.map {|s| s[:inflight_count]}.reduce(0, :+)
-    {
-        :events_filtered => events_filtered,
-        :events_consumed => events_consumed,
-        :inflight_count => inflight_count,
-        :worker_states => worker_states_snap,
-        :output_info => output_info,
-        :thread_info => pipeline.plugin_threads_info,
-        :stalling_threads_info => pipeline.stalling_threads_info
-    }
-  end
-
-  private
-
-  def events_filtered
-    pipeline.events_filtered.sum
-  end
-
-  def events_consumed
-    pipeline.events_consumed.sum
-  end
-
-  def plugin_threads
-    pipeline.plugin_threads
-  end
-
-  # Not threadsafe! ensure synchronization
-  def worker_states(batch_map)
-    pipeline.worker_threads.map.with_index do |thread, idx|
-      status = thread.status || "dead"
-      batch = batch_map[thread]
-      inflight_count = batch ? batch.size : 0
-      {
-        :status => status,
-        :alive => thread.alive?,
-        :index => idx,
-        :inflight_count => inflight_count
-      }
-    end
-  end
-
-  def output_info
-    pipeline.outputs.map do |output_delegator|
-      {
-        :type => output_delegator.config_name,
-        :id => output_delegator.id,
-        :concurrency => output_delegator.concurrency,
-      }
-    end
-  end
-end end
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly require it.

data/lib/logstash/plugin.rb

@@ -1,9 +1,5 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require "logstash/config/mixin"
-require "logstash/instrument/null_metric"
-require "logstash/util/dead_letter_queue_manager"
 require "concurrent"
 require "securerandom"
 
@@ -21,9 +17,9 @@ class LogStash::Plugin
   # for a specific plugin.
   config :enable_metric, :validate => :boolean, :default => true
 
-  # Add a unique `ID` to the plugin configuration. If no ID is specified, Logstash will generate one. 
-  # It is strongly recommended to set this ID in your configuration. This is particularly useful 
-  # when you have two or more plugins of the same type, for example, if you have 2 grok filters. 
+  # Add a unique `ID` to the plugin configuration. If no ID is specified, Logstash will generate one.
+  # It is strongly recommended to set this ID in your configuration. This is particularly useful
+  # when you have two or more plugins of the same type, for example, if you have 2 grok filters.
   # Adding a named ID in this case will help in monitoring Logstash when using the monitoring APIs.
   #
   # [source,ruby]
@@ -137,6 +133,7 @@ class LogStash::Plugin
   # Should I remove this now and make sure the pipeline invoke the Registry or I should wait for 6.0
   # Its not really part of the public api but its used by the tests a lot to mock the plugins.
   def self.lookup(type, name)
+    require "logstash/plugins/registry"
     LogStash::PLUGIN_REGISTRY.lookup_pipeline_plugin(type, name)
   end
 end # class LogStash::Plugin

data/lib/logstash/plugins/hooks_registry.rb

@@ -1,63 +1 @@
-# encoding: utf-8
-module LogStash module Plugins
-  # This calls allow logstash to expose the endpoints for listeners
-  class HooksRegistry
-    java_import "java.util.concurrent.ConcurrentHashMap"
-    java_import "java.util.concurrent.CopyOnWriteArrayList"
-
-    def initialize
-      @registered_emitters = ConcurrentHashMap.new
-      @registered_hooks = ConcurrentHashMap.new
-    end
-
-    def register_emitter(emitter_scope, dispatcher)
-      @registered_emitters.put(emitter_scope, dispatcher)
-      sync_hooks
-    end
-
-    def remove_emitter(emitter_scope)
-      @registered_emitters.remove(emitter_scope)
-    end
-
-    def register_hooks(emitter_scope, callback)
-      callbacks = @registered_hooks.computeIfAbsent(emitter_scope) do
-        CopyOnWriteArrayList.new
-      end
-
-      callbacks.add(callback)
-      sync_hooks
-    end
-
-    def emitters_count
-      @registered_emitters.size
-    end
-
-    def hooks_count(emitter_scope = nil)
-      if emitter_scope.nil?
-        @registered_hooks.elements().collect(&:size).reduce(0, :+)
-      else
-        callbacks = @registered_hooks.get(emitter_scope)
-        callbacks.nil? ? 0 : @registered_hooks.get(emitter_scope).size
-      end
-    end
-
-    def registered_hook?(emitter_scope, klass)
-      callbacks = @registered_hooks[emitter_scope]
-      return false if callbacks.nil?
-      callbacks.collect(&:class).include?(klass)
-    end
-
-    private
-    def sync_hooks
-      @registered_emitters.each do |emitter, dispatcher|
-        listeners = @registered_hooks.get(emitter)
-
-        unless listeners.nil?
-          listeners.each do |listener|
-            dispatcher.add_listener(listener)
-          end
-        end
-      end
-    end
-  end
-end end
+# encoding: utf-8

data/lib/logstash/plugins/registry.rb

@@ -1,9 +1,10 @@
 # encoding: utf-8
 require "rubygems/package"
-require "logstash/util/loggable"
 require "logstash/plugin"
-require "logstash/plugins/hooks_registry"
 require "logstash/modules/scaffold"
+require "logstash/codecs/base"
+require "logstash/filters/base"
+require "logstash/outputs/base"
 
 module LogStash module Plugins
   class Registry

data/lib/logstash/runner.rb

@@ -18,7 +18,6 @@ end
 require "clamp"
 require "net/http"
 
-require "logstash/namespace"
 require "logstash-core/logstash-core"
 require "logstash/environment"
 require "logstash/modules/cli_parser"
@@ -28,15 +27,12 @@ LogStash::Environment.load_locale!
 
 require "logstash/agent"
 require "logstash/config/defaults"
-require "logstash/shutdown_watcher"
 require "logstash/patches/clamp"
 require "logstash/settings"
 require "logstash/version"
 require 'logstash/plugins'
 require "logstash/modules/util"
 require "logstash/bootstrap_check/default_config"
-require "logstash/bootstrap_check/bad_java"
-require "logstash/bootstrap_check/bad_ruby"
 require "logstash/bootstrap_check/persisted_queue_config"
 require "set"
 
@@ -50,8 +46,6 @@ class LogStash::Runner < Clamp::StrictCommand
   # Ordered list of check to run before starting logstash
   # theses checks can be changed by a plugin loaded into memory.
   DEFAULT_BOOTSTRAP_CHECKS = [
-      LogStash::BootstrapCheck::BadRuby,
-      LogStash::BootstrapCheck::BadJava,
       LogStash::BootstrapCheck::DefaultConfig,
       LogStash::BootstrapCheck::PersistedQueueConfig
   ]
@@ -74,6 +68,11 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("config.string"),
     :attribute_name => "config.string"
 
+  option ["--field-reference-parser"], "MODE",
+         I18n.t("logstash.runner.flag.field-reference-parser"),
+         :attribute_name => "config.field_reference.parser",
+         :default => LogStash::SETTINGS.get_default("config.field_reference.parser")
+
   # Module settings
   option ["--modules"], "MODULES",
     I18n.t("logstash.runner.flag.modules"),
@@ -281,7 +280,7 @@ class LogStash::Runner < Clamp::StrictCommand
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
 
-    module_parser = LogStash::Modules::CLIParser.new(@modules_list, @modules_variable_list)
+    module_parser = LogStash::Modules::CLIParser.new(setting("modules_list"), setting("modules_variable_list"))
     # Now populate Setting for modules.list with our parsed array.
     @settings.set("modules.cli", module_parser.output)
 

data/lib/logstash/settings.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/util/loggable"
 require "fileutils"
 require "logstash/util/byte_value"
 require "logstash/util/substitution_variables"
@@ -10,7 +9,7 @@ module LogStash
 
     include LogStash::Util::SubstitutionVariables
     include LogStash::Util::Loggable
-    
+
     def initialize
       @settings = {}
       # Theses settings were loaded from the yaml file
@@ -119,7 +118,7 @@ module LogStash
       self.merge(deep_replace(flatten_hash(settings)), true)
       self
     end
-    
+
     def post_process
       if @post_process_callbacks
         @post_process_callbacks.each do |callback|
@@ -127,7 +126,7 @@ module LogStash
         end
       end
     end
-    
+
     def on_post_process(&block)
       @post_process_callbacks ||= []
       @post_process_callbacks << block
@@ -439,7 +438,7 @@ module LogStash
       def initialize(name, default=nil, strict=false)
         super(name, ::String, default, strict)
       end
-      
+
       def validate(path)
         super(path)
 

data/lib/logstash/shutdown_watcher.rb

@@ -1,120 +1 @@
 # encoding: utf-8
-require "concurrent/atomic/atomic_fixnum"
-require "concurrent/atomic/atomic_boolean"
-
-module LogStash
-  class ShutdownWatcher
-    include LogStash::Util::Loggable
-
-    CHECK_EVERY = 1 # second
-    REPORT_EVERY = 5 # checks
-    ABORT_AFTER = 3 # stalled reports
-
-    attr_reader :cycle_period, :report_every, :abort_threshold
-
-    def initialize(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
-      @pipeline = pipeline
-      @cycle_period = cycle_period
-      @report_every = report_every
-      @abort_threshold = abort_threshold
-      @reports = []
-      @attempts_count = Concurrent::AtomicFixnum.new(0)
-      @running = Concurrent::AtomicBoolean.new(false)
-    end
-
-    def self.unsafe_shutdown=(boolean)
-      @unsafe_shutdown = boolean
-    end
-
-    def self.unsafe_shutdown?
-      @unsafe_shutdown
-    end
-
-    def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
-      controller = self.new(pipeline, cycle_period, report_every, abort_threshold)
-      Thread.new(controller) { |controller| controller.start }
-    end
-
-    def logger
-      self.class.logger
-    end
-
-    def attempts_count
-      @attempts_count.value
-    end
-
-    def stop!
-      @running.make_false
-    end
-
-    def stopped?
-      @running.false?
-    end
-
-    def start
-      sleep(@cycle_period)
-      cycle_number = 0
-      stalled_count = 0
-      running!
-      Stud.interval(@cycle_period) do
-        @attempts_count.increment
-        break if stopped?
-        break unless @pipeline.thread.alive?
-        @reports << pipeline_report_snapshot
-        @reports.delete_at(0) if @reports.size > @report_every # expire old report
-        if cycle_number == (@report_every - 1) # it's report time!
-          logger.warn(@reports.last.to_s)
-
-          if shutdown_stalled?
-            logger.error("The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.") if stalled_count == 0
-            stalled_count += 1
-
-            if self.class.unsafe_shutdown? && @abort_threshold == stalled_count
-              logger.fatal("Forcefully quitting logstash..")
-              force_exit()
-              break
-            end
-          else
-            stalled_count = 0
-          end
-        end
-        cycle_number = (cycle_number + 1) % @report_every
-      end
-    ensure
-      stop!
-    end
-
-    def pipeline_report_snapshot
-      @pipeline.reporter.snapshot
-    end
-
-    # A pipeline shutdown is stalled if
-    # * at least REPORT_EVERY reports have been created
-    # * the inflight event count is in monotonically increasing
-    # * there are worker threads running which aren't blocked on SizedQueue pop/push
-    # * the stalled thread list is constant in the previous REPORT_EVERY reports
-    def shutdown_stalled?
-      return false unless @reports.size == @report_every #
-      # is stalled if inflight count is either constant or increasing
-      stalled_event_count = @reports.each_cons(2).all? do |prev_report, next_report|
-        prev_report.inflight_count <= next_report.inflight_count
-      end
-      if stalled_event_count
-        @reports.each_cons(2).all? do |prev_report, next_report|
-          prev_report.stalling_threads == next_report.stalling_threads
-        end
-      else
-        false
-      end
-    end
-
-    def force_exit
-      exit(-1)
-    end
-
-    private
-    def running!
-      @running.make_true
-    end
-  end
-end