logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/pipeline_action.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "logstash/pipeline_action/base"
+require "logstash/pipeline_action/create"
+require "logstash/pipeline_action/stop"
+require "logstash/pipeline_action/reload"
+
+module LogStash module PipelineAction
+  ORDERING = {
+    LogStash::PipelineAction::Create => 100,
+    LogStash::PipelineAction::Reload => 200,
+    LogStash::PipelineAction::Stop => 300
+  }
+end end

data/lib/logstash/pipeline_action/base.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+# I've decided to take the action strategy, I think this make the code a bit easier to understand.
+# maybe in the context of config management we will want to have force kill on the
+# threads instead of waiting forever or sending feedback to the host
+#
+# Some actions could be retryable, or have a delay or timeout.
+module LogStash module PipelineAction
+  class Base
+    # Only used for debugging purpose and in the logger statement.
+    def inspect
+      "#{self.class.name}/pipeline_id:#{pipeline_id}"
+    end
+    alias_method :to_s, :inspect
+
+    def execute(agent, pipelines)
+      raise "`#execute` Not implemented!"
+    end
+
+    # See the definition in `logstash/pipeline_action.rb` for the default ordering
+    def execution_priority
+      ORDERING.fetch(self.class)
+    end
+
+    def <=>(other)
+      order = self.execution_priority <=> other.execution_priority
+      order.nonzero? ? order : self.pipeline_id <=> other.pipeline_id
+    end
+  end
+end end

data/lib/logstash/pipeline_action/create.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/pipeline_action/base"
+require "logstash/pipeline"
+require "logstash/converge_result"
+require "logstash/util/loggable"
+
+module LogStash module PipelineAction
+  class Create < Base
+    include LogStash::Util::Loggable
+
+    # We currently pass around the metric object again this
+    # is needed to correctly create a pipeline, in a future
+    # PR we could pass a factory to create the pipeline so we pass the logic
+    # to create the pipeline instead.
+    def initialize(pipeline_config, metric)
+      @pipeline_config = pipeline_config
+      @metric = metric
+    end
+
+    def pipeline_id
+      @pipeline_config.pipeline_id
+    end
+
+    # Make sure we execution system pipeline like the monitoring
+    # before any user defined pipelines, system pipeline register hooks into the system that will be
+    # triggered by the user defined pipeline.
+    def execution_priority
+      default_priority = super
+      @pipeline_config.system? ? default_priority * -1 : default_priority
+    end
+
+    # The execute assume that the thread safety access of the pipeline
+    # is managed by the caller.
+    def execute(agent, pipelines)
+      pipeline = LogStash::Pipeline.new(@pipeline_config.config_string, @pipeline_config.settings, @metric, agent)
+      
+      status = pipeline.start # block until the pipeline is correctly started or crashed
+
+      if status
+        pipelines[pipeline_id] = pipeline # The pipeline is successfully started we can add it to the hash
+      end
+
+      LogStash::ConvergeResult::ActionResult.create(self, status)
+    end
+
+  end
+end end

data/lib/logstash/pipeline_action/reload.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/pipeline_action/base"
+require "logstash/pipeline_action/create"
+require "logstash/pipeline_action/stop"
+require "logstash/errors"
+require "logstash/util/loggable"
+require "logstash/converge_result"
+
+module LogStash module PipelineAction
+  class Reload < Base
+    include LogStash::Util::Loggable
+
+    def initialize(pipeline_config, metric)
+      @pipeline_config = pipeline_config
+      @metric = metric
+    end
+
+    def pipeline_id
+      @pipeline_config.pipeline_id
+    end
+
+    def execute(agent, pipelines)
+      old_pipeline = pipelines[pipeline_id]
+
+      if !old_pipeline.reloadable?
+        return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
+      end
+
+      begin
+        pipeline_validator = LogStash::BasePipeline.new(@pipeline_config.config_string, @pipeline_config.settings)
+      rescue => e
+        return LogStash::ConvergeResult::FailedAction.from_exception(e)
+      end
+
+      if !pipeline_validator.reloadable?
+        return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the new pipeline is not reloadable")
+      end
+
+      status = Stop.new(pipeline_id).execute(agent, pipelines)
+
+      if status
+        return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
+      else
+        return status
+      end
+    end
+  end
+end end

data/lib/logstash/pipeline_action/stop.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/pipeline_action/base"
+require "logstash/shutdown_watcher"
+require "logstash/converge_result"
+
+module LogStash module PipelineAction
+  class Stop < Base
+    attr_reader :pipeline_id
+
+    def initialize(pipeline_id)
+      @pipeline_id = pipeline_id
+    end
+
+    def execute(agent, pipelines)
+      pipeline = pipelines[pipeline_id]
+      pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
+      pipelines.delete(pipeline_id)
+      # If we reach this part of the code we have succeeded because
+      # the shutdown call will block.
+      return LogStash::ConvergeResult::SuccessfulAction.new
+    end
+  end
+end end

data/lib/logstash/plugin.rb

@@ -3,7 +3,6 @@ require "logstash/namespace"
 require "logstash/logging"
 require "logstash/config/mixin"
 require "logstash/instrument/null_metric"
-require "logstash/util/dead_letter_queue_manager"
 require "concurrent"
 require "securerandom"
 

data/lib/logstash/plugins/hooks_registry.rb

@@ -41,6 +41,12 @@ module LogStash module Plugins
       end
     end
 
+    def registered_hook?(emitter_scope, klass)
+      callbacks = @registered_hooks[emitter_scope]
+      return false if callbacks.nil?
+      callbacks.collect(&:class).include?(klass)
+    end
+
     private
     def sync_hooks
       @registered_emitters.each do |emitter, dispatcher|

data/lib/logstash/plugins/registry.rb

@@ -3,7 +3,6 @@ require "rubygems/package"
 require "logstash/util/loggable"
 require "logstash/plugin"
 require "logstash/plugins/hooks_registry"
-require "logstash/modules/scaffold"
 
 module LogStash module Plugins
   class Registry

data/lib/logstash/program.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash::Program
+  public
+  def exit(value)
+    if RUBY_ENGINE == "jruby"
+      # Kernel::exit() in jruby just tosses an exception? Let's actually exit.
+      Java::java.lang.System.exit(value)
+    else
+      Kernel::exit(value)
+    end
+  end # def exit
+end # module LogStash::Program

data/lib/logstash/queue_factory.rb

@@ -19,6 +19,10 @@ module LogStash
       queue_path = ::File.join(settings.get("path.queue"), settings.get("pipeline.id"))
 
       case queue_type
+      when "memory_acked"
+        # memory_acked is used in tests/specs
+        FileUtils.mkdir_p(queue_path)
+        LogStash::Util::WrappedAckedQueue.create_memory_based(queue_path, queue_page_capacity, queue_max_events, queue_max_bytes)
       when "persisted"
         # persisted is the disk based acked queue
         FileUtils.mkdir_p(queue_path)
@@ -27,7 +31,7 @@ module LogStash
         # memory is the legacy and default setting
         LogStash::Util::WrappedSynchronousQueue.new
       else
-        raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are 'memory' and 'persisted'"
+        raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are: 'memory_acked', 'memory', 'persisted'"
       end
     end
   end

data/lib/logstash/runner.rb

@@ -9,19 +9,20 @@ require "net/http"
 require "logstash/namespace"
 require "logstash-core/logstash-core"
 require "logstash/environment"
-require "logstash/modules/cli_parser"
 
 LogStash::Environment.load_locale!
 
 require "logstash/agent"
 require "logstash/config/defaults"
-require "logstash/config/modules_common"
 require "logstash/shutdown_watcher"
 require "logstash/patches/clamp"
 require "logstash/settings"
 require "logstash/version"
 require "logstash/plugins/registry"
-require "logstash/modules/util"
+require "logstash/bootstrap_check/default_config"
+require "logstash/bootstrap_check/bad_java"
+require "logstash/bootstrap_check/bad_ruby"
+require "set"
 
 java_import 'org.logstash.FileLockFactory'
 
@@ -34,6 +35,14 @@ class LogStash::Runner < Clamp::StrictCommand
   LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
   LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
 
+  # Ordered list of check to run before starting logstash
+  # theses checks can be changed by a plugin loaded into memory.
+  DEFAULT_BOOTSTRAP_CHECKS = [
+      LogStash::BootstrapCheck::BadRuby,
+      LogStash::BootstrapCheck::BadJava,
+      LogStash::BootstrapCheck::DefaultConfig
+  ]
+
   # Node Settings
   option ["-n", "--node.name"], "NAME",
     I18n.t("logstash.runner.flag.name"),
@@ -52,22 +61,6 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("config.string"),
     :attribute_name => "config.string"
 
-  # Module settings
-  option ["--modules"], "MODULES",
-    I18n.t("logstash.runner.flag.modules"),
-    :multivalued => true,
-    :attribute_name => "modules_list"
-
-  option ["-M", "--modules.variable"], "MODULES_VARIABLE",
-    I18n.t("logstash.runner.flag.modules_variable"),
-    :multivalued => true,
-    :attribute_name => "modules_variable_list"
-
-  option ["--setup"], :flag,
-    I18n.t("logstash.runner.flag.modules_setup"),
-    :default => LogStash::SETTINGS.get_default("modules_setup"),
-    :attribute_name => "modules_setup"
-
   # Pipeline settings
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
@@ -172,10 +165,16 @@ class LogStash::Runner < Clamp::StrictCommand
     I18n.t("logstash.runner.flag.quiet"),
     :new_flag => "log.level", :new_value => "error"
 
-  attr_reader :agent
+  attr_reader :agent, :settings
+  attr_accessor :bootstrap_checks
 
   def initialize(*args)
     @settings = LogStash::SETTINGS
+    @bootstrap_checks = DEFAULT_BOOTSTRAP_CHECKS.dup
+
+    # Default we check local sources: `-e`, `-f` and the logstash.yml options.
+    LogStash::Config::SOURCE_LOADER.add_source(LogStash::Config::Source::Local.new(@settings))
+
     super(*args)
   end
 
@@ -221,11 +220,7 @@ class LogStash::Runner < Clamp::StrictCommand
     java.lang.System.setProperty("ls.log.level", setting("log.level"))
     unless java.lang.System.getProperty("log4j.configurationFile")
       log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
-
-      # Windows safe way to produce a file: URI.
-      file_schema = "file://" + (LogStash::Environment.windows? ? "/" : "")
-      LogStash::Logging::Logger::reconfigure(URI.join(file_schema + File.absolute_path(log4j_config_location)).to_s)
-
+      LogStash::Logging::Logger::initialize("file:///" + log4j_config_location)
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
@@ -234,70 +229,55 @@ class LogStash::Runner < Clamp::StrictCommand
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
 
-    # Add local modules to the registry before everything else
-    LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME)
+    # Skip any validation and just return the version
+    if version?
+      show_version
+      return 0
+    end
 
     # We configure the registry and load any plugin that can register hooks
     # with logstash, this need to be done before any operation.
     LogStash::PLUGIN_REGISTRY.setup!
-    @settings.validate_all
 
-    LogStash::Util::set_thread_name(self.class.name)
+    @dispatcher = LogStash::EventDispatcher.new(self)
+    LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
 
-    if RUBY_VERSION < "1.9.2"
-      logger.fatal "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
-      return 1
-    end
+    @settings.validate_all
+    @dispatcher.fire(:before_bootstrap_checks)
+
+    return start_shell(setting("interactive"), binding) if setting("interactive")
 
-    # Exit on bad java versions
-    java_version = LogStash::Util::JavaVersion.version
-    if LogStash::Util::JavaVersion.bad_java_version?(java_version)
-      logger.fatal "Java version 1.8.0 or later is required. (You are running: #{java_version})"
+    begin
+      @bootstrap_checks.each { |bootstrap| bootstrap.check(@settings) }
+    rescue LogStash::BootstrapCheckError => e
+      signal_usage_error(e.message)
       return 1
     end
+    @dispatcher.fire(:after_bootstrap_checks)
 
-    module_parser = LogStash::Modules::CLIParser.new(@modules_list, @modules_variable_list)
-    # Now populate Setting for modules.list with our parsed array.
-    @settings.set("modules.cli", module_parser.output)
+    LogStash::Util::set_thread_name(self.class.name)
 
     LogStash::ShutdownWatcher.unsafe_shutdown = setting("pipeline.unsafe_shutdown")
 
     configure_plugin_paths(setting("path.plugins"))
 
-    if version?
-      show_version
-      return 0
-    end
-
-    return start_shell(setting("interactive"), binding) if setting("interactive")
 
     @settings.format_settings.each {|line| logger.debug(line) }
 
-    module_configs = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
-    module_config_hash = module_configs.first
-    if !module_config_hash.nil?
-      @settings.set_value("config.string", module_config_hash["config_string"])
-    end
-    if module_configs.size > 1
-      logger.warn "Multiple modules defined in logstash.yml - using the first one: #{module_config_hash["pipeline_id"]}"
-    end
-
-    if setting("config.string").nil? && setting("path.config").nil?
-      fail(I18n.t("logstash.runner.missing-configuration"))
-    end
-
-    if setting("config.reload.automatic") && setting("path.config").nil?
-      # there's nothing to reload
-      signal_usage_error(I18n.t("logstash.runner.reload-without-config-path"))
-    end
-
     if setting("config.test_and_exit")
-      config_loader = LogStash::Config::Loader.new(logger)
-      config_str = config_loader.format_config(setting("path.config"), setting("config.string"))
       begin
-        LogStash::BasePipeline.new(config_str)
-        puts "Configuration OK"
-        logger.info "Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash"
+        results = LogStash::Config::SOURCE_LOADER.fetch
+
+        # TODO(ph): make it better for multiple pipeline
+        if results.success?
+          results.response.each do |pipeline_config|
+            LogStash::BasePipeline.new(pipeline_config.config_string)
+          end
+          puts "Configuration OK"
+          logger.info "Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash"
+        else
+          raise "Could not load the configuration file"
+        end
         return 0
       rescue => e
         logger.fatal I18n.t("logstash.runner.invalid-configuration", :error => e.message)
@@ -306,11 +286,11 @@ class LogStash::Runner < Clamp::StrictCommand
     end
 
     # lock path.data before starting the agent
-    @data_path_lock = FileLockFactory.obtainLock(java.nio.file.Paths.get(setting("path.data")).to_absolute_path, ".lock")
+    @data_path_lock = FileLockFactory.getDefault().obtainLock(setting("path.data"), ".lock");
 
-    @agent = create_agent(@settings)
-
-    @agent.register_pipeline(@settings)
+    @dispatcher.fire(:before_agent)
+    @agent = create_agent(@settings, LogStash::Config::SOURCE_LOADER)
+    @dispatcher.fire(:after_agent)
 
     # enable sigint/sigterm before starting the agent
     # to properly handle a stalled agent
@@ -351,7 +331,7 @@ class LogStash::Runner < Clamp::StrictCommand
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
     Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
     Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
-    FileLockFactory.releaseLock(@data_path_lock) if @data_path_lock
+    FileLockFactory.getDefault().releaseLock(@data_path_lock) if @data_path_lock
     @log_fd.close if @log_fd
   end # def self.main
 
@@ -427,7 +407,7 @@ class LogStash::Runner < Clamp::StrictCommand
   def trap_sighup
     Stud::trap("HUP") do
       logger.warn(I18n.t("logstash.agent.sighup"))
-      @agent.reload_state!(true)
+      @agent.converge_state_and_update
     end
   end
 
@@ -442,10 +422,8 @@ class LogStash::Runner < Clamp::StrictCommand
     Stud::trap("INT") do
       if @interrupted_once
         logger.fatal(I18n.t("logstash.agent.forced_sigint"))
-        # calling just Kernel.exit only raises SystemExit exception
-        # and doesn't guarantee the process will terminate
-        # We must call Kernel.exit! so java.lang.System.exit is called
-        exit!
+        @agent.force_shutdown!
+        exit
       else
         logger.warn(I18n.t("logstash.agent.sigint"))
         Thread.new(logger) {|lg| sleep 5; lg.warn(I18n.t("logstash.agent.slow_shutdown")) }
@@ -476,7 +454,7 @@ class LogStash::Runner < Clamp::StrictCommand
       nil
     end
   end
-
+  
   # is the user asking for CLI help subcommand?
   def cli_help?(args)
     # I know, double negative