logstash-core 6.8.17-java → 7.0.0.alpha1-java

data/lib/logstash/pipeline_action/base.rb

@@ -12,7 +12,7 @@ module LogStash module PipelineAction
     end
     alias_method :to_s, :inspect
 
-    def execute(agent, pipelines_registry)
+    def execute(agent, pipelines)
       raise "`#execute` Not implemented!"
     end
 

data/lib/logstash/pipeline_action/create.rb

@@ -3,7 +3,6 @@ require "logstash/pipeline_action/base"
 require "logstash/pipeline"
 require "logstash/java_pipeline"
 
-
 module LogStash module PipelineAction
   class Create < Base
     include LogStash::Util::Loggable
@@ -31,26 +30,33 @@ module LogStash module PipelineAction
 
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
-    def execute(agent, pipelines_registry)
-      new_pipeline =
+    def execute(agent, pipelines)
+      pipeline =
         if @pipeline_config.settings.get_value("pipeline.java_execution")
           LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
         else
           agent.exclusive do
             # The Ruby pipeline initialization is not thread safe because of the module level
             # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
-            # executed simultaneously in different threads and we need to synchronize this initialization.
+            # executed simultaneously in different threads and we need to synchonize this initialization.
             LogStash::Pipeline.new(@pipeline_config, @metric, agent)
           end
         end
 
-      success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
-        new_pipeline.start # block until the pipeline is correctly started or crashed
+      status = nil
+      pipelines.compute(pipeline_id) do |id,value|
+        if value
+          LogStash::ConvergeResult::ActionResult.create(self, true)
+        end
+        status = pipeline.start # block until the pipeline is correctly started or crashed
+        pipeline # The pipeline is successfully started we can add it to the map
       end
 
-      LogStash::ConvergeResult::ActionResult.create(self, success)
+
+      LogStash::ConvergeResult::ActionResult.create(self, status)
     end
 
+
     def to_s
       "PipelineAction::Create<#{pipeline_id}>"
     end

data/lib/logstash/pipeline_action/reload.rb

@@ -20,12 +20,8 @@ module LogStash module PipelineAction
       "PipelineAction::Reload<#{pipeline_id}>"
     end
 
-    def execute(agent, pipelines_registry)
-      old_pipeline = pipelines_registry.get_pipeline(pipeline_id)
-
-      if old_pipeline.nil?
-        return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
-      end
+    def execute(agent, pipelines)
+      old_pipeline = pipelines[pipeline_id]
 
       if !old_pipeline.reloadable?
         return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
@@ -38,8 +34,8 @@ module LogStash module PipelineAction
           else
             agent.exclusive do
               # The Ruby pipeline initialization is not thread safe because of the module level
-              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
-              # executed simultaneously in different threads and we need to synchronize this initialization.
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
+              # executed simultaneously in different threads and we need to synchonize this initialization.
               LogStash::BasePipeline.new(@pipeline_config)
             end
           end
@@ -53,35 +49,16 @@ module LogStash module PipelineAction
 
       logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
 
-      success = pipelines_registry.reload_pipeline(pipeline_id) do
-        # important NOT to explicitly return from block here
-        # the block must emit a success boolean value
+      pipelines.compute(pipeline_id) do |_,pipeline|
+        status = Stop.new(pipeline_id).execute(agent, pipelines)
 
-        # First shutdown old pipeline
-        old_pipeline.shutdown { LogStash::ShutdownWatcher.start(old_pipeline) }
-        old_pipeline.thread.join
-
-        # Then create a new pipeline
-        new_pipeline =
-          if @pipeline_config.settings.get_value("pipeline.java_execution")
-            LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
-          else
-            agent.exclusive do
-              # The Ruby pipeline initialization is not thread safe because of the module level
-              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
-              # executed simultaneously in different threads and we need to synchronize this initialization.
-              LogStash::Pipeline.new(@pipeline_config, @metric, agent)
-            end
-          end
-
-        success = new_pipeline.start # block until the pipeline is correctly started or crashed
-
-        # return success and new_pipeline to registry reload_pipeline
-        [success, new_pipeline]
+        if status
+          return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
+        else
+          return status
+        end
+        pipeline
       end
-
-      LogStash::ConvergeResult::ActionResult.create(self, success)
     end
-
   end
 end end

data/lib/logstash/pipeline_action/stop.rb

@@ -9,13 +9,15 @@ module LogStash module PipelineAction
       @pipeline_id = pipeline_id
     end
 
-    def execute(agent, pipelines_registry)
-      pipelines_registry.terminate_pipeline(pipeline_id) do |pipeline|
+    def execute(agent, pipelines)
+      pipelines.compute(pipeline_id) do |_,pipeline|
         pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
         pipeline.thread.join
+        nil # delete the pipeline
       end
-
-      LogStash::ConvergeResult::SuccessfulAction.new
+      # If we reach this part of the code we have succeeded because
+      # the shutdown call will block.
+      return LogStash::ConvergeResult::SuccessfulAction.new
     end
 
     def to_s

data/lib/logstash/pipeline_settings.rb

@@ -28,7 +28,6 @@ module LogStash
       "queue.checkpoint.acks",
       "queue.checkpoint.interval",
       "queue.checkpoint.writes",
-      "queue.checkpoint.retry",
       "queue.drain",
       "queue.max_bytes",
       "queue.max_events",
@@ -47,7 +46,7 @@ module LogStash
 
     def register(setting)
       unless SETTINGS_WHITE_LIST.include?(setting.name)
-        raise ArgumentError.new("Only pipeline related settings can be registed in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
+        raise ArgumentError.new("Only pipeline related settings can be registered in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
       end
       super(setting)
     end

data/lib/logstash/plugins/registry.rb

@@ -262,14 +262,11 @@ module LogStash module Plugins
     # @param name [String] plugin name
     # @return [Boolean] true if klass is a valid plugin for name
     def is_a_plugin?(klass, name)
-      (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
-      (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
+      klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
     end
 
     def add_plugin(type, name, klass)
-      if klass.respond_to?("javaClass", true)
-        @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
-      elsif !exists?(type, name)
+      if !exists?(type, name)
         specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
         @registry[key_for(type, name)] = specification_klass.new(type, name, klass)
       else

data/lib/logstash/runner.rb

@@ -276,17 +276,6 @@ class LogStash::Runner < Clamp::StrictCommand
     LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
 
     @settings.validate_all
-
-    if @settings.registered?("xpack.monitoring.enabled")
-      verify_deprecated_setting("xpack.monitoring.elasticsearch.url", "xpack.monitoring.elasticsearch.hosts")
-      verify_deprecated_setting("xpack.monitoring.elasticsearch.ssl.ca", "xpack.monitoring.elasticsearch.ssl.certificate_authority")
-    end
-
-    if @settings.registered?("xpack.management.enabled")
-      verify_deprecated_setting("xpack.management.elasticsearch.url", "xpack.management.elasticsearch.hosts")
-      verify_deprecated_setting("xpack.management.elasticsearch.ssl.ca", "xpack.management.elasticsearch.ssl.certificate_authority")
-    end
-
     @dispatcher.fire(:before_bootstrap_checks)
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
@@ -380,8 +369,6 @@ class LogStash::Runner < Clamp::StrictCommand
 
     @agent.shutdown
 
-    logger.info("Logstash shut down.")
-
     # flush any outstanding log messages during shutdown
     org.apache.logging.log4j.LogManager.shutdown
 
@@ -514,15 +501,4 @@ class LogStash::Runner < Clamp::StrictCommand
     @settings.get_value(key)
   end
 
-  def verify_deprecated_setting(old_setting, new_setting)
-    if @settings.set?(old_setting)
-      logger.warn("Deprecated setting `#{old_setting}` please use `#{new_setting}`")
-      if @settings.set?(new_setting)
-        signal_usage_error("Both `#{new_setting}` and the deprecated `#{old_setting}` settings are set, please remove `#{old_setting}`")
-      else
-        @settings.set_value(new_setting, @settings.get_value(old_setting))
-      end
-    end
-  end
-
 end

data/lib/logstash/settings.rb

@@ -368,7 +368,7 @@ module LogStash
         case value
         when ::Range
           value
-        when ::Integer
+        when ::Fixnum
           value..value
         when ::String
           first, last = value.split(PORT_SEPARATOR)
@@ -481,11 +481,11 @@ module LogStash
 
     class Bytes < Coercible
       def initialize(name, default=nil, strict=true)
-        super(name, ::Integer, default, strict=true) { |value| valid?(value) }
+        super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
       end
 
       def valid?(value)
-        value.is_a?(::Integer) && value >= 0
+        value.is_a?(Fixnum) && value >= 0
       end
 
       def coerce(value)
@@ -508,11 +508,11 @@ module LogStash
 
     class TimeValue < Coercible
       def initialize(name, default, strict=true, &validator_proc)
-        super(name, ::Integer, default, strict, &validator_proc)
+        super(name, ::Fixnum, default, strict, &validator_proc)
       end
 
       def coerce(value)
-        return value if value.is_a?(::Integer)
+        return value if value.is_a?(::Fixnum)
         Util::TimeValue.from_value(value).to_nanos
       end
     end

data/lib/logstash/state_resolver.rb

@@ -10,11 +10,11 @@ module LogStash
       @metric = metric
     end
 
-    def resolve(pipelines_registry, pipeline_configs)
+    def resolve(pipelines, pipeline_configs)
       actions = []
 
       pipeline_configs.each do |pipeline_config|
-        pipeline = pipelines_registry.get_pipeline(pipeline_config.pipeline_id)
+        pipeline = pipelines[pipeline_config.pipeline_id]
 
         if pipeline.nil?
           actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
@@ -25,12 +25,12 @@ module LogStash
         end
       end
 
-      configured_pipelines = pipeline_configs.collect(&:pipeline_id)
+      running_pipelines = pipeline_configs.collect(&:pipeline_id)
 
       # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
       # stop it.
-      pipelines_registry.running_pipelines.keys
-        .select { |pipeline_id| !configured_pipelines.include?(pipeline_id) }
+      pipelines.keys
+        .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
         .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
 
       actions.sort # See logstash/pipeline_action.rb

data/lib/logstash/util.rb

@@ -9,8 +9,6 @@ module LogStash::Util
 
   PR_SET_NAME = 15
   def self.set_thread_name(name)
-    previous_name = Java::java.lang.Thread.currentThread.getName() if block_given?
-
     if RUBY_ENGINE == "jruby"
       # Keep java and ruby thread names in sync.
       Java::java.lang.Thread.currentThread.setName(name)
@@ -23,14 +21,6 @@ module LogStash::Util
       # since MRI 1.9, JRuby, and Rubinius use system threads for this.
       LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
     end
-
-    if block_given?
-      begin
-        yield
-      ensure
-        set_thread_name(previous_name)
-      end
-    end
   end # def set_thread_name
 
   def self.set_thread_plugin(plugin)
@@ -201,7 +191,7 @@ module LogStash::Util
       o.inject({}) {|h, (k,v)| h[k] = deep_clone(v); h }
     when Array
       o.map {|v| deep_clone(v) }
-    when Integer, Symbol, IO, TrueClass, FalseClass, NilClass
+    when Fixnum, Symbol, IO, TrueClass, FalseClass, NilClass
       o
     when LogStash::Codecs::Base
       o.clone

data/lib/logstash/util/duration_formatter.rb

@@ -7,7 +7,7 @@ module LogStash::Util::DurationFormatter
   # a format that a human can understand. This is currently used by
   # the API.
   #
-  # @param [Integer] Duration in milliseconds
+  # @param [Fixnum] Duration in milliseconds
   # @return [String] Duration in human format
   def self.human_format(duration)
     ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)

data/lib/logstash/util/safe_uri.rb

@@ -26,7 +26,6 @@ class LogStash::Util::SafeURI
            else
              raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
            end
-    raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
   end
 
   def to_s

data/lib/logstash/util/substitution_variables.rb

@@ -2,17 +2,12 @@
 
 java_import "org.logstash.secret.store.SecretStoreExt"
 
-require_relative 'lazy_singleton'
-
 module ::LogStash::Util::SubstitutionVariables
 
   include LogStash::Util::Loggable
 
   SUBSTITUTION_PLACEHOLDER_REGEX = /\${(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?}/
 
-  SECRET_STORE = ::LogStash::Util::LazySingleton.new { load_secret_store }
-  private_constant :SECRET_STORE
-
   # Recursive method to replace substitution variable references in parameters
   def deep_replace(value)
     if value.is_a?(Hash)
@@ -47,7 +42,7 @@ module ::LogStash::Util::SubstitutionVariables
       logger.debug("Replacing `#{placeholder}` with actual value")
 
       #check the secret store if it exists
-      secret_store = SECRET_STORE.instance
+      secret_store = SecretStoreExt.getIfExists(LogStash::SETTINGS.get_setting("keystore.file").value, LogStash::SETTINGS.get_setting("keystore.classname").value)
       replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(SecretStoreExt.getStoreId(name))
       #check the environment
       replacement = ENV.fetch(name, default) if replacement.nil?
@@ -59,20 +54,4 @@ module ::LogStash::Util::SubstitutionVariables
     end
   end # def replace_placeholders
 
-  class << self
-    private
-
-    # loads a secret_store from disk if available, or returns nil
-    #
-    # @api private
-    # @return [SecretStoreExt,nil]
-    def load_secret_store
-      SecretStoreExt.getIfExists(LogStash::SETTINGS.get_setting("keystore.file").value, LogStash::SETTINGS.get_setting("keystore.classname").value)
-    end
-
-    # @api test
-    def reset_secret_store
-      SECRET_STORE.reset!
-    end
-  end
 end

data/lib/logstash/util/thread_dump.rb

@@ -5,7 +5,7 @@ module LogStash
   module Util
     class ThreadDump
       SKIPPED_THREADS             = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
-      THREADS_COUNT_DEFAULT       = 10.freeze
+      THREADS_COUNT_DEFAULT       = 3.freeze
       IGNORE_IDLE_THREADS_DEFAULT = true.freeze
 
       attr_reader :top_count, :ignore, :dump

data/locales/en.yml

@@ -229,32 +229,23 @@ en:
           If you wish to use both defaults, please use
           the empty string for the '-e' flag.
         field-reference-parser: |+
+          (DEPRECATED) This option is no longer
+          configurable.
+
           Use the given MODE when parsing field
           references.
 
           The field reference parser is used to expand
           field references in your pipeline configs,
-          and will be becoming more strict to better
-          handle illegal and ambbiguous inputs in a
-          future release of Logstash.
+          and has become more strict to better handle
+          ambiguous- and illegal-syntax inputs.
 
-          Available MODEs are:
-           - `LEGACY`: parse with the legacy parser,
-             which is known to handle ambiguous- and
-             illegal-syntax in surprising ways;
-             warnings will not be emitted.
-           - `COMPAT`: warn once for each distinct
-             ambiguous- or illegal-syntax input, but
-             continue to expand field references with
-             the legacy parser.
+          The only available MODE is:
            - `STRICT`: parse in a strict manner; when
              given ambiguous- or illegal-syntax input,
              raises a runtime exception that should
              be handled by the calling plugin.
 
-           The MODE can also be set with
-           `config.field_reference.parser`
-
         modules: |+
           Load Logstash modules.
           Modules can be defined using multiple instances
@@ -298,7 +289,7 @@ en:
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         java-execution: |+
-          Use Java execution engine.
+          (Beta) Use new Java execution engine.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+