logstash-core 6.8.17-java → 7.0.0.alpha1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b8da19f938e6b15255f8088a1fce18d781bad1dfcc2bbb6d616d242a9ebb16d
-  data.tar.gz: c9c9ceb47edce2e36116239bb7e7dfb20d3fbafb68caec24cf6803d67bfecc55
+  metadata.gz: aac6580e3fcdb7a4ef97667263de8081ef5f747948508322dc44dcf58f649102
+  data.tar.gz: 62fe96390ead958576494f6507200be79183eb4d36daf3724b9b3f9116f686ea
 SHA512:
-  metadata.gz: 5fe210acf54443eaa7adaffb5564215ce354b2b3f0e78e1205a32a551f0276125961621405cd931cef25d951f41cb50ab1d206a8556d0de11aac4dbf7904f847
-  data.tar.gz: a890dc1973a8f5df0598b84d95cc7ee04d5b78f2e6d0c4e40b2d16a8e7b7ed67f80c44bf4b2d7af4e88db97d0dab94098859246ae4f82a650b041a3d7c4160f6
+  metadata.gz: 551463904e10b3d091962829e5f55c6a32435d6f39799d725a72c8b5fa7bf5051abc4cfdd2ab6e8b509a70d28b33e9cce1deedaae16e70fefd49fb7100a61b07
+  data.tar.gz: '08c65c30b5223731ff293615a5c321262eb2e93d6fa896d25b064ccc52de2e2b9b877e01976b156919f3e0bf4852f3c3ae261f40375332217ad246ebbb982d85'

data/lib/logstash-core/version.rb

@@ -7,6 +7,8 @@ if !defined?(ALL_VERSIONS)
   require 'yaml'
   ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
 end
+
 if !defined?(LOGSTASH_CORE_VERSION)
-  LOGSTASH_CORE_VERSION = ALL_VERSIONS.fetch("logstash-core")
+  # PACKAGE_SUFFIX is declared in the artifact namespace from artifacts.rake
+  LOGSTASH_CORE_VERSION = defined?(PACKAGE_SUFFIX) ? "#{ALL_VERSIONS.fetch("logstash-core")}#{PACKAGE_SUFFIX}" : ALL_VERSIONS.fetch("logstash-core")
 end

data/lib/logstash/agent.rb

@@ -8,7 +8,6 @@ require "logstash/webserver"
 require "logstash/config/source_loader"
 require "logstash/pipeline_action"
 require "logstash/state_resolver"
-require "logstash/pipelines_registry"
 require "stud/trap"
 require "uri"
 require "socket"
@@ -20,7 +19,7 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :name, :settings, :dispatcher, :ephemeral_id, :pipeline_bus
+  attr_reader :metric, :name, :settings, :webserver, :dispatcher, :ephemeral_id, :pipelines, :pipeline_bus
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -37,14 +36,11 @@ class LogStash::Agent
     # Mutex to synchonize in the exclusive method
     # Initial usage for the Ruby pipeline initialization which is not thread safe
     @exclusive_lock = Mutex.new
-    @webserver_control_lock = Mutex.new
-
-    @convergence_lock = Mutex.new
 
     # Special bus object for inter-pipelines communications. Used by the `pipeline` input/output
     @pipeline_bus = org.logstash.plugins.pipeline.PipelineBus.new
 
-    @pipelines_registry = LogStash::PipelinesRegistry.new
+    @pipelines = java.util.concurrent.ConcurrentHashMap.new();
 
     @name = setting("node.name")
     @http_host = setting("http.host")
@@ -54,14 +50,10 @@ class LogStash::Agent
     id
 
     # Set the global FieldReference parsing mode
-    parsing_mode = case setting('config.field_reference.parser')
-                   when 'COMPAT' then org.logstash.FieldReference::ParsingMode::COMPAT;
-                   when 'LEGACY' then org.logstash.FieldReference::ParsingMode::LEGACY;
-                   when 'STRICT' then org.logstash.FieldReference::ParsingMode::STRICT;
-                   else fail('Unsupported FieldReference parsing mode')
-                   end
-    logger.debug("Setting global FieldReference parsing mode: #{parsing_mode}")
-    org.logstash.FieldReference::set_parsing_mode(parsing_mode)
+    if @settings.set?('config.field_reference.parser')
+      # TODO: i18n
+      logger.warn("deprecated setting `config.field_reference.parser` set; field reference parsing is strict by default")
+    end
 
     # This is for backward compatibility in the tests
     if source_loader.nil?
@@ -122,17 +114,14 @@ class LogStash::Agent
         converge_state_and_update unless stopped?
       end
     else
-      # exit with error status if the initial converge_state_and_update did not create any pipeline
-      return 1 if @pipelines_registry.empty?
+      return 1 if clean_state?
 
       while !Stud.stop?
-        # exit if all pipelines are terminated and none are reloading
-        break if no_pipeline?
-
-        # exit if there are no user defined pipelines (not system pipeline) and none are reloading
-        break if !running_user_defined_pipelines?
-
-        sleep(0.5)
+        if clean_state? || running_user_defined_pipelines?
+          sleep(0.5)
+        else
+          break
+        end
       end
     end
 
@@ -146,11 +135,11 @@ class LogStash::Agent
   end
 
   def running?
-    @running.true?
+    @running.value
   end
 
   def stopped?
-    @running.false?
+    !@running.value
   end
 
   def converge_state_and_update
@@ -165,7 +154,12 @@ class LogStash::Agent
       end
     end
 
-    converge_result = resolve_actions_and_converge_state(results.response)
+    # We Lock any access on the pipelines, since the actions will modify the
+    # content of it.
+    converge_result = nil
+
+    pipeline_actions = resolve_actions(results.response)
+    converge_result = converge_state(pipeline_actions)
     update_metrics(converge_result)
 
     logger.info(
@@ -184,7 +178,7 @@ class LogStash::Agent
 
   # Calculate the Logstash uptime in milliseconds
   #
-  # @return [Integer] Uptime in milliseconds
+  # @return [Fixnum] Uptime in milliseconds
   def uptime
     ((Time.now.to_f - STARTED_AT.to_f) * 1000.0).to_i
   end
@@ -239,48 +233,43 @@ class LogStash::Agent
     @id_path ||= ::File.join(settings.get("path.data"), "uuid")
   end
 
-  #
-  # Backward compatibility proxies to the PipelineRegistry
-  #
-
   def get_pipeline(pipeline_id)
-    @pipelines_registry.get_pipeline(pipeline_id)
+    pipelines.get(pipeline_id)
   end
 
   def pipelines_count
-    @pipelines_registry.size
+    pipelines.size
   end
 
   def running_pipelines
-    @pipelines_registry.running_pipelines
-   end
+    pipelines.select {|id,pipeline| running_pipeline?(id) }
+  end
 
   def non_running_pipelines
-    @pipelines_registry.non_running_pipelines
+    pipelines.select {|id,pipeline| !running_pipeline?(id) }
   end
 
   def running_pipelines?
-    @pipelines_registry.running_pipelines.any?
+    running_pipelines_count > 0
   end
 
   def running_pipelines_count
-    @pipelines_registry.running_pipelines.size
+    running_pipelines.size
   end
 
   def running_user_defined_pipelines?
-    @pipelines_registry.running_user_defined_pipelines.any?
+    !running_user_defined_pipelines.empty?
   end
 
   def running_user_defined_pipelines
-    @pipelines_registry.running_user_defined_pipelines
+    pipelines.select {|id, pipeline| running_pipeline?(id) && !pipeline.system? }
   end
 
-  def no_pipeline?
-    @pipelines_registry.running_pipelines.empty?
+  def with_running_user_defined_pipelines
+    yield running_user_defined_pipelines
   end
 
   private
-
   def transition_to_stopped
     @running.make_false
   end
@@ -289,15 +278,6 @@ class LogStash::Agent
     @running.make_true
   end
 
-  # @param pipeline_configs [Array<Config::PipelineConfig>]
-  # @return [ConvergeResult]
-  def resolve_actions_and_converge_state(pipeline_configs)
-    @convergence_lock.synchronize do
-      pipeline_actions = resolve_actions(pipeline_configs)
-      converge_state(pipeline_actions)
-    end
-  end
-
   # We depends on a series of task derived from the internal state and what
   # need to be run, theses actions are applied to the current pipelines to converge to
   # the desired state.
@@ -310,12 +290,11 @@ class LogStash::Agent
   #
   def converge_state(pipeline_actions)
     logger.debug("Converging pipelines state", :actions_count => pipeline_actions.size)
-    fail("Illegal access to `LogStash::Agent#converge_state()` without exclusive lock at #{caller[1]}") unless @convergence_lock.owned?
 
     converge_result = LogStash::ConvergeResult.new(pipeline_actions.size)
 
     pipeline_actions.map do |action|
-      Thread.new(action, converge_result) do |action, converge_result|
+      Thread.new do
         java.lang.Thread.currentThread().setName("Converge #{action}");
         # We execute every task we need to converge the current state of pipelines
         # for every task we will record the action result, that will help us
@@ -331,36 +310,34 @@ class LogStash::Agent
         # that we currently have.
         begin
           logger.debug("Executing action", :action => action)
-          action_result = action.execute(self, @pipelines_registry)
+          action_result = action.execute(self, pipelines)
           converge_result.add(action, action_result)
 
           unless action_result.successful?
-            logger.error("Failed to execute action",
-              :id => action.pipeline_id,
-              :action_type => action_result.class,
-              :message => action_result.message,
-              :backtrace => action_result.backtrace
-            )
+            logger.error("Failed to execute action", :id => action.pipeline_id,
+                         :action_type => action_result.class, :message => action_result.message,
+                         :backtrace => action_result.backtrace)
           end
-        rescue SystemExit, Exception => e
+        rescue SystemExit => e
+          converge_result.add(action, e)
+        rescue Exception => e
           logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
           converge_result.add(action, e)
         end
       end
     end.each(&:join)
 
-    logger.trace? && logger.trace("Converge results",
-      :success => converge_result.success?,
-      :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
-      :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" }
-    )
+    if logger.trace?
+      logger.trace("Converge results", :success => converge_result.success?,
+                   :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
+                   :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" })
+    end
 
     converge_result
   end
 
   def resolve_actions(pipeline_configs)
-    fail("Illegal access to `LogStash::Agent#resolve_actions()` without exclusive lock at #{caller[1]}") unless @convergence_lock.owned?
-    @state_resolver.resolve(@pipelines_registry, pipeline_configs)
+    @state_resolver.resolve(@pipelines, pipeline_configs)
   end
 
   def dispatch_events(converge_results)
@@ -377,24 +354,20 @@ class LogStash::Agent
   end
 
   def start_webserver
-    @webserver_control_lock.synchronize do
-      options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
-      @webserver = LogStash::WebServer.new(@logger, self, options)
-      @webserver_thread = Thread.new(@webserver) do |webserver|
-        LogStash::Util.set_thread_name("Api Webserver")
-        webserver.run
-      end
+    options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
+    @webserver = LogStash::WebServer.new(@logger, self, options)
+    @webserver_thread = Thread.new(@webserver) do |webserver|
+      LogStash::Util.set_thread_name("Api Webserver")
+      webserver.run
     end
   end
 
   def stop_webserver
-    @webserver_control_lock.synchronize do
-      if @webserver
-        @webserver.stop
-        if @webserver_thread.join(5).nil?
-          @webserver_thread.kill
-          @webserver_thread.join
-        end
+    if @webserver
+      @webserver.stop
+      if @webserver_thread.join(5).nil?
+        @webserver_thread.kill
+        @webserver_thread.join
       end
     end
   end
@@ -422,14 +395,25 @@ class LogStash::Agent
   end
 
   def shutdown_pipelines
-    logger.debug("Shutting down all pipelines", :pipelines_count => running_pipelines_count)
+    logger.debug("Shutting down all pipelines", :pipelines_count => pipelines_count)
 
     # In this context I could just call shutdown, but I've decided to
     # use the stop action implementation for that so we have the same code.
     # This also give us some context into why a shutdown is failing
-    resolve_actions_and_converge_state([]) # We stop all the pipeline, so we converge to a empty state
+    pipeline_actions = resolve_actions([]) # We stop all the pipeline, so we converge to a empty state
+    converge_state(pipeline_actions)
   end
 
+  def running_pipeline?(pipeline_id)
+    pipeline = get_pipeline(pipeline_id)
+    return false unless pipeline
+    thread = pipeline.thread
+    thread.is_a?(Thread) && thread.alive?
+  end
+
+  def clean_state?
+    pipelines.empty?
+  end
 
   def setting(key)
     @settings.get(key)

data/lib/logstash/api/modules/stats.rb

@@ -10,7 +10,7 @@ module LogStash
         # return hot threads information
         get "/jvm/hot_threads" do
           begin
-            top_threads_count = params["threads"] || 10
+            top_threads_count = params["threads"] || 3
             ignore_idle_threads = params["ignore_idle_threads"] || true
             options = {
               :threads => top_threads_count.to_i,

data/lib/logstash/compiler/lscl.rb

@@ -3,7 +3,6 @@ require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
 require "logstash/compiler/lscl/helpers"
 require "logstash/config/string_escape"
-require "logstash/util"
 
 java_import org.logstash.config.ir.DSL
 java_import org.logstash.common.SourceWithMetadata
@@ -112,11 +111,11 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
           # hash value; e.g., `{"match" => {"baz" => "bar"}, "match" => {"foo" => "bulb"}}` is
           # interpreted as `{"match" => {"baz" => "bar", "foo" => "blub"}}`.
           # (NOTE: this bypasses `AST::Hash`'s ability to detect duplicate keys)
-          hash[k] = ::LogStash::Util.hash_merge_many(existing, v)
+          hash[k] = existing.merge(v)
         elsif existing.kind_of?(::Array)
           hash[k] = existing.push(*v)
         else
-          hash[k] = [existing, v] unless v == existing
+          hash[k] = existing + v
         end
         hash
       end
@@ -165,8 +164,8 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
   class Number < Value
     def expr
       jdsl.eValue(source_meta, text_value.include?(".") ?
-                                   Float(text_value) :
-                                   Integer(text_value))
+        text_value.to_f :
+        text_value.to_i)
     end
   end
 
@@ -318,9 +317,10 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
 
     def precedence(op)
       #  Believe this is right for logstash?
-      if op == AND_METHOD
+      case op
+      when AND_METHOD
         2
-      elsif op == OR_METHOD
+      when OR_METHOD
         1
       else
         raise ArgumentError, "Unexpected operator #{op}"

data/lib/logstash/config/config_ast.rb

@@ -136,7 +136,7 @@ module LogStash; module Config; module AST
               events.each{|e| block.call(e)}
             end
 
-            if !@generated_objects[:#{name}].nil? && @generated_objects[:#{name}].has_flush
+            if @generated_objects[:#{name}].respond_to?(:flush)
               @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
               @shutdown_flushers << @generated_objects[:#{name}_flush]
             end

data/lib/logstash/config/mixin.rb

@@ -323,7 +323,7 @@ module LogStash::Config::Mixin
       if config_settings[:list]
         value = Array(value) # coerce scalars to lists
         # Empty lists are converted to nils
-        return true, [] if value.empty?
+        return true, nil if value.empty?
 
         validated_items = value.map {|v| validate_value(v, config_val)}
         is_valid = validated_items.all? {|sr| sr[0] }

data/lib/logstash/config/modules_common.rb

@@ -27,7 +27,7 @@ module LogStash module Config
           end
 
       if modules_array.empty?
-        # no specifed modules
+        # no specified modules
         return pipelines
       end
       logger.debug("Specified modules", :modules_array => modules_array.to_s)
@@ -73,9 +73,9 @@ module LogStash module Config
           LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash)
           current_module.with_settings(module_hash)
           config_test = settings.get("config.test_and_exit")
-          modul_setup = settings.get("modules_setup")
+          module_setup = settings.get("modules_setup")
           # Only import data if it's not a config test and --setup is true
-          if !config_test && modul_setup
+          if !config_test && module_setup
             logger.info("Setting up the #{module_name} module")
             esclient = LogStash::ElasticsearchClient.build(module_hash)
             kbnclient = LogStash::Modules::KibanaClient.new(module_hash)

data/lib/logstash/dependency_report.rb

@@ -32,9 +32,8 @@ class LogStash::DependencyReport < Clamp::Command
     command = ["./gradlew", "generateLicenseReport", "-PlicenseReportInputCSV=#{ruby_output_path}", "-PlicenseReportOutputCSV=#{output_path}"]
     puts "Executing #{command}"
     system(*command)
-
     if $?.exitstatus != 0
-      raise "generateLicenseReport failed with exit status #{$?.exitstatus}"
+      raise "Could not run gradle java deps! Exit status #{$?.exitstatus}"
     end
 
     nil

data/lib/logstash/environment.rb

@@ -33,7 +33,7 @@ module LogStash
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds
            Setting::Boolean.new("config.support_escapes", false),
-            Setting::String.new("config.field_reference.parser", "COMPAT", true, %w(STRICT COMPAT LEGACY)),
+            Setting::String.new("config.field_reference.parser", "STRICT", true, %w(STRICT)),
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
            Setting::Boolean.new("pipeline.system", false),
@@ -42,7 +42,7 @@ module LogStash
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
            Setting::Numeric.new("pipeline.batch.delay", 50), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
-           Setting::Boolean.new("pipeline.java_execution", false),
+           Setting::Boolean.new("pipeline.java_execution", true),
            Setting::Boolean.new("pipeline.reloadable", true),
                     Setting.new("path.plugins", Array, []),
     Setting::NullableString.new("interactive", nil, false),
@@ -62,7 +62,6 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
-            Setting::Boolean.new("queue.checkpoint.retry", false),
             Setting::Boolean.new("dead_letter_queue.enable", false),
             Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
@@ -164,15 +163,11 @@ module LogStash
     end
 
     def windows?
-      host_os =~ WINDOW_OS_RE
+      RbConfig::CONFIG['host_os'] =~ WINDOW_OS_RE
     end
 
     def linux?
-      host_os =~ LINUX_OS_RE
-    end
-
-    def host_os
-      RbConfig::CONFIG['host_os']
+      RbConfig::CONFIG['host_os'] =~ LINUX_OS_RE
     end
 
     def locales_path(path)