logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/config/source_loader.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "logstash/config/source/local"
+require "logstash/errors"
+require "thread"
+require "set"
+
+module LogStash module Config
+  class SourceLoader
+    class SuccessfulFetch
+      attr_reader :response
+
+      def initialize(response)
+        @response = response
+      end
+
+      def success?
+        true
+      end
+    end
+
+    class FailedFetch
+      attr_reader :error
+
+      def initialize(error)
+        @error = error
+      end
+
+      def success?
+        false
+      end
+    end
+
+    include LogStash::Util::Loggable
+
+    def initialize(settings = LogStash::SETTINGS)
+      @sources_lock = Mutex.new
+      @sources = Set.new
+      @settings = settings
+    end
+
+    # This return a ConfigLoader object that will
+    # abstract the call to the different sources and will return multiples pipeline
+    def fetch
+      sources_loaders = []
+
+      sources do |source|
+        sources_loaders << source if source.match?
+      end
+
+      if sources_loaders.empty?
+        # This shouldn't happen with the settings object or with any external plugins.
+        # but lets add a guard so we fail fast.
+        raise LogStash::InvalidSourceLoaderSettingError, "Can't find an appropriate config loader with current settings"
+      else
+        begin
+          pipeline_configs = sources_loaders
+            .collect { |source| source.pipeline_configs }
+            .compact
+            .flatten
+
+          duplicate_ids = find_duplicate_ids(pipeline_configs)
+
+          if duplicate_ids.any?
+            logger.debug("Fetching pipelines with duplicate ids", duplicate_ids.each { |k, v| v.collect(&:pipeline_id) } )
+            return FailedFetch.new("Found duplicate ids in your source: #{duplicate_ids.keys.sort.join(", ")}")
+          end
+
+          if config_debug?
+            pipeline_configs.each { |pipeline_config| pipeline_config.display_debug_information }
+          end
+
+          if pipeline_configs.empty?
+            logger.error("No configuration found in the configured sources.")
+          end
+
+          SuccessfulFetch.new(pipeline_configs)
+        rescue => e
+          logger.error("Could not fetch all the sources", :exception => e.class, :message => e.message, :backtrace => e.backtrace)
+          FailedFetch.new(e.message)
+        end
+      end
+    end
+
+    def sources
+      @sources_lock.synchronize do
+        if block_given?
+          @sources.each do |source|
+            yield source
+          end
+        else
+          @sources
+        end
+      end
+    end
+
+    def remove_source(klass)
+      @sources_lock.synchronize do
+        @sources.delete_if { |source| source == klass || source.is_a?(klass) }
+      end
+    end
+
+    def configure_sources(new_sources)
+      new_sources = Array(new_sources).to_set
+      logger.debug("Configure sources", :sources => new_sources.collect(&:to_s))
+      @sources_lock.synchronize { @sources = new_sources }
+    end
+
+    def add_source(new_source)
+      logger.debug("Adding source", :source => new_source.to_s)
+      @sources_lock.synchronize { @sources << new_source }
+    end
+
+    private
+    def config_debug?
+      @settings.get_value("config.debug") && logger.debug?
+    end
+
+    def find_duplicate_ids(pipeline_configs)
+      pipeline_configs.group_by { |pipeline_config| pipeline_config.pipeline_id }
+        .select { |group, pipeline_configs| pipeline_configs.size > 1 }
+    end
+  end
+
+  SOURCE_LOADER = SourceLoader.new
+end end

data/lib/logstash/converge_result.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require "logstash/errors"
+
+module LogStash
+  # This class allow us to keep track and uniform all the return values from the
+  # action task
+  class ConvergeResult
+    class ActionResult
+      attr_reader :executed_at
+
+      def initialize
+        @executed_at = LogStash::Timestamp.now
+      end
+
+      # Until all the action have more granularity in the validation
+      # or execution we make the ConvergeResult works with primitives and exceptions
+      def self.create(action, action_result)
+        if action_result.is_a?(ActionResult)
+          action_result
+        elsif action_result.is_a?(Exception)
+          FailedAction.from_exception(action_result)
+        elsif action_result == true
+          SuccessfulAction.new
+        elsif action_result == false
+          FailedAction.from_action(action, action_result)
+        else
+          raise LogStash::Error, "Don't know how to handle `#{action_result.class}` for `#{action}`"
+        end
+      end
+    end
+
+    class FailedAction < ActionResult
+      attr_reader :message, :backtrace
+
+      def initialize(message, backtrace = nil)
+        super()
+
+        @message = message
+        @backtrace = backtrace
+      end
+
+      def self.from_exception(exception)
+        FailedAction.new(exception.message, exception.backtrace)
+      end
+
+      def self.from_action(action, action_result)
+        FailedAction.new("Could not execute action: #{action}, action_result: #{action_result}")
+      end
+
+      def successful?
+        false
+      end
+    end
+
+    class SuccessfulAction < ActionResult
+      def successful?
+        true
+      end
+    end
+
+    def initialize(expected_actions_count)
+      @expected_actions_count = expected_actions_count
+      @actions = {}
+    end
+
+    def add(action, action_result)
+      @actions[action] = ActionResult.create(action, action_result)
+    end
+
+    def failed_actions
+      filter_by_successful_state(false)
+    end
+
+    def successful_actions
+      filter_by_successful_state(true)
+    end
+
+    def complete?
+      total == @expected_actions_count
+    end
+
+    def success?
+      failed_actions.empty? && complete?
+    end
+
+    def fails_count
+      failed_actions.size
+    end
+
+    def success_count
+      successful_actions.size
+    end
+
+    def total
+      @actions.size
+    end
+
+    private
+    def filter_by_successful_state(predicate)
+      @actions.select { |action, action_result| action_result.successful? == predicate }
+    end
+  end
+end

data/lib/logstash/environment.rb

@@ -20,21 +20,18 @@ module LogStash
     Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
     Setting::NullableString.new("config.string", nil, false),
-                    Setting.new("modules.cli", Array, []),
-                    Setting.new("modules", Array, []),
-           Setting::Boolean.new("modules_setup", false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
-           Setting::Boolean.new("config.support_escapes", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
-            Setting::Boolean.new("pipeline.system", false),
+           Setting::Boolean.new("pipeline.system", false),
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
    Setting::PositiveInteger.new("pipeline.output.workers", 1),
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
            Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
+           Setting::Boolean.new("pipeline.reloadable", true),
                     Setting.new("path.plugins", Array, []),
     Setting::NullableString.new("interactive", nil, false),
            Setting::Boolean.new("config.debug", false),
@@ -45,7 +42,7 @@ module LogStash
             Setting::String.new("http.host", "127.0.0.1"),
             Setting::PortRange.new("http.port", 9600..9700),
             Setting::String.new("http.environment", "production"),
-            Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
+            Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
             Setting::Boolean.new("queue.drain", false),
             Setting::Bytes.new("queue.page_capacity", "250mb"),
             Setting::Bytes.new("queue.max_bytes", "1024mb"),
@@ -53,8 +50,6 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
-            Setting::Boolean.new("dead_letter_queue.enable", false),
-            Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
             Setting::TimeValue.new("slowlog.threshold.info", "-1"),
             Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
@@ -64,21 +59,13 @@ module LogStash
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
-  # Compute the default dead_letter_queue path based on `path.data`
-  default_dlq_file_path = ::File.join(SETTINGS.get("path.data"), "dead_letter_queue")
-  SETTINGS.register Setting::WritableDirectory.new("path.dead_letter_queue", default_dlq_file_path)
-
+  
   SETTINGS.on_post_process do |settings|
     # If the data path is overridden but the queue path isn't recompute the queue path
     # We need to do this at this stage because of the weird execution order
     # our monkey-patched Clamp follows
-    if settings.set?("path.data")
-      if !settings.set?("path.queue")
-        settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
-      end
-      if !settings.set?("path.dead_letter_queue")
-        settings.set_value("path.dead_letter_queue", ::File.join(settings.get("path.data"), "dead_letter_queue"))
-      end
+    if settings.set?("path.data") && !settings.set?("path.queue")
+      settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
     end
   end
 

data/lib/logstash/errors.rb

@@ -6,8 +6,10 @@ module LogStash
   class PluginLoadingError < Error; end
   class ShutdownSignal < StandardError; end
   class PluginNoVersionError < Error; end
+  class BootstrapCheckError < Error; end
 
   class Bug < Error; end
   class ThisMethodWasRemoved < Bug; end
   class ConfigLoadingError < Error; end
+  class InvalidSourceLoaderSettingError < Error; end
 end

data/lib/logstash/execution_context.rb

@@ -1,16 +1,13 @@
 # encoding: utf-8
-require "logstash/util/dead_letter_queue_manager"
 module LogStash
   class ExecutionContext
-    attr_reader :pipeline, :dlq_writer
+    attr_reader :pipeline, :agent
 
-    def initialize(pipeline, plugin_id, plugin_type, dlq_writer)
+    def initialize(pipeline, agent)
       @pipeline = pipeline
-      @plugin_id = plugin_id
-      @plugin_type = plugin_type
-      @dlq_writer = LogStash::Util::PluginDeadLetterQueueWriter.new(dlq_writer, @plugin_id, @plugin_type)
+      @agent = agent
     end
-
+    
     def pipeline_id
       @pipeline.pipeline_id
     end

data/lib/logstash/filter_delegator.rb

@@ -26,9 +26,6 @@ module LogStash
       @filter.execution_context = execution_context
 
       @metric_events = namespaced_metric.namespace(:events)
-      @metric_events_in = @metric_events.counter(:in)
-      @metric_events_out = @metric_events.counter(:out)
-      @metric_events_time = @metric_events.counter(:duration_in_millis)
       namespaced_metric.gauge(:name, config_name)
 
       # Not all the filters will do bufferings
@@ -40,19 +37,19 @@ module LogStash
     end
 
     def multi_filter(events)
-      @metric_events_in.increment(events.size)
+      @metric_events.increment(:in, events.size)
 
-      start_time = java.lang.System.current_time_millis
+      clock = @metric_events.time(:duration_in_millis)
       new_events = @filter.multi_filter(events)
-      @metric_events_time.increment(java.lang.System.current_time_millis - start_time)
+      clock.stop
 
       # There is no guarantee in the context of filter
       # that EVENTS_INT == EVENTS_OUT, see the aggregates and
       # the split filter
       c = new_events.count { |event| !event.cancelled? }
+      @metric_events.increment(:out, c) if c > 0
 
-      @metric_events_out.increment(c) if c > 0
-      new_events
+      return new_events
     end
 
     private
@@ -64,7 +61,7 @@ module LogStash
 
         # Filter plugins that does buffering or spooling of events like the
         # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
-        @metric_events_out.increment(new_events.size) if new_events && new_events.size > 0
+        @metric_events.increment(:out, new_events.size) if new_events && new_events.size > 0
         new_events
       end
     end

data/lib/logstash/inputs/base.rb

@@ -105,8 +105,6 @@ class LogStash::Inputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
-    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
-    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/instrument/collector.rb

@@ -33,7 +33,11 @@ module LogStash module Instrument
     #
     def push(namespaces_path, key, type, *metric_type_params)
       begin
-        get(namespaces_path, key, type).execute(*metric_type_params)
+        metric = @metric_store.fetch_or_store(namespaces_path, key) do
+          LogStash::Instrument::MetricType.create(type, namespaces_path, key)
+        end
+
+        metric.execute(*metric_type_params)
       rescue MetricStore::NamespacesExpectedError => e
         logger.error("Collector: Cannot record metric", :exception => e)
       rescue NameError => e
@@ -47,12 +51,6 @@ module LogStash module Instrument
       end
     end
 
-    def get(namespaces_path, key, type)
-      @metric_store.fetch_or_store(namespaces_path, key) do
-        LogStash::Instrument::MetricType.create(type, namespaces_path, key)
-      end
-    end
-
     # Snapshot the current Metric Store and return it immediately,
     # This is useful if you want to get access to the current metric store without
     # waiting for a periodic call.

data/lib/logstash/instrument/metric_store.rb

@@ -41,26 +41,26 @@ module LogStash module Instrument
     # @param [Symbol] The metric key
     # @return [Object] Return the new_value of the retrieve object in the tree
     def fetch_or_store(namespaces, key, default_value = nil)
+      provided_value =  block_given? ? yield(key) : default_value
 
       # We first check in the `@fast_lookup` store to see if we have already see that metrics before,
       # This give us a `o(1)` access, which is faster than searching through the structured
       # data store (Which is a `o(n)` operation where `n` is the number of element in the namespace and
-      # the value of the key). If the metric is already present in the `@fast_lookup`, then that value is sent
-      # back directly to the caller.
+      # the value of the key). If the metric is already present in the `@fast_lookup`, the call to
+      # `#put_if_absent` will return the value. This value is send back directly to the caller.
       #
-      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and we assume that we don't
+      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
+      # `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
       # have it in the `@metric_store` for structured search so we add it there too.
-
-      value = @fast_lookup.get(namespaces.dup << key)
-      if value.nil?
-        value = block_given? ? yield(key) : default_value
-        @fast_lookup.put(namespaces.dup << key, value)
+      if found_value = @fast_lookup.put_if_absent(namespaces.dup << key, provided_value)
+        return found_value
+      else
         @structured_lookup_mutex.synchronize do
-            # If we cannot find the value this mean we need to save it in the store.
-          fetch_or_store_namespaces(namespaces).fetch_or_store(key, value)
+          # If we cannot find the value this mean we need to save it in the store.
+          fetch_or_store_namespaces(namespaces).fetch_or_store(key, provided_value)
         end
+        return provided_value
       end
-      return value;
     end
 
     # This method allow to retrieve values for a specific path,
@@ -222,7 +222,7 @@ module LogStash module Instrument
       key_candidates = extract_filter_keys(key_paths.shift)
 
       key_candidates.each do |key_candidate|
-        raise MetricNotFound, "For path: #{key_candidate}" if map[key_candidate].nil?
+        raise MetricNotFound, "For path: #{key_candidate}. Map keys: #{map.keys}" if map[key_candidate].nil?
 
         if key_paths.empty? # End of the user requested path
           if map[key_candidate].is_a?(Concurrent::Map)