logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/filter_delegator.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+#
+module LogStash
+  class FilterDelegator
+    extend Forwardable
+    DELEGATED_METHODS = [
+      :register,
+      :close,
+      :threadsafe?,
+      :do_close,
+      :do_stop,
+      :periodic_flush
+    ]
+    def_delegators :@filter, *DELEGATED_METHODS
+
+    def initialize(logger, klass, metric, *args)
+      options = args.reduce({}, :merge)
+
+      @logger = logger
+      @klass = klass
+      @filter = klass.new(options)
+
+      # Scope the metrics to the plugin
+      namespaced_metric = metric.namespace(@filter.plugin_unique_name.to_sym)
+      @filter.metric = metric
+
+      @metric_events = namespaced_metric.namespace(:events)
+
+      # Not all the filters will do bufferings
+      define_flush_method if @filter.respond_to?(:flush)
+    end
+
+    def config_name
+      @klass.config_name
+    end
+
+    def multi_filter(events)
+      @metric_events.increment(:in, events.size)
+
+      new_events = @filter.multi_filter(events)
+
+      # There is no garantee in the context of filter
+      # that EVENTS_INT == EVENTS_OUT, see the aggregates and
+      # the split filter
+      c = new_events.count { |event| !event.cancelled? }
+      @metric_events.increment(:out, c) if c > 0
+
+      return new_events
+    end
+
+    private
+    def define_flush_method
+      define_singleton_method(:flush) do |options = {}|
+        # we also need to trace the number of events
+        # coming from a specific filters.
+        new_events = @filter.flush(options)
+
+        # Filter plugins that does buffering or spooling of events like the
+        # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
+        @metric_events.increment(:out, new_events.size) if new_events && new_events.size > 0
+        new_events
+      end
+    end
+  end
+end

data/lib/logstash/inputs/base.rb

@@ -78,7 +78,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
 
   public
   def do_stop
-    @logger.debug("stopping", :plugin => self.class.name)
+    @logger.debug("stopping", :plugin => self)
     @stop_called.make_true
     stop
   end

data/lib/logstash/inputs/metrics.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/inputs/base"
+require "logstash/instrument/collector"
+
+module LogStash module Inputs
+  # The Metrics inputs is responable of registring itself to the collector.
+  # The collector class will periodically emits new snapshot of the system,
+  # The metrics need to take that information and transform it into
+  # a `Logstash::Event`, which can be consumed by the shipper and send to
+  # Elasticsearch
+  class Metrics < LogStash::Inputs::Base
+    config_name "metrics"
+    milestone 3
+
+    def register
+    end
+
+    def run(queue)
+      @logger.debug("Metric: input started")
+      @queue = queue
+
+      # we register to the collector after receiving the pipeline queue
+      LogStash::Instrument::Collector.instance.add_observer(self)
+
+      # Keep this plugin thread alive,
+      # until we shutdown the metric pipeline
+      sleep(1) while !stop?
+    end
+
+    def stop
+      @logger.debug("Metrics input: stopped")
+      LogStash::Instrument::Collector.instance.delete_observer(self)
+    end
+
+    def update(snapshot)
+      @logger.debug("Metrics input: received a new snapshot", :created_at => snapshot.created_at, :snapshot => snapshot, :event => snapshot.metric_store.to_event) if @logger.debug?
+
+      # The back pressure is handled in the collector's
+      # scheduled task (running into his own thread) if something append to one of the listener it will
+      # will timeout. In a sane pipeline, with a low traffic of events it shouldn't be a problems.
+      snapshot.metric_store.each do |metric|
+        @queue << LogStash::Event.new({ "@timestamp" => snapshot.created_at }.merge(metric.to_hash))
+      end
+    end
+  end
+end;end

data/lib/logstash/instrument/collector.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+require "logstash/instrument/snapshot"
+require "logstash/instrument/metric_store"
+require "logstash/util/loggable"
+require "concurrent/timer_task"
+require "observer"
+require "singleton"
+require "thread"
+
+module LogStash module Instrument
+  # The Collector singleton is the single point of reference for all
+  # the metrics collection inside logstash, the metrics library will make
+  # direct calls to this class.
+  #
+  # This class is an observable responsable of periodically emitting view of the system
+  # to other components like the internal metrics pipelines.
+  class Collector
+    include LogStash::Util::Loggable
+    include Observable
+    include Singleton
+
+    SNAPSHOT_ROTATION_TIME_SECS = 1 # seconds
+    SNAPSHOT_ROTATION_TIMEOUT_INTERVAL_SECS = 10 * 60 # seconds
+
+    attr_accessor :agent
+
+    def initialize
+      @metric_store = MetricStore.new
+      @agent = nil
+      start_periodic_snapshotting
+    end
+
+    # The metric library will call this unique interface
+    # its the job of the collector to update the store with new metric
+    # of update the metric
+    #
+    # If there is a problem with the key or the type of metric we will record an error
+    # but we wont stop processing events, theses errors are not considered fatal.
+    #
+    def push(namespaces_path, key, type, *metric_type_params)
+      begin
+        metric = @metric_store.fetch_or_store(namespaces_path, key) do
+          LogStash::Instrument::MetricType.create(type, namespaces_path, key)
+        end
+
+        metric.execute(*metric_type_params)
+
+        changed # we had changes coming in so we can notify the observers
+      rescue MetricStore::NamespacesExpectedError => e
+        logger.error("Collector: Cannot record metric", :exception => e)
+      rescue NameError => e
+        logger.error("Collector: Cannot create concrete class for this metric type",
+                     :type => type,
+                     :namespaces_path => namespaces_path,
+                     :key => key,
+                     :metrics_params => metric_type_params,
+                     :exception => e,
+                     :stacktrace => e.backtrace)
+      end
+    end
+
+    def clear
+      @metric_store = MetricStore.new
+    end
+
+    # Monitor the `Concurrent::TimerTask` this update is triggered on every successful or not
+    # run of the task, TimerTask implement Observable and the collector acts as
+    # the observer and will keep track if something went wrong in the execution.
+    #
+    # @param [Time] Time of execution
+    # @param [result] Result of the execution
+    # @param [Exception] Exception
+    def update(time_of_execution, result, exception)
+      return true if exception.nil?
+      logger.error("Collector: Something went wrong went sending data to the observers",
+                   :execution_time => time_of_execution,
+                   :result => result,
+                   :exception => exception)
+    end
+
+    # Snapshot the current Metric Store and return it immediately,
+    # This is useful if you want to get access to the current metric store without
+    # waiting for a periodic call.
+    #
+    # @return [LogStash::Instrument::MetricStore]
+    def snapshot_metric
+      Snapshot.new(@metric_store)
+    end
+
+    # Configure and start the periodic task for snapshotting the `MetricStore`
+    def start_periodic_snapshotting
+      @snapshot_task = Concurrent::TimerTask.new { publish_snapshot }
+      @snapshot_task.execution_interval = SNAPSHOT_ROTATION_TIME_SECS
+      @snapshot_task.timeout_interval = SNAPSHOT_ROTATION_TIMEOUT_INTERVAL_SECS
+      @snapshot_task.add_observer(self)
+      @snapshot_task.execute
+    end
+
+    # Create a snapshot of the MetricStore and send it to to the registered observers
+    # The observer will receive the following signature in the update methode.
+    #
+    # `#update(created_at, metric_store)`
+    def publish_snapshot
+      created_at = Time.now
+      logger.debug("Collector: Sending snapshot to observers", :created_at => created_at) if logger.debug?
+      notify_observers(snapshot_metric)
+    end
+  end
+end; end

data/lib/logstash/instrument/metric.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+require "logstash/instrument/collector"
+require "concurrent"
+
+module LogStash module Instrument
+  class MetricException < Exception; end
+  class MetricNoKeyProvided < MetricException; end
+  class MetricNoBlockProvided < MetricException; end
+  class MetricNoNamespaceProvided < MetricException; end
+
+  # This class provide the interface between the code, the collector and the format
+  # of the recorded metric.
+  class Metric
+    attr_reader :collector
+
+    def initialize(collector = LogStash::Instrument::Collector.instance)
+      @collector = collector
+    end
+
+    def increment(namespace, key, value = 1)
+      validate_key!(key)
+      collector.push(namespace, key, :counter, :increment, value)
+    end
+
+    def decrement(namespace, key, value = 1)
+      validate_key!(key)
+      collector.push(namespace, key, :counter, :decrement, value)
+    end
+
+    def gauge(namespace, key, value)
+      validate_key!(key)
+      collector.push(namespace, key, :gauge, :set, value)
+    end
+
+    def time(namespace, key)
+      validate_key!(key)
+
+      if block_given?
+        timer = TimedExecution.new(self, namespace, key)
+        content = yield
+        timer.stop
+        return content
+      else
+        TimedExecution.new(self, namespace, key)
+      end
+    end
+
+    def report_time(namespace, key, duration)
+      collector.push(namespace, key, :mean, :increment, duration)
+    end
+
+    # This method return a metric instance tied to a specific namespace
+    # so instead of specifying the namespace on every call.
+    #
+    # Example:
+    #   metric.increment(:namespace, :mykey, 200)
+    #   metric.increment(:namespace, :mykey_2, 200)
+    #
+    #   namespaced_metric = metric.namespace(:namespace)
+    #   namespaced_metric.increment(:mykey, 200)
+    #   namespaced_metric.increment(:mykey_2, 200)
+    # ```
+    #
+    # @param name [Array<String>] Name of the namespace
+    # @param name [String] Name of the namespace
+    def namespace(name)
+      raise MetricNoNamespaceProvided if name.nil? || name.empty?
+
+      NamespacedMetric.new(self, name)
+    end
+
+    private
+    def validate_key!(key)
+      raise MetricNoKeyProvided if key.nil? || key.empty?
+    end
+
+    # Allow to calculate the execution of a block of code.
+    # This class support 2 differents syntax a block or the return of
+    # the object itself, but in the later case the metric wont be recorded
+    # Until we call `#stop`.
+    #
+    # @see LogStash::Instrument::Metric#time
+    class TimedExecution
+      MILLISECONDS = 1_000_000.0.freeze
+
+      def initialize(metric, namespace, key)
+        @metric = metric
+        @namespace = namespace
+        @key = key
+        start
+      end
+
+      def start
+        @start_time = Time.now
+      end
+
+      def stop
+        @metric.report_time(@namespace, @key, (MILLISECONDS * (Time.now - @start_time)).to_i)
+      end
+    end
+  end
+end; end

data/lib/logstash/instrument/metric_store.rb

@@ -0,0 +1,228 @@
+# encoding: utf-8
+require "concurrent"
+require "logstash/event"
+require "logstash/instrument/metric_type"
+
+module LogStash module Instrument
+  # The Metric store the data structure that make sure the data is
+  # saved in a retrievable way, this is a wrapper around multiples ConcurrentHashMap
+  # acting as a tree like structure.
+  class MetricStore
+    class NamespacesExpectedError < StandardError; end
+    class MetricNotFound < StandardError; end
+
+    KEY_PATH_SEPARATOR = "/".freeze
+
+    # Lets me a bit flexible on the coma usage in the path
+    # definition
+    FILTER_KEYS_SEPARATOR = /\s?*,\s*/.freeze
+
+    def initialize
+      # We keep the structured cache to allow
+      # the api to search the content of the differents nodes
+      @store = Concurrent::Map.new
+
+      # This hash has only one dimension
+      # and allow fast retrieval of the metrics
+      @fast_lookup = Concurrent::Map.new
+    end
+
+    # This method use the namespace and key to search the corresponding value of
+    # the hash, if it doesn't exist it will create the appropriate namespaces
+    # path in the hash and return `new_value`
+    #
+    # @param [Array] The path where the values should be located
+    # @param [Symbol] The metric key
+    # @return [Object] Return the new_value of the retrieve object in the tree
+    def fetch_or_store(namespaces, key, default_value = nil)
+      provided_value =  block_given? ? yield(key) : default_value
+
+      # We first check in the `@fast_lookup` store to see if we have already see that metrics before,
+      # This give us a `o(1)` access, which is faster than searching through the structured
+      # data store (Which is a `o(n)` operation where `n` is the number of element in the namespace and
+      # the value of the key). If the metric is already present in the `@fast_lookup`, the call to
+      # `#put_if_absent` will return the value. This value is send back directly to the caller.
+      #
+      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
+      # `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
+      # have it in the `@metric_store` for structured search so we add it there too.
+      #
+      # The problem with only using the `@metric_store` directly all the time would require us
+      # to use the mutex around the structure since its a multi-level hash, without that it wont
+      # return a consistent value of the metric and this would slow down the code and would
+      # complixity the code.
+      if found_value = @fast_lookup.put_if_absent([namespaces, key], provided_value)
+        return found_value
+      else
+        # If we cannot find the value this mean we need to save it in the store.
+        fetch_or_store_namespaces(namespaces).fetch_or_store(key, provided_value)
+        return provided_value
+      end
+    end
+
+    # This method allow to retrieve values for a specific path,
+    # This method support the following queries
+    #
+    # stats/pipelines/pipeline_X
+    # stats/pipelines/pipeline_X,pipeline_2
+    # stats/os,jvm
+    #
+    # If you use the `,` on a key the metric store will return the both values at that level
+    #
+    # The returned hash will keep the same structure as it had in the `Concurrent::Map`
+    # but will be a normal ruby hash. This will allow the api to easily seriliaze the content
+    # of the map
+    #
+    # @param [Array] The path where values should be located
+    # @return [Hash]
+    def get_with_path(path)
+      key_paths = path.gsub(/^#{KEY_PATH_SEPARATOR}+/, "").split(KEY_PATH_SEPARATOR)
+      get(*key_paths)
+    end
+
+    # Similar to `get_with_path` but use symbols instead of string
+    #
+    # @param [Array<Symbol>
+    # @return [Hash]
+    def get(*key_paths)
+      # Normalize the symbols access
+      key_paths.map(&:to_sym)
+      new_hash = Hash.new
+
+      get_recursively(key_paths, @store, new_hash)
+
+      new_hash
+    end
+
+    # Return all the individuals Metric,
+    # This call mimic a Enum's each if a block is provided
+    #
+    # @param path [String] The search path for metrics
+    # @param [Array] The metric for the specific path
+    def each(path = nil, &block)
+      metrics = if path.nil?
+        get_all
+      else
+        transform_to_array(get_with_path(path))
+      end
+
+      block_given? ? metrics.each(&block) : metrics
+    end
+    alias_method :all, :each
+
+    private
+    def get_all
+      @fast_lookup.values
+    end
+
+    # This method take an array of keys and recursively search the metric store structure
+    # and return a filtered hash of the structure. This method also take into consideration
+    # getting two different branchs.
+    #
+    #
+    # If one part of the `key_paths` contains a filter key with the following format.
+    # "pipeline01, pipeline_02", It know that need to fetch the branch `pipeline01` and `pipeline02`
+    #
+    # Look at the rspec test for more usage.
+    #
+    # @param key_paths [Array<Symbol>] The list of keys part to filter
+    # @param map [Concurrent::Map] The the part of map to search in
+    # @param new_hash [Hash] The hash to populate with the results.
+    # @return Hash
+    def get_recursively(key_paths, map, new_hash)
+      key_candidates = extract_filter_keys(key_paths.shift)
+
+      key_candidates.each do |key_candidate|
+        raise MetricNotFound, "For path: #{key_candidate}" if map[key_candidate].nil?
+
+        if key_paths.empty? # End of the user requested path
+          if map[key_candidate].is_a?(Concurrent::Map)
+            new_hash[key_candidate] = transform_to_hash(map[key_candidate])
+          else
+            new_hash[key_candidate] = map[key_candidate]
+          end
+        else
+          if map[key_candidate].is_a?(Concurrent::Map)
+            new_hash[key_candidate] = get_recursively(key_paths, map[key_candidate], {})
+          else
+            new_hash[key_candidate] = map[key_candidate]
+          end
+        end
+      end
+      return new_hash
+    end
+
+    def extract_filter_keys(key)
+      key.to_s.strip.split(FILTER_KEYS_SEPARATOR).map(&:to_sym)
+    end
+
+    # Take a hash and recursively flatten it into an array.
+    # This is useful if you are only interested in the leaf of the tree.
+    # Mostly used with `each` to get all the metrics from a specific namespaces
+    #
+    # This could be moved to `LogStash::Util` once this api stabilize
+    #
+    # @return [Array] One dimension array
+     def transform_to_array(map)
+      map.values.collect do |value|
+        value.is_a?(Hash) ? transform_to_array(value) : value
+      end.flatten
+    end
+
+    # Transform the Concurrent::Map hash into a ruby hash format,
+    # This is used to be serialize at the web api layer.
+    #
+    # This could be moved to `LogStash::Util` once this api stabilize
+    #
+    # @return [Hash]
+    def transform_to_hash(map, new_hash = Hash.new)
+      map.each_pair do |key, value|
+        if value.is_a?(Concurrent::Map)
+          new_hash[key] = {}
+          transform_to_hash(value, new_hash[key])
+        else
+          new_hash[key] = value
+        end
+      end
+
+      return new_hash
+    end
+
+    # This method iterate through the namespace path and try to find the corresponding
+    # value for the path, if any part of the path is not found it will
+    # create it.
+    #
+    # @param [Array] The path where values should be located
+    # @raise [ConcurrentMapExpected] Raise if the retrieved object isn't a `Concurrent::Map`
+    # @return [Concurrent::Map] Map where the metrics should be saved
+    def fetch_or_store_namespaces(namespaces_path)
+      path_map = fetch_or_store_namespace_recursively(@store, namespaces_path)
+
+      # This mean one of the namespace and key are colliding
+      # and we have to deal it upstream.
+      unless path_map.is_a?(Concurrent::Map)
+        raise NamespacesExpectedError, "Expecting a `Namespaces` but found class:  #{path_map.class.name} for namespaces_path: #{namespaces_path}"
+      end
+
+      return path_map
+    end
+
+    # Recursively fetch or create the namespace paths through the `MetricStove`
+    # This algorithm use an index to known which keys to search in the map.
+    # This doesn't cloning the array if we want to give a better feedback to the user
+    #
+    # @param [Concurrent::Map] Map to search for the key
+    # @param [Array] List of path to create
+    # @param [Fixnum] Which part from the list to create
+    #
+    def fetch_or_store_namespace_recursively(map, namespaces_path, idx = 0)
+      current = namespaces_path[idx]
+
+      # we are at the end of the namespace path, break out of the recursion
+      return map if current.nil?
+
+      new_map = map.fetch_or_store(current) { Concurrent::Map.new }
+      return fetch_or_store_namespace_recursively(new_map, namespaces_path, idx + 1)
+    end
+  end
+end; end