logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/instrument/snapshot.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require "logstash/event"
+
+module LogStash module Instrument
+  class Snapshot
+    include LogStash::Util::Loggable
+
+    attr_reader :metric_store, :created_at
+
+    def initialize(metric_store, created_at = Time.now)
+      @metric_store = metric_store
+      @created_at = created_at
+    end
+  end
+end; end

data/lib/logstash/json.rb

@@ -41,13 +41,12 @@ module LogStash
       raise LogStash::Json::ParserError.new(e.message)
     end
 
-    def jruby_dump(o)
+    def jruby_dump(o, options={})
       # TODO [guyboertje] remove these comments in 5.0
       # test for enumerable here to work around an omission in JrJackson::Json.dump to
       # also look for Java::JavaUtil::ArrayList, see TODO submit issue
       # o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
-
-      JrJackson::Base.generate(o, {})
+      JrJackson::Base.generate(o, options)
 
     rescue => e
       raise LogStash::Json::GeneratorError.new(e.message)

data/lib/logstash/namespace.rb

@@ -10,4 +10,5 @@ module LogStash
   module Util; end
   module PluginMixins; end
   module PluginManager; end
+  module Api; end
 end # module LogStash

data/lib/logstash/output_delegator.rb

@@ -13,7 +13,7 @@ module LogStash class OutputDelegator
 
   # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
   # Internally these just get merged together into a single hash
-  def initialize(logger, klass, default_worker_count, *plugin_args)
+  def initialize(logger, klass, default_worker_count, metric, *plugin_args)
     @logger = logger
     @threadsafe = klass.threadsafe?
     @config = plugin_args.reduce({}, :merge)
@@ -21,6 +21,14 @@ module LogStash class OutputDelegator
     @workers = java.util.concurrent.CopyOnWriteArrayList.new
     @default_worker_count = default_worker_count
     @registered = false
+
+    # Create an instance of the input so we can fetch the identifier
+    output = @klass.new(@config)
+
+    # Scope the metrics to the plugin
+    namespaced_metric = metric.namespace(output.plugin_unique_name.to_sym)
+    @metric_events = namespaced_metric.namespace(:events)
+
     @events_received = Concurrent::AtomicFixnum.new(0)
   end
 
@@ -74,7 +82,7 @@ module LogStash class OutputDelegator
     @workers << @klass.new(@config)
     @workers.first.register # Needed in case register calls `workers_not_supported`
 
-    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
 
     # Threadsafe versions don't need additional workers
     setup_additional_workers!(target_worker_count) unless @threadsafe
@@ -88,6 +96,7 @@ module LogStash class OutputDelegator
 
     (target_worker_count - 1).times do
       inst = @klass.new(@config)
+      inst.metric = @metric
       @workers << inst
     end
 
@@ -118,23 +127,27 @@ module LogStash class OutputDelegator
 
   def threadsafe_multi_receive(events)
     @events_received.increment(events.length)
+    @metric_events.increment(:in, events.length)
 
     @threadsafe_worker.multi_receive(events)
+    @metric_events.increment(:out, events.length)
   end
 
   def worker_multi_receive(events)
     @events_received.increment(events.length)
+    @metric_events.increment(:in, events.length)
 
     worker = @worker_queue.pop
     begin
       worker.multi_receive(events)
+      @metric_events.increment(:out, events.length)
     ensure
       @worker_queue.push(worker)
     end
   end
 
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass.name)
+    @logger.debug("closing output delegator", :klass => @klass)
 
     if @threadsafe
       @workers.each(&:do_close)

data/lib/logstash/outputs/base.rb

@@ -27,22 +27,6 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
 
-    # Set or return concurrency type
-  def self.concurrency(type=nil)
-    if type
-      @concurrency = type
-      
-      if type == :shared
-        declare_threadsafe!
-      elsif type == :single
-        declare_workers_not_supported!("This plugin only supports one worker!")
-      end
-      
-    else
-      @concurrency || :legacy # default is :legacyo
-    end
-  end
-
   def self.declare_threadsafe!
     declare_workers_not_supported!
     @threadsafe = true
@@ -81,8 +65,6 @@ class LogStash::Outputs::Base < LogStash::Plugin
     # If we're running with a single thread we must enforce single-threaded concurrency by default
     # Maybe in a future version we'll assume output plugins are threadsafe
     @single_worker_mutex = Mutex.new
-    
-    @receives_encoded = self.methods.include?(:multi_receive_encoded)
   end
 
   public
@@ -95,23 +77,10 @@ class LogStash::Outputs::Base < LogStash::Plugin
     raise "#{self.class}#receive must be overidden"
   end # def receive
 
-  public
-  def concurrency
-    self.class.concurrency
-  end
-
   public
   # To be overriden in implementations
   def multi_receive(events)
-    if @receives_encoded
-      self.multi_receive_encoded(codec.multi_encode(events))
-    else
-      events.each {|event| receive(event) }
-    end
-  end
-
-  def codec
-    params["codec"]
+    events.each {|event| receive(event) }
   end
 
   private

data/lib/logstash/pipeline.rb

@@ -14,10 +14,28 @@ require "logstash/util/defaults_printer"
 require "logstash/shutdown_watcher"
 require "logstash/util/wrapped_synchronous_queue"
 require "logstash/pipeline_reporter"
+require "logstash/instrument/metric"
+require "logstash/instrument/namespaced_metric"
+require "logstash/instrument/null_metric"
+require "logstash/instrument/collector"
 require "logstash/output_delegator"
+require "logstash/filter_delegator"
 
 module LogStash; class Pipeline
-  attr_reader :inputs, :filters, :outputs, :worker_threads, :events_consumed, :events_filtered, :reporter, :pipeline_id, :logger, :thread, :config_str, :original_settings
+ attr_reader :inputs,
+    :filters,
+    :outputs,
+    :worker_threads,
+    :events_consumed,
+    :events_filtered,
+    :reporter,
+    :pipeline_id,
+    :metric,
+    :logger,
+    :started_at,
+    :thread,
+    :config_str,
+    :original_settings
 
   DEFAULT_OUTPUT_WORKERS = 1
 
@@ -52,6 +70,16 @@ module LogStash; class Pipeline
 
     @worker_threads = []
 
+    # Metric object should be passed upstream, multiple pipeline share the same metric
+    # and collector only the namespace will changes.
+    # If no metric is given, we use a `NullMetric` for all internal calls.
+    # We also do this to make the changes backward compatible with previous testing of the
+    # pipeline.
+    #
+    # This need to be configured before we evaluate the code to make
+    # sure the metric instance is correctly send to the plugin.
+    @metric = settings.fetch(:metric, Instrument::NullMetric.new)
+
     grammar = LogStashConfigParser.new
     @config = grammar.parse(config_str)
     if @config.nil?
@@ -61,11 +89,15 @@ module LogStash; class Pipeline
     # The code will initialize all the plugins and define the
     # filter and output methods.
     code = @config.compile
+    @code = code
+
     # The config code is hard to represent as a log message...
     # So just print it.
-    if @settings[:debug_config]
-      @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
+
+    if @settings[:debug_config] && logger.debug?
+      logger.debug("Compiled pipeline code", :code => code)
     end
+
     begin
       eval(code)
     rescue => e
@@ -101,7 +133,7 @@ module LogStash; class Pipeline
     safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
 
     if unsafe_filters.any?
-      plugins = unsafe_filters.collect { |f| f.class.config_name }
+      plugins = unsafe_filters.collect { |f| f.config_name }
       case thread_count
       when nil
         # user did not specify a worker thread count
@@ -130,6 +162,9 @@ module LogStash; class Pipeline
   end
 
   def run
+    @started_at = Time.now
+
+    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
     @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
     @thread = Thread.current
 
@@ -179,7 +214,7 @@ module LogStash; class Pipeline
     begin
       start_inputs
       @outputs.each {|o| o.register }
-      @filters.each {|f| f.register}
+      @filters.each {|f| f.register }
 
       pipeline_workers = safe_pipeline_worker_count
       batch_size = @settings[:pipeline_batch_size]
@@ -209,16 +244,21 @@ module LogStash; class Pipeline
   end
 
   # Main body of what a worker thread does
-  # Repeatedly takes batches off the queu, filters, then outputs them
+  # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
     running = true
 
+    namespace_events = metric.namespace([:stats, :events])
+    namespace_pipeline = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
+
     while running
       # To understand the purpose behind this synchronize please read the body of take_batch
       input_batch, signal = @input_queue_pop_mutex.synchronize { take_batch(batch_size, batch_delay) }
       running = false if signal == LogStash::SHUTDOWN
 
       @events_consumed.increment(input_batch.size)
+      namespace_events.increment(:in, input_batch.size)
+      namespace_pipeline.increment(:in, input_batch.size)
 
       filtered_batch = filter_batch(input_batch)
 
@@ -229,8 +269,14 @@ module LogStash; class Pipeline
 
       @events_filtered.increment(filtered_batch.size)
 
+      namespace_events.increment(:filtered, filtered_batch.size)
+      namespace_pipeline.increment(:filtered, filtered_batch.size)
+
       output_batch(filtered_batch)
 
+      namespace_events.increment(:out, filtered_batch.size)
+      namespace_pipeline.increment(:out, filtered_batch.size)
+
       inflight_batches_synchronize { set_current_thread_inflight_batch(nil) }
     end
   end
@@ -392,7 +438,7 @@ module LogStash; class Pipeline
   def shutdown_workers
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
-      @logger.debug("Pushing shutdown", :thread => t.inspect)
+      @logger.debug("Pushing shutdown", :thread => t)
       @input_queue.push(LogStash::SHUTDOWN)
     end
 
@@ -409,10 +455,14 @@ module LogStash; class Pipeline
     args << {} if args.empty?
     args.first.merge!(LogStash::Config::Mixin::ALLOW_ENV_FLAG => @allow_env)
 
+    pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
+
     klass = LogStash::Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      LogStash::OutputDelegator.new(@logger, klass, DEFAULT_OUTPUT_WORKERS, *args)
+      LogStash::OutputDelegator.new(@logger, klass, DEFAULT_OUTPUT_WORKERS, pipeline_scoped_metric.namespace(:outputs), *args)
+    elsif plugin_type == "filter"
+      LogStash::FilterDelegator.new(@logger, klass, pipeline_scoped_metric.namespace(:filters), *args)
     else
       klass.new(*args)
     end
@@ -460,6 +510,15 @@ module LogStash; class Pipeline
     end
   end
 
+
+  # Calculate the uptime in milliseconds
+  #
+  # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
+  def uptime
+    return 0 if started_at.nil?
+    ((Time.now.to_f - started_at.to_f) * 1000.0).to_i
+  end
+
   # perform filters flush into the output queue
   # @param options [Hash]
   # @option options [Boolean] :final => true to signal a final shutdown flush

data/lib/logstash/plugin.rb

@@ -5,6 +5,7 @@ require "logstash/config/mixin"
 require "logstash/instrument/null_metric"
 require "cabin"
 require "concurrent"
+require "securerandom"
 
 class LogStash::Plugin
   attr_accessor :params
@@ -12,35 +13,69 @@ class LogStash::Plugin
 
   NL = "\n"
 
-  public
+  include LogStash::Config::Mixin
+
+  # Disable or enable metric logging for this specific plugin instance
+  # by default we record all the metrics we can, but you can disable metrics collection
+  # for a specific plugin.
+  config :enable_metric, :validate => :boolean, :default => true
+
+  # Add a unique `ID` to the plugin instance, this `ID` is used for tracking
+  # information for a specific configuration of the plugin.
+  #
+  # ```
+  # output {
+  #  stdout {
+  #    id => "ABC"
+  #  }
+  # }
+  # ```
+  #
+  # If you don't explicitely set this variable Logstash will generate a unique name.
+  config :id, :validate => :string
+
   def hash
     params.hash ^
     self.class.name.hash
   end
 
-  public
+
   def eql?(other)
     self.class.name == other.class.name && @params == other.params
   end
 
-  public
   def initialize(params=nil)
     @params = LogStash::Util.deep_clone(params)
     @logger = Cabin::Channel.get(LogStash)
-    @metric_plugin = LogStash::Instrument::NullMetric.new
+  end
+
+  # Return a uniq ID for this plugin configuration, by default
+  # we will generate a UUID
+  #
+  # If the user defines a `id => 'ABC'` in the configuration we will return
+  #
+  # @return [String] A plugin ID
+  def id
+    (@params["id"].nil? || @params["id"].empty?) ? SecureRandom.uuid : @params["id"]
+  end
+
+  # Return a unique_name, This is composed by the name of
+  # the plugin and the generated ID (of the configured one)
+  #
+  # @return [String] a unique name
+  def plugin_unique_name
+    "#{config_name}_#{id}"
   end
 
   # close is called during shutdown, after the plugin worker
   # main task terminates
-  public
   def do_close
-    @logger.debug("closing", :plugin => self.class.name)
+    @logger.debug("closing", :plugin => self)
     close
   end
 
   # Subclasses should implement this close method if you need to perform any
   # special tasks during shutdown (like flushing, etc.)
-  public
   def close
     # ..
   end
@@ -49,15 +84,6 @@ class LogStash::Plugin
     return "#{self.class.name}: #{@params}"
   end
 
-  # This is a shim to make sure that plugin
-  # that record metric still work with 2.4
-  #
-  # https://github.com/elastic/logstash/issues/5539
-  def metric
-    @metric_plugin
-  end
-
-  public
   def inspect
     if !@params.nil?
       description = @params
@@ -69,13 +95,26 @@ class LogStash::Plugin
     end
   end
 
-  public
   def debug_info
     [self.class.to_s, original_params]
   end
 
+  def metric=(new_metric)
+    @metric = new_metric
+  end
+
+  def metric
+    @metric_plugin ||= enable_metric ? @metric : LogStash::Instrument::NullMetric.new
+  end
+
+  # return the configured name of this plugin
+  # @return [String] The name of the plugin defined by `config_name`
+  def config_name
+    self.class.config_name
+  end
+
+
   # Look up a plugin by type and name.
-  public
   def self.lookup(type, name)
     path = "logstash/#{type}s/#{name}"
 
@@ -96,7 +135,6 @@ class LogStash::Plugin
   end
 
   private
-
   # lookup a plugin by type and name in the existing LogStash module namespace
   # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
   # @param type [String] plugin type, "input", "ouput", "filter"