logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/logging/logger.rb

@@ -7,13 +7,10 @@ module LogStash
     java_import org.apache.logging.log4j.LogManager
     java_import org.apache.logging.log4j.core.config.Configurator
     java_import org.apache.logging.log4j.core.config.DefaultConfiguration
-    java_import org.apache.logging.log4j.core.config.LoggerConfig
-    java_import org.logstash.log.LogstashLoggerContextFactory
-    java_import org.apache.logging.log4j.core.LoggerContext
-    java_import java.net.URI
 
     class Logger
       @@config_mutex = Mutex.new
+      @@logging_context = nil
 
       def initialize(name)
         @logger = LogManager.getLogger(name)
@@ -68,60 +65,31 @@ module LogStash
       end
 
       def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
-        @@config_mutex.synchronize { set_level(level, path) }
+        @@config_mutex.synchronize { Configurator.setLevel(path, Level.valueOf(level)) }
       rescue Exception => e
         raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
       end
 
-      def self.reconfigure(config_location)
+      def self.initialize(config_location)
         @@config_mutex.synchronize do
-          config_location_uri = URI.create(config_location)
-          file_path = config_location_uri.path
-          if ::File.exists?(file_path)
-            logs_location = java.lang.System.getProperty("ls.logs")
-            puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
-            #reconfigure the default context to use our log4j2.properties file
-            get_logging_context.setConfigLocation(URI.create(config_location))
-            #ensure everyone agrees which context to use for the LogManager
-            context_factory = LogstashLoggerContextFactory.new(get_logging_context)
-            LogManager.setFactory(context_factory)
-          else
-            # fall back to default config
-            puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs errors to the console"
+          if @@logging_context.nil?
+            file_path = URI(config_location).path
+            if ::File.exists?(file_path)
+              logs_location = java.lang.System.getProperty("ls.logs")
+              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+              @@logging_context = Configurator.initialize(nil, config_location)
+            else
+              # fall back to default config
+              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
+              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
+            end
           end
         end
       end
 
-      # until dev_utils/rspec/spec_helper is changed, we need to have both methods
-      singleton_class.send(:alias_method, :initialize, :reconfigure)
-
       def self.get_logging_context
-        return  LoggerContext.getContext(false)
-      end
-
-      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but ensure the proper context is used
-      def self.set_level(_level, path)
-        configuration =  get_logging_context.getConfiguration()
-        level = Level.valueOf(_level)
-        if path.nil? || path.strip.empty?
-          root_logger = configuration.getRootLogger()
-          if root_logger.getLevel() != level
-            root_logger.setLevel(level)
-            get_logging_context.updateLoggers()
-          end
-        else
-          package_logger = configuration.getLoggerConfig(path)
-          if package_logger.name != path #no package logger found
-            configuration.addLogger(path, LoggerConfig.new(path, level, true))
-            get_logging_context.updateLoggers()
-          elsif package_logger.getLevel() != level
-            package_logger.setLevel(level)
-            get_logging_context.updateLoggers()
-          end
-        end
+        return @@logging_context
       end
-
-      private_class_method :set_level
     end
 
     class SlowLogger

data/lib/logstash/namespace.rb

@@ -11,5 +11,4 @@ module LogStash
   module PluginMixins; end
   module PluginManager; end
   module Api; end
-  module Modules; end
 end # module LogStash

data/lib/logstash/output_delegator.rb

@@ -19,9 +19,7 @@ module LogStash class OutputDelegator
     @namespaced_metric = metric.namespace(id.to_sym)
     @namespaced_metric.gauge(:name, config_name)
     @metric_events = @namespaced_metric.namespace(:events)
-    @in_counter = @metric_events.counter(:in)
-    @out_counter = @metric_events.counter(:out)
-    @time_metric = @metric_events.counter(:duration_in_millis)
+
     @strategy = strategy_registry.
                   class_for(self.concurrency).
                   new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
@@ -44,11 +42,11 @@ module LogStash class OutputDelegator
   end
 
   def multi_receive(events)
-    @in_counter.increment(events.length)
-    start_time = java.lang.System.current_time_millis
+    @metric_events.increment(:in, events.length)
+    clock = @metric_events.time(:duration_in_millis)
     @strategy.multi_receive(events)
-    @time_metric.increment(java.lang.System.current_time_millis - start_time)
-    @out_counter.increment(events.length)
+    clock.stop
+    @metric_events.increment(:out, events.length)
   end
 
   def do_close

data/lib/logstash/outputs/base.rb

@@ -109,8 +109,6 @@ class LogStash::Outputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
-    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
-    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/pipeline.rb

@@ -18,25 +18,26 @@ require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
 require "logstash/instrument/wrapped_write_client"
-require "logstash/util/dead_letter_queue_manager"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
 require "logstash/queue_factory"
+require "logstash/compiler"
 require "logstash/execution_context"
 
-java_import org.logstash.common.DeadLetterQueueFactory
-java_import org.logstash.common.io.DeadLetterQueueWriter
-
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
 
-  attr_reader :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id
+  attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :execution_context
 
-  def initialize(config_str, settings = SETTINGS)
+  def initialize(config_str, settings = SETTINGS, namespaced_metric = nil, agent = nil)
     @logger = self.logger
-    @mutex = Mutex.new
+
     @config_str = config_str
+    @settings = settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
+
+    @lir = compile_lir
+
     # Every time #plugin is invoked this is incremented to give each plugin
     # a unique id when auto-generating plugin ids
     @plugin_counter ||= 0
@@ -48,25 +49,18 @@ module LogStash; class BasePipeline
     @inputs = nil
     @filters = nil
     @outputs = nil
-
-    if settings.get_value("dead_letter_queue.enable")
-      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"), settings.get_value("dead_letter_queue.max_bytes"))
-    else
-      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
-    end
+    @execution_context = LogStash::ExecutionContext.new(self, agent)
 
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
     raise(ConfigurationError, grammar.failure_reason) if parsed_config.nil?
 
-    parsed_config.process_escape_sequences = settings.get_value("config.support_escapes")
-
     config_code = parsed_config.compile
 
     # config_code = BasePipeline.compileConfig(config_str)
 
     if settings.get_value("config.debug") && @logger.debug?
-      @logger.debug("Compiled pipeline code", :code => config_code)
+      @logger.debug("Compiled pipeline code", default_logging_keys(:code => config_code))
     end
 
     # Evaluate the config compiled code that will initialize all the plugins and define the
@@ -78,6 +72,10 @@ module LogStash; class BasePipeline
     end
   end
 
+  def compile_lir
+    LogStash::Compiler.compile_pipeline(self.config_str)
+  end
+
   def plugin(plugin_type, name, *args)
     @plugin_counter += 1
 
@@ -102,34 +100,34 @@ module LogStash; class BasePipeline
 
     klass = Plugin.lookup(plugin_type, name)
 
-    execution_context = ExecutionContext.new(self, id, klass.config_name, @dlq_writer)
-
     if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
+      OutputDelegator.new(@logger, klass, type_scoped_metric, @execution_context, OutputDelegatorStrategyRegistry.instance, args)
     elsif plugin_type == "filter"
-      FilterDelegator.new(@logger, klass, type_scoped_metric, execution_context, args)
+      FilterDelegator.new(@logger, klass, type_scoped_metric, @execution_context, args)
     else # input
       input_plugin = klass.new(args)
       scoped_metric = type_scoped_metric.namespace(id.to_sym)
       scoped_metric.gauge(:name, input_plugin.config_name)
       input_plugin.metric = scoped_metric
-      input_plugin.execution_context = execution_context
+      input_plugin.execution_context = @execution_context
       input_plugin
     end
   end
 
   def reloadable?
-    non_reloadable_plugins.empty?
+    configured_as_reloadable? && reloadable_plugins?
   end
 
-  def non_reloadable_plugins
-    (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
+  def configured_as_reloadable?
+    settings.get("pipeline.reloadable")
   end
 
-  private
+  def reloadable_plugins?
+    non_reloadable_plugins.empty?
+  end
 
-  def default_logging_keys(other_keys = {})
-    { :pipeline_id => pipeline_id }.merge(other_keys)
+  def non_reloadable_plugins
+    (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
   end
 end; end
 
@@ -149,7 +147,7 @@ module LogStash; class Pipeline < BasePipeline
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
-  def initialize(config_str, settings = SETTINGS, namespaced_metric = nil)
+  def initialize(config_str, settings = SETTINGS, namespaced_metric = nil, agent = nil)
     # This needs to be configured before we call super which will evaluate the code to make
     # sure the metric instance is correctly send to the plugins to make the namespace scoping work
     @metric = if namespaced_metric
@@ -162,12 +160,12 @@ module LogStash; class Pipeline < BasePipeline
     @reporter = PipelineReporter.new(@logger, self)
     @worker_threads = []
 
-    super(config_str, settings)
+    super
 
     begin
       @queue = LogStash::QueueFactory.create(settings)
     rescue => e
-      @logger.error("Logstash failed to create queue", "exception" => e.message, "backtrace" => e.backtrace)
+      @logger.error("Logstash failed to create queue", default_logging_keys("exception" => e.message, "backtrace" => e.backtrace))
       raise e
     end
 
@@ -182,7 +180,6 @@ module LogStash; class Pipeline < BasePipeline
     )
     @drain_queue =  @settings.get_value("queue.drain")
 
-
     @events_filtered = Concurrent::AtomicFixnum.new(0)
     @events_consumed = Concurrent::AtomicFixnum.new(0)
 
@@ -191,8 +188,11 @@ module LogStash; class Pipeline < BasePipeline
     @ready = Concurrent::AtomicBoolean.new(false)
     @running = Concurrent::AtomicBoolean.new(false)
     @flushing = Concurrent::AtomicReference.new(false)
+    @force_shutdown = Concurrent::AtomicBoolean.new(false)
   end # def initialize
 
+
+
   def ready?
     @ready.value
   end
@@ -207,15 +207,14 @@ module LogStash; class Pipeline < BasePipeline
 
     if @settings.set?("pipeline.workers")
       if pipeline_workers > 1
-        @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads",
-                     :worker_threads => pipeline_workers, :filters => plugins)
+        @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads", default_logging_keys(:worker_threads => pipeline_workers, :filters => plugins))
       end
     else
       # user did not specify a worker thread count
       # warn if the default is multiple
       if default > 1
         @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
-                     :count_was => default, :filters => plugins)
+                     default_logging_keys(:count_was => default, :filters => plugins))
         return 1 # can't allow the default value to propagate if there are unsafe filters
       end
     end
@@ -226,15 +225,61 @@ module LogStash; class Pipeline < BasePipeline
     return @filters.any?
   end
 
+  def start
+    # Since we start lets assume that the metric namespace is cleared
+    # this is useful in the context of pipeline reloading
+    collect_stats
+
+    logger.debug("Starting pipeline", default_logging_keys)
+
+    @finished_execution = Concurrent::AtomicBoolean.new(false)
+
+    @thread = Thread.new do
+      begin
+        LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
+        run
+        @finished_execution.make_true
+      rescue => e
+        close
+        logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+      end
+    end
+
+    status = wait_until_started
+
+    if status
+      logger.debug("Pipeline started successfully", default_logging_keys(:pipeline_id => pipeline_id))
+    end
+
+    status
+  end
+
+  def wait_until_started
+    while true do
+      # This should be changed with an appropriate FSM
+      # It's an edge case, if we have a pipeline with
+      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
+      # because the execution of the thread was successful and complete
+      if @finished_execution.true?
+        return true
+      elsif !thread.alive?
+        return false
+      elsif running?
+        return true
+      else
+        sleep 0.01
+      end
+    end
+  end
+
   def run
     @started_at = Time.now
-
     @thread = Thread.current
     Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
 
     start_workers
 
-    @logger.info("Pipeline #{@pipeline_id} started")
+    @logger.info("Pipeline started", default_logging_keys)
 
     # Block until all inputs have stopped
     # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
@@ -244,14 +289,14 @@ module LogStash; class Pipeline < BasePipeline
     wait_inputs
     transition_to_stopped
 
-    @logger.debug("Input plugins stopped! Will shutdown filter/output workers.")
+    @logger.debug("Input plugins stopped! Will shutdown filter/output workers.", default_logging_keys)
 
     shutdown_flusher
     shutdown_workers
 
     close
 
-    @logger.debug("Pipeline #{@pipeline_id} has been shutdown")
+    @logger.debug("Pipeline has been shutdown", default_logging_keys)
 
     # exit code
     return 0
@@ -260,7 +305,6 @@ module LogStash; class Pipeline < BasePipeline
   def close
     @filter_queue_client.close
     @queue.close
-    @dlq_writer.close
   end
 
   def transition_to_running
@@ -290,7 +334,7 @@ module LogStash; class Pipeline < BasePipeline
     plugin.register
     plugin
   rescue => e
-    @logger.error("Error registering plugin", :plugin => plugin.inspect, :error => e.message)
+    @logger.error("Error registering plugin", default_logging_keys(:plugin => plugin.inspect, :error => e.message))
     raise e
   end
 
@@ -322,18 +366,14 @@ module LogStash; class Pipeline < BasePipeline
       config_metric.gauge(:batch_delay, batch_delay)
       config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
       config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
-      config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
-      config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
-
 
-      @logger.info("Starting pipeline",
-                   "id" => self.pipeline_id,
-                   "pipeline.workers" => pipeline_workers,
-                   "pipeline.batch.size" => batch_size,
-                   "pipeline.batch.delay" => batch_delay,
-                   "pipeline.max_inflight" => max_inflight)
+      @logger.info("Starting pipeline", default_logging_keys(
+        "pipeline.workers" => pipeline_workers,
+        "pipeline.batch.size" => batch_size,
+        "pipeline.batch.delay" => batch_delay,
+        "pipeline.max_inflight" => max_inflight))
       if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
-        @logger.warn "CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})"
+        @logger.warn("CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})", default_logging_keys)
       end
 
       pipeline_workers.times do |t|
@@ -359,10 +399,6 @@ module LogStash; class Pipeline < BasePipeline
     end
   end
 
-  def dlq_enabled?
-    @settings.get("dead_letter_queue.enable")
-  end
-
   # Main body of what a worker thread does
   # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
@@ -379,6 +415,7 @@ module LogStash; class Pipeline < BasePipeline
       filter_batch(batch)
       flush_filters_to_batch(batch, :final => false) if signal.flush?
       output_batch(batch)
+      break if @force_shutdown.true? # Do not ack the current batch
       @filter_queue_client.close_batch(batch)
 
       # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
@@ -390,14 +427,19 @@ module LogStash; class Pipeline < BasePipeline
     batch = @filter_queue_client.new_batch
     @filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
     flush_filters_to_batch(batch, :final => true)
+    return if @force_shutdown.true? # Do not ack the current batch
     output_batch(batch)
     @filter_queue_client.close_batch(batch)
   end
 
   def filter_batch(batch)
-    filter_func(batch.to_a).each do |e|
-      #these are both original and generated events
-      batch.merge(e) unless e.cancelled?
+    batch.each do |event|
+      return if @force_shutdown.true?
+
+      filter_func(event).each do |e|
+        #these are both original and generated events
+        batch.merge(e) unless e.cancelled?
+      end
     end
     @filter_queue_client.add_filtered_metrics(batch)
     @events_filtered.increment(batch.size)
@@ -409,7 +451,7 @@ module LogStash; class Pipeline < BasePipeline
     # Users need to check their configuration or see if there is a bug in the
     # plugin.
     @logger.error("Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.",
-                  "exception" => e.message, "backtrace" => e.backtrace)
+                  default_logging_keys("exception" => e.message, "backtrace" => e.backtrace))
 
     raise e
   end
@@ -431,6 +473,7 @@ module LogStash; class Pipeline < BasePipeline
     # Now that we have our output to event mapping we can just invoke each output
     # once with its list of events
     output_events_map.each do |output, events|
+      return if @force_shutdown.true?
       output.multi_receive(events)
     end
 
@@ -471,20 +514,21 @@ module LogStash; class Pipeline < BasePipeline
     rescue => e
       if plugin.stop?
         @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
-                      :plugin => plugin.class.config_name, :exception => e.message,
-                      :backtrace => e.backtrace)
+                      default_logging_keys(:plugin => plugin.class.config_name, :exception => e.message, :backtrace => e.backtrace))
         return
       end
 
       # otherwise, report error and restart
       if @logger.debug?
         @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
-                             :plugin => plugin.inspect, :error => e.message,
-                             :exception => e.class,
-                             :stacktrace => e.backtrace.join("\n")))
+                             default_logging_keys(
+                               :plugin => plugin.inspect,
+                               :error => e.message,
+                               :exception => e.class,
+                               :stacktrace => e.backtrace.join("\n"))))
       else
         @logger.error(I18n.t("logstash.pipeline.worker-error",
-                             :plugin => plugin.inspect, :error => e.message))
+                             default_logging_keys(:plugin => plugin.inspect, :error => e.message)))
       end
 
       # Assuming the failure that caused this exception is transient,
@@ -509,23 +553,42 @@ module LogStash; class Pipeline < BasePipeline
 
     before_stop.call if block_given?
 
-    @logger.debug "Closing inputs"
-    @inputs.each(&:do_stop)
-    @logger.debug "Closed inputs"
+    stop_inputs
+
+    # We make this call blocking, so we know for sure when the method return the shtudown is
+    # stopped
+    wait_for_workers
+    clear_pipeline_metrics
   end # def shutdown
 
+  def force_shutdown!
+    @force_shutdown.make_true
+  end
+
+  def wait_for_workers
+    @logger.debug("Closing inputs", default_logging_keys)
+    @worker_threads.map(&:join)
+    @logger.debug("Worker closed", default_logging_keys)
+  end
+
+  def stop_inputs
+    @logger.debug("Closing inputs", default_logging_keys)
+    @inputs.each(&:do_stop)
+    @logger.debug("Closed inputs", default_logging_keys)
+  end
+
   # After `shutdown` is called from an external thread this is called from the main thread to
   # tell the worker threads to stop and then block until they've fully stopped
   # This also stops all filter and output plugins
   def shutdown_workers
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
-      @logger.debug("Pushing shutdown", :thread => t.inspect)
+      @logger.debug("Pushing shutdown", default_logging_keys(:thread => t.inspect))
       @signal_queue.push(SHUTDOWN)
     end
 
     @worker_threads.each do |t|
-      @logger.debug("Shutdown waiting for worker thread #{t}")
+      @logger.debug("Shutdown waiting for worker thread" , default_logging_keys(:thread => t.inspect))
       t.join
     end
 
@@ -537,7 +600,7 @@ module LogStash; class Pipeline < BasePipeline
   # in the pipeline anymore.
   def filter(event, &block)
     # filter_func returns all filtered events, including cancelled ones
-    filter_func([event]).each {|e| block.call(e)}
+    filter_func(event).each { |e| block.call(e) }
   end
 
 
@@ -548,6 +611,7 @@ module LogStash; class Pipeline < BasePipeline
     flushers = options[:final] ? @shutdown_flushers : @periodic_flushers
 
     flushers.each do |flusher|
+      return if @force_shutdown.true?
       flusher.call(options, &block)
     end
   end
@@ -570,12 +634,11 @@ module LogStash; class Pipeline < BasePipeline
 
   def flush
     if @flushing.compare_and_set(false, true)
-      @logger.debug? && @logger.debug("Pushing flush onto pipeline")
+      @logger.debug? && @logger.debug("Pushing flush onto pipeline", default_logging_keys)
       @signal_queue.push(FLUSH)
     end
   end
 
-
   # Calculate the uptime in milliseconds
   #
   # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
@@ -590,8 +653,10 @@ module LogStash; class Pipeline < BasePipeline
   # @param options [Hash]
   def flush_filters_to_batch(batch, options = {})
     flush_filters(options) do |event|
+      return if @force_shutdown.true?
+
       unless event.cancelled?
-        @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
+        @logger.debug? and @logger.debug("Pushing flushed events", default_logging_keys(:event => event))
         batch.merge(event)
       end
     end
@@ -613,13 +678,6 @@ module LogStash; class Pipeline < BasePipeline
       .each {|t| t.delete("status") }
   end
 
-  def collect_dlq_stats
-    if dlq_enabled?
-      dlq_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :dlq])
-      dlq_metric.gauge(:queue_size_in_bytes, @dlq_writer.get_current_queue_size)
-    end
-  end
-
   def collect_stats
     pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
     pipeline_metric.gauge(:type, settings.get("queue.type"))
@@ -645,6 +703,20 @@ module LogStash; class Pipeline < BasePipeline
     end
   end
 
+  def clear_pipeline_metrics
+    # TODO(ph): I think the metric should also proxy that call correctly to the collector
+    # this will simplify everything since the null metric would simply just do a noop
+    collector = @metric.collector
+
+    unless collector.nil?
+      # selectively reset metrics we don't wish to keep after reloading
+      # these include metrics about the plugins and number of processed events
+      # we want to keep other metrics like reload counts and error messages
+      collector.clear("stats/pipelines/#{pipeline_id}/plugins")
+      collector.clear("stats/pipelines/#{pipeline_id}/events")
+    end
+  end
+
   # Sometimes we log stuff that will dump the pipeline which may contain
   # sensitive information (like the raw syntax tree which can contain passwords)
   # We want to hide most of what's in here
@@ -661,9 +733,12 @@ module LogStash; class Pipeline < BasePipeline
   private
 
   def default_logging_keys(other_keys = {})
-    keys = super
-    keys[:thread] = thread.inspect if thread
-    keys
+    default_options = if thread
+                        { :pipeline_id => pipeline_id, :thread => thread.inspect }
+                      else
+                        { :pipeline_id => pipeline_id }
+                      end
+    default_options.merge(other_keys)
   end
 
   def draining_queue?
@@ -671,9 +746,6 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   def wrapped_write_client(plugin)
-    #need to ensure that metrics are initialized one plugin at a time, else a race condition can exist.
-    @mutex.synchronize do
-      LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
-    end
+    LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
   end
 end; end