logstash-core 6.3.2-java → 6.4.0-java

data/lib/logstash/filters/base.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/event"
-require "logstash/logging"
 require "logstash/plugin"
 require "logstash/config/mixin"
 require "logstash/util/decorators"

data/lib/logstash/inputs/base.rb

@@ -1,8 +1,6 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/event"
 require "logstash/plugin"
-require "logstash/logging"
 require "logstash/config/mixin"
 require "logstash/codecs/base"
 require "logstash/util/decorators"
@@ -94,7 +92,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
   def stop?
     @stop_called.value
   end
-  
+
   def clone
     cloned = super
     cloned.codec = @codec.clone if @codec
@@ -105,7 +103,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
-    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their
     #               parent plugin's
     @codec.execution_context = context
     context

data/lib/logstash/inputs/threadable.rb

@@ -1,15 +1,14 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/inputs/base"
 
-# This is the threadable class for logstash inputs. 
+# This is the threadable class for logstash inputs.
 # Use this class in your inputs if it can support multiple threads
 class LogStash::Inputs::Threadable < LogStash::Inputs::Base
 
   # Set this to the number of threads you want this input to spawn.
   # This is the same as declaring the input multiple times
   config :threads, :validate => :number, :default => 1
- 
+
   def initialize(params)
     super
     @threadable = true

data/lib/logstash/instrument/collector.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
-require "logstash/instrument/snapshot"
 require "logstash/instrument/metric_store"
-require "logstash/util/loggable"
 require "concurrent/timer_task"
 require "observer"
 require "singleton"

data/lib/logstash/instrument/metric.rb

@@ -1,105 +1 @@
-# encoding: utf-8
-require "logstash/instrument/collector"
-require "concurrent"
-
-module LogStash module Instrument
-  class MetricException < Exception; end
-  class MetricNoKeyProvided < MetricException; end
-  class MetricNoBlockProvided < MetricException; end
-  class MetricNoNamespaceProvided < MetricException; end
-
-  # This class provide the interface between the code, the collector and the format
-  # of the recorded metric.
-  class Metric
-    attr_reader :collector
-
-    def initialize(collector)
-      @collector = collector
-    end
-
-    def increment(namespace, key, value = 1)
-      self.class.validate_key!(key)
-      collector.push(namespace, key, :counter, :increment, value)
-    end
-
-    def decrement(namespace, key, value = 1)
-      self.class.validate_key!(key)
-      collector.push(namespace, key, :counter, :decrement, value)
-    end
-
-    def gauge(namespace, key, value)
-      self.class.validate_key!(key)
-      collector.push(namespace, key, :gauge, :set, value)
-    end
-
-    def time(namespace, key)
-      self.class.validate_key!(key)
-
-      if block_given?
-        timer = TimedExecution.new(self, namespace, key)
-        content = yield
-        timer.stop
-        return content
-      else
-        TimedExecution.new(self, namespace, key)
-      end
-    end
-
-    def report_time(namespace, key, duration)
-      self.class.validate_key!(key)
-      collector.push(namespace, key, :counter, :increment, duration)
-    end
-
-    # This method return a metric instance tied to a specific namespace
-    # so instead of specifying the namespace on every call.
-    #
-    # Example:
-    #   metric.increment(:namespace, :mykey, 200)
-    #   metric.increment(:namespace, :mykey_2, 200)
-    #
-    #   namespaced_metric = metric.namespace(:namespace)
-    #   namespaced_metric.increment(:mykey, 200)
-    #   namespaced_metric.increment(:mykey_2, 200)
-    # ```
-    #
-    # @param name [Array<String>] Name of the namespace
-    # @param name [String] Name of the namespace
-    def namespace(name)
-      raise MetricNoNamespaceProvided if name.nil? || name.empty?
-
-      NamespacedMetric.new(self, name)
-    end
-
-    def self.validate_key!(key)
-      raise MetricNoKeyProvided if key.nil? || key.empty?
-    end
-
-    private
-    # Allow to calculate the execution of a block of code.
-    # This class support 2 differents syntax a block or the return of
-    # the object itself, but in the later case the metric won't be recorded
-    # Until we call `#stop`.
-    #
-    # @see LogStash::Instrument::Metric#time
-    class TimedExecution
-      MILLISECONDS = 1_000.0.freeze
-
-      def initialize(metric, namespace, key)
-        @metric = metric
-        @namespace = namespace
-        @key = key
-        start
-      end
-
-      def start
-        @start_time = Time.now
-      end
-
-      def stop
-        execution_time = (MILLISECONDS * (Time.now - @start_time)).to_i
-        @metric.report_time(@namespace, @key, execution_time)
-        execution_time
-      end
-    end
-  end
-end; end
+# This file is kept for backwards compatibility with plugins that include it.

data/lib/logstash/instrument/namespaced_metric.rb

@@ -1,58 +1 @@
-# encoding: utf-8
-require "logstash/instrument/metric"
-
-module LogStash module Instrument
-  # This class acts a a proxy between the metric library and the user calls.
-  #
-  # This is the class that plugins authors will use to interact with the `MetricStore`
-  # It has the same public interface as `Metric` class but doesnt require to send
-  # the namespace on every call.
-  #
-  # @see Logstash::Instrument::Metric
-  class NamespacedMetric
-    attr_reader :namespace_name
-    # Create metric with a specific namespace
-    #
-    # @param metric [LogStash::Instrument::Metric] The metric instance to proxy
-    # @param namespace [Array] The namespace to use
-    def initialize(metric, namespace_name)
-      @metric = metric
-      @namespace_name = Array(namespace_name)
-    end
-
-    def increment(key, value = 1)
-      @metric.increment(namespace_name, key, value)
-    end
-
-    def decrement(key, value = 1)
-      @metric.decrement(namespace_name, key, value)
-    end
-
-    def gauge(key, value)
-      @metric.gauge(namespace_name, key, value)
-    end
-
-    def report_time(key, duration)
-      @metric.report_time(namespace_name, key, duration)
-    end
-
-    def time(key, &block)
-      @metric.time(namespace_name, key, &block)
-    end
-
-    def collector
-      @metric.collector
-    end
-    
-    def counter(key)
-      collector.get(@namespace_name, key, :counter)
-    end
-
-    def namespace(name)
-      NamespacedMetric.new(metric, namespace_name + Array(name))
-    end
-
-    private
-    attr_reader :metric
-  end
-end; end
+# This file is kept for backwards compatibility with plugins that include it.

data/lib/logstash/instrument/namespaced_null_metric.rb

@@ -1,58 +1 @@
-# encoding: utf-8
-require "logstash/instrument/null_metric"
-
-module LogStash module Instrument
-  # This class acts a a proxy between the metric library and the user calls.
-  #
-  # This is the class that plugins authors will use to interact with the `MetricStore`
-  # It has the same public interface as `Metric` class but doesnt require to send
-  # the namespace on every call.
-  #
-  # @see Logstash::Instrument::Metric
-  class NamespacedNullMetric
-    attr_reader :namespace_name
-    # Create metric with a specific namespace
-    #
-    # @param metric [LogStash::Instrument::Metric] The metric instance to proxy
-    # @param namespace [Array] The namespace to use
-    def initialize(metric = nil, namespace_name = :null)
-      @metric = metric
-      @namespace_name = Array(namespace_name)
-    end
-
-    def increment(key, value = 1)
-    end
-
-    def decrement(key, value = 1)
-    end
-
-    def gauge(key, value)
-    end
-
-    def report_time(key, duration)
-    end
-
-    def time(key, &block)
-      if block_given?
-        yield
-      else
-        ::LogStash::Instrument::NullMetric::NullTimedExecution
-      end
-    end
-
-    def collector
-      @metric.collector
-    end
-
-    def counter(_)
-      ::LogStash::Instrument::NullMetric::NullGauge
-    end
-
-    def namespace(name)
-      NamespacedNullMetric.new(metric, namespace_name + Array(name))
-    end
-
-    private
-    attr_reader :metric
-  end
-end; end
+# This file is kept for backwards compatibility with plugins that include it directly.

data/lib/logstash/instrument/null_metric.rb

@@ -1,71 +1,2 @@
-# encoding: utf-8
-require "logstash/instrument/metric"
-
-module LogStash module Instrument
-  # This class is used in the context when we disable the metric collection
-  # for specific plugin to replace the `NamespacedMetric` class with this one
-  # which doesn't produce any metric to the collector.
-  class NullMetric
-    attr_reader :namespace_name, :collector
-
-    def initialize(collector = nil)
-      @collector = collector
-    end
-
-    def increment(namespace, key, value = 1)
-      Metric.validate_key!(key)
-    end
-
-    def decrement(namespace, key, value = 1)
-      Metric.validate_key!(key)
-    end
-
-    def gauge(namespace, key, value)
-      Metric.validate_key!(key)
-    end
-
-    def report_time(namespace, key, duration)
-      Metric.validate_key!(key)
-    end
-
-    # We have to manually redefine this method since it can return an
-    # object this object also has to be implemented as a NullObject
-    def time(namespace, key)
-      Metric.validate_key!(key)
-      if block_given?
-        yield
-      else
-        NullTimedExecution
-      end
-    end
-
-    def counter(_)
-      NullGauge
-    end
-
-    def namespace(name)
-      raise MetricNoNamespaceProvided if name.nil? || name.empty?
-      NamespacedNullMetric.new(self, name)
-    end
-
-    def self.validate_key!(key)
-      raise MetricNoKeyProvided if key.nil? || key.empty?
-    end
-
-    private
-
-    class NullGauge
-      def self.increment(_)
-      end
-    end
-
-    # Null implementation of the internal timer class
-    #
-    # @see LogStash::Instrument::TimedExecution`
-    class NullTimedExecution
-      def self.stop
-        0
-      end
-    end
-  end
-end; end
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly require it.

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/util/loggable"
 require "logstash/util"
 require "concurrent"
 

data/lib/logstash/instrument/periodic_poller/cgroup.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "pathname"
-require "logstash/util/loggable"
 
 # Logic from elasticsearch/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
 # Move to ruby to remove any existing dependency

data/lib/logstash/java_pipeline.rb

@@ -1,174 +1,30 @@
 # encoding: utf-8
 require "thread"
 require "concurrent"
-require "logstash/namespace"
-require "logstash/errors"
 require "logstash/event"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
-require "logstash/shutdown_watcher"
-require "logstash/pipeline_reporter"
-require "logstash/instrument/metric"
-require "logstash/instrument/namespaced_metric"
-require "logstash/instrument/null_metric"
-require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
-require "logstash/util/dead_letter_queue_manager"
-require "logstash/queue_factory"
 require "logstash/compiler"
-require "securerandom"
-
-java_import org.logstash.common.DeadLetterQueueFactory
-java_import org.logstash.common.SourceWithMetadata
-java_import org.logstash.common.io.DeadLetterQueueWriter
-java_import org.logstash.config.ir.CompiledPipeline
-java_import org.logstash.config.ir.ConfigCompiler
-
-module LogStash; class JavaBasePipeline
-  include LogStash::Util::Loggable
-
-  attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :ephemeral_id
-  attr_reader :pipeline_config
-
-  def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
-    @logger = self.logger
-    @ephemeral_id = SecureRandom.uuid
-
-    @pipeline_config = pipeline_config
-    @config_str = pipeline_config.config_string
-    @settings = pipeline_config.settings
-    @config_hash = Digest::SHA1.hexdigest(@config_str)
-
-    @lir = ConfigCompiler.configToPipelineIR(
-      @config_str, @settings.get_value("config.support_escapes")
-    )
-
-    @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
-    @agent = agent
-    @dlq_writer = dlq_writer
-    @plugin_factory = LogStash::Plugins::PluginFactory.new(
-      # use NullMetric if called in the BasePipeline context otherwise use the @metric value
-      @lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, @metric || Instrument::NullMetric.new),
-      LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
-      JavaFilterDelegator
-    )
-    @lir_execution = CompiledPipeline.new(@lir, @plugin_factory)
-    if settings.get_value("config.debug") && @logger.debug?
-      @logger.debug("Compiled pipeline code", default_logging_keys(:code => @lir.get_graph.to_string))
-    end
-    @inputs = @lir_execution.inputs
-    @filters = @lir_execution.filters
-    @outputs = @lir_execution.outputs
-  end
-
-  def dlq_writer
-    if settings.get_value("dead_letter_queue.enable")
-      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"), settings.get_value("dead_letter_queue.max_bytes"))
-    else
-      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
-    end
-  end
-
-  def close_dlq_writer
-    @dlq_writer.close
-    if settings.get_value("dead_letter_queue.enable")
-      DeadLetterQueueFactory.release(pipeline_id)
-    end
-  end
-
-  def buildOutput(name, line, column, *args)
-    plugin("output", name, line, column, *args)
-  end
-
-  def buildFilter(name, line, column, *args)
-    plugin("filter", name, line, column, *args)
-  end
-
-  def buildInput(name, line, column, *args)
-    plugin("input", name, line, column, *args)
-  end
-
-  def buildCodec(name, *args)
-   plugin("codec", name, 0, 0, *args)
-  end
-
-  def plugin(plugin_type, name, line, column, *args)
-    @plugin_factory.plugin(plugin_type, name, line, column, *args)
-  end
-
-  def reloadable?
-    configured_as_reloadable? && reloadable_plugins?
-  end
-
-  def configured_as_reloadable?
-    settings.get("pipeline.reloadable")
-  end
-
-  def reloadable_plugins?
-    non_reloadable_plugins.empty?
-  end
-
-  def non_reloadable_plugins
-    (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
-  end
-
-  private
-
-  def default_logging_keys(other_keys = {})
-    { :pipeline_id => pipeline_id }.merge(other_keys)
-  end
-end; end
 
 module LogStash; class JavaPipeline < JavaBasePipeline
+  include LogStash::Util::Loggable
   attr_reader \
     :worker_threads,
     :events_consumed,
     :events_filtered,
-    :reporter,
     :started_at,
-    :thread,
-    :settings,
-    :metric,
-    :filter_queue_client,
-    :input_queue_client,
-    :queue
+    :thread
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
   def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
-    @settings = pipeline_config.settings
-    # This needs to be configured before we call super which will evaluate the code to make
-    # sure the metric instance is correctly send to the plugins to make the namespace scoping work
-    @metric = if namespaced_metric
-      settings.get("metric.collect") ? namespaced_metric : Instrument::NullMetric.new(namespaced_metric.collector)
-    else
-      Instrument::NullMetric.new
-    end
-
-    @ephemeral_id = SecureRandom.uuid
-    @settings = settings
-    @reporter = PipelineReporter.new(@logger, self)
+    @logger = self.logger
+    super pipeline_config, namespaced_metric, @logger, agent
     @worker_threads = []
 
-    super
-
-    begin
-      @queue = LogStash::QueueFactory.create(settings)
-    rescue => e
-      @logger.error("Logstash failed to create queue", default_logging_keys("exception" => e.message, "backtrace" => e.backtrace))
-      raise e
-    end
-
-    @input_queue_client = @queue.write_client
-    @filter_queue_client = @queue.read_client
-    # Note that @inflight_batches as a central mechanism for tracking inflight
-    # batches will fail if we have multiple read clients here.
-    @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
-    @filter_queue_client.set_pipeline_metric(
-        metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
-    )
-    @drain_queue =  @settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
+    @drain_queue =  settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
 
     @events_filtered = java.util.concurrent.atomic.LongAdder.new
     @events_consumed = java.util.concurrent.atomic.LongAdder.new
@@ -189,14 +45,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def safe_pipeline_worker_count
-    default = @settings.get_default("pipeline.workers")
-    pipeline_workers = @settings.get("pipeline.workers") #override from args "-w 8" or config
-    safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
+    default = settings.get_default("pipeline.workers")
+    pipeline_workers = settings.get("pipeline.workers") #override from args "-w 8" or config
+    safe_filters, unsafe_filters = filters.partition(&:threadsafe?)
     plugins = unsafe_filters.collect { |f| f.config_name }
 
     return pipeline_workers if unsafe_filters.empty?
 
-    if @settings.set?("pipeline.workers")
+    if settings.set?("pipeline.workers")
       if pipeline_workers > 1
         @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads", default_logging_keys(:worker_threads => pipeline_workers, :filters => plugins))
       end
@@ -213,7 +69,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def filters?
-    @filters.any?
+    filters.any?
   end
 
   def start
@@ -271,7 +127,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     start_workers
 
-    @logger.info("Pipeline started", "pipeline.id" => @pipeline_id)
+    @logger.info("Pipeline started", "pipeline.id" => pipeline_id)
 
     # Block until all inputs have stopped
     # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
@@ -294,12 +150,6 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     return 0
   end # def run
 
-  def close
-    @filter_queue_client.close
-    @queue.close
-    close_dlq_writer
-  end
-
   def transition_to_running
     @running.make_true
   end
@@ -316,10 +166,6 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @running.false?
   end
 
-  def system?
-    settings.get_value("pipeline.system")
-  end
-
   # register_plugins calls #register_plugin on the plugins list and upon exception will call Plugin#do_close on all registered plugins
   # @param plugins [Array[Plugin]] the list of plugins to register
   def register_plugins(plugins)
@@ -340,8 +186,8 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       maybe_setup_out_plugins
 
       pipeline_workers = safe_pipeline_worker_count
-      batch_size = @settings.get("pipeline.batch.size")
-      batch_delay = @settings.get("pipeline.batch.delay")
+      batch_size = settings.get("pipeline.batch.size")
+      batch_delay = settings.get("pipeline.batch.delay")
 
       max_inflight = batch_size * pipeline_workers
 
@@ -349,10 +195,10 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       config_metric.gauge(:workers, pipeline_workers)
       config_metric.gauge(:batch_size, batch_size)
       config_metric.gauge(:batch_delay, batch_delay)
-      config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
-      config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
+      config_metric.gauge(:config_reload_automatic, settings.get("config.reload.automatic"))
+      config_metric.gauge(:config_reload_interval, settings.get("config.reload.interval"))
       config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
-      config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
+      config_metric.gauge(:dead_letter_queue_path, dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
 
 
       @logger.info("Starting pipeline", default_logging_keys(
@@ -364,12 +210,12 @@ module LogStash; class JavaPipeline < JavaBasePipeline
         @logger.warn("CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})", default_logging_keys)
       end
 
-      @filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
+      filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
 
       pipeline_workers.times do |t|
         thread = Thread.new do
           org.logstash.execution.WorkerLoop.new(
-              @lir_execution, @filter_queue_client, @events_filtered, @events_consumed,
+              lir_execution, filter_queue_client, @events_filtered, @events_consumed,
               @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
         end
         thread.name="[#{pipeline_id}]>worker#{t}"
@@ -392,30 +238,26 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     end
   end
 
-  def dlq_enabled?
-    @settings.get("dead_letter_queue.enable")
-  end
-
   def wait_inputs
     @input_threads.each(&:join)
   end
 
   def start_inputs
     moreinputs = []
-    @inputs.each do |input|
+    inputs.each do |input|
       if input.threadable && input.threads > 1
         (input.threads - 1).times do |i|
           moreinputs << input.clone
         end
       end
     end
-    @inputs += moreinputs
+    moreinputs.each {|i| inputs << i}
 
     # first make sure we can register all input plugins
-    register_plugins(@inputs)
+    register_plugins(inputs)
 
     # then after all input plugins are successfully registered, start them
-    @inputs.each { |input| start_input(input) }
+    inputs.each { |input| start_input(input) }
   end
 
   def start_input(plugin)
@@ -425,7 +267,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   def inputworker(plugin)
     Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
     begin
-      plugin.run(LogStash::WrappedWriteClient.new(@input_queue_client, @pipeline_id.to_s.to_sym, metric, plugin.id.to_sym))
+      plugin.run(wrapped_write_client(plugin.id.to_sym))
     rescue => e
       if plugin.stop?
         @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
@@ -469,7 +311,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     # stopped
     wait_for_workers
     clear_pipeline_metrics
-    @logger.info("Pipeline terminated", "pipeline.id" => @pipeline_id)
+    @logger.info("Pipeline terminated", "pipeline.id" => pipeline_id)
   end # def shutdown
 
   def wait_for_workers
@@ -480,7 +322,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def stop_inputs
     @logger.debug("Closing inputs", default_logging_keys)
-    @inputs.each(&:do_stop)
+    inputs.each(&:do_stop)
     @logger.debug("Closed inputs", default_logging_keys)
   end
 
@@ -495,8 +337,8 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       t.join
     end
 
-    @filters.each(&:do_close)
-    @outputs.each(&:do_close)
+    filters.each(&:do_close)
+    outputs.each(&:do_close)
   end
 
   # for backward compatibility in devutils for the rspec helpers, this method is not used
@@ -542,41 +384,10 @@ module LogStash; class JavaPipeline < JavaBasePipeline
       .each {|t| t.delete("status") }
   end
 
-  def collect_dlq_stats
-    if dlq_enabled?
-      dlq_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :dlq])
-      dlq_metric.gauge(:queue_size_in_bytes, @dlq_writer.get_current_queue_size)
-    end
-  end
-
-  def collect_stats
-    pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
-    pipeline_metric.gauge(:type, settings.get("queue.type"))
-    if @queue.is_a?(LogStash::WrappedAckedQueue) && @queue.queue.is_a?(LogStash::AckedQueue)
-      queue = @queue.queue
-      dir_path = queue.dir_path
-      file_store = Files.get_file_store(Paths.get(dir_path))
-
-      pipeline_metric.namespace([:capacity]).tap do |n|
-        n.gauge(:page_capacity_in_bytes, queue.page_capacity)
-        n.gauge(:max_queue_size_in_bytes, queue.max_size_in_bytes)
-        n.gauge(:max_unread_events, queue.max_unread_events)
-        n.gauge(:queue_size_in_bytes, queue.persisted_size_in_bytes)
-      end
-      pipeline_metric.namespace([:data]).tap do |n|
-        n.gauge(:free_space_in_bytes, file_store.get_unallocated_space)
-        n.gauge(:storage_type, file_store.type)
-        n.gauge(:path, dir_path)
-      end
-
-      pipeline_metric.gauge(:events, queue.unread_count)
-    end
-  end
-
   def clear_pipeline_metrics
     # TODO(ph): I think the metric should also proxy that call correctly to the collector
     # this will simplify everything since the null metric would simply just do a noop
-    collector = @metric.collector
+    collector = metric.collector
 
     unless collector.nil?
       # selectively reset metrics we don't wish to keep after reloading
@@ -592,8 +403,8 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   # We want to hide most of what's in here
   def inspect
     {
-      :pipeline_id => @pipeline_id,
-      :settings => @settings.inspect,
+      :pipeline_id => pipeline_id,
+      :settings => settings.inspect,
       :ready => @ready,
       :running => @running,
       :flushing => @flushing
@@ -604,13 +415,13 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def maybe_setup_out_plugins
     if @outputs_registered.make_true
-      register_plugins(@outputs)
-      register_plugins(@filters)
+      register_plugins(outputs)
+      register_plugins(filters)
     end
   end
 
   def default_logging_keys(other_keys = {})
-    keys = super
+    keys = {:pipeline_id => pipeline_id}.merge other_keys
     keys[:thread] ||= thread.inspect if thread
     keys
   end