logstash-core 6.3.2-java → 6.4.0-java

data/lib/logstash/json.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/environment"
 require "jrjackson"
-require "logstash/java_integration"
 
 module LogStash
   module Json

data/lib/logstash/logging.rb

@@ -1,3 +1 @@
 # encoding: utf-8
-require "logstash/logging/logger"
-require "logstash/namespace"

data/lib/logstash/logging/logger.rb

@@ -1,159 +1 @@
-require "logstash/java_integration"
-require "uri"
-
-module LogStash
-  module Logging
-    java_import org.apache.logging.log4j.Level
-    java_import org.apache.logging.log4j.LogManager
-    java_import org.apache.logging.log4j.core.config.Configurator
-    java_import org.apache.logging.log4j.core.config.DefaultConfiguration
-    java_import org.apache.logging.log4j.core.config.LoggerConfig
-    java_import org.logstash.log.LogstashLoggerContextFactory
-    java_import org.apache.logging.log4j.core.LoggerContext
-    java_import java.net.URI
-
-    class Logger
-      @@config_mutex = Mutex.new
-
-      def initialize(name)
-        @logger = LogManager.getLogger(name)
-      end
-
-      def debug?
-        @logger.is_debug_enabled
-      end
-
-      def info?
-        @logger.is_info_enabled
-      end
-
-      def error?
-        @logger.is_error_enabled
-      end
-
-      def warn?
-        @logger.is_warn_enabled
-      end
-
-      def fatal?
-        @logger.is_fatal_enabled
-      end
-
-      def trace?
-        @logger.is_trace_enabled
-      end
-
-      def debug(message, data = {})
-        @logger.debug(message, data)
-      end
-
-      def warn(message, data = {})
-        @logger.warn(message, data)
-      end
-
-      def info(message, data = {})
-        @logger.info(message, data)
-      end
-
-      def error(message, data = {})
-        @logger.error(message, data)
-      end
-
-      def fatal(message, data = {})
-        @logger.fatal(message, data)
-      end
-
-      def trace(message, data = {})
-        @logger.trace(message, data)
-      end
-
-      def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
-        @@config_mutex.synchronize { set_level(level, path) }
-      rescue Exception => e
-        raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
-      end
-
-      def self.reconfigure(config_location)
-        @@config_mutex.synchronize do
-          config_location_uri = URI.create(config_location)
-          file_path = config_location_uri.path
-          if ::File.exists?(file_path)
-            logs_location = java.lang.System.getProperty("ls.logs")
-            puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
-            #reconfigure the default context to use our log4j2.properties file
-            get_logging_context.setConfigLocation(URI.create(config_location))
-            #ensure everyone agrees which context to use for the LogManager
-            context_factory = LogstashLoggerContextFactory.new(get_logging_context)
-            LogManager.setFactory(context_factory)
-          else
-            # fall back to default config
-            puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs errors to the console"
-          end
-        end
-      end
-
-      # until dev_utils/rspec/spec_helper is changed, we need to have both methods
-      singleton_class.send(:alias_method, :initialize, :reconfigure)
-
-      def self.get_logging_context
-        return  LoggerContext.getContext(false)
-      end
-
-      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but ensure the proper context is used
-      def self.set_level(_level, path)
-        configuration =  get_logging_context.getConfiguration()
-        level = Level.valueOf(_level)
-        if path.nil? || path.strip.empty?
-          root_logger = configuration.getRootLogger()
-          if root_logger.getLevel() != level
-            root_logger.setLevel(level)
-            get_logging_context.updateLoggers()
-          end
-        else
-          package_logger = configuration.getLoggerConfig(path)
-          if package_logger.name != path #no package logger found
-            configuration.addLogger(path, LoggerConfig.new(path, level, true))
-            get_logging_context.updateLoggers()
-          elsif package_logger.getLevel() != level
-            package_logger.setLevel(level)
-            get_logging_context.updateLoggers()
-          end
-        end
-      end
-
-      private_class_method :set_level
-    end
-
-    class SlowLogger
-      def initialize(name, warn_threshold, info_threshold, debug_threshold, trace_threshold)
-        slowlog_name = ["slowlog", name].join('.')
-        @slowlogger = LogManager.getLogger(slowlog_name)
-        @warn_threshold = warn_threshold
-        @info_threshold = info_threshold
-        @debug_threshold = debug_threshold
-        @trace_threshold = trace_threshold
-      end
-
-      def as_data(plugin_params, event, took_in_nanos)
-        {
-          :plugin_params => plugin_params,
-          :took_in_nanos => took_in_nanos,
-          :took_in_millis => took_in_nanos / 1000000,
-          :event => event.to_json
-        }
-      end
-
-      def on_event(message, plugin_params, event, took_in_nanos)
-        if @warn_threshold >= 0 and took_in_nanos > @warn_threshold
-          @slowlogger.warn(message, as_data(plugin_params, event, took_in_nanos))
-        elsif @info_threshold >= 0 and took_in_nanos > @info_threshold
-          @slowlogger.info(message, as_data(plugin_params, event, took_in_nanos))
-        elsif @debug_threshold >= 0 and took_in_nanos > @debug_threshold
-          @slowlogger.debug(message, as_data(plugin_params, event, took_in_nanos))
-        elsif @trace_threshold >= 0 and took_in_nanos > @trace_threshold
-          @slowlogger.trace(message, as_data(plugin_params, event, took_in_nanos))
-        end
-      end
-    end
-  end
-end
+# Keeping this file for backwards compatibility with plugins that include it directly.

data/lib/logstash/modules/cli_parser.rb

@@ -1,8 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require "logstash/errors"
-
 module LogStash module Modules class CLIParser
   include LogStash::Util::Loggable
 

data/lib/logstash/modules/elasticsearch_config.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-
 require_relative "elasticsearch_resource"
 
 module LogStash module Modules class ElasticsearchConfig

data/lib/logstash/modules/elasticsearch_importer.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-
 module LogStash module Modules class ElasticsearchImporter
   include LogStash::Util::Loggable
 

data/lib/logstash/modules/elasticsearch_resource.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require_relative "resource_base"
 
 module LogStash module Modules class ElasticsearchResource

data/lib/logstash/modules/file_reader.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require "logstash/json"
 
 module LogStash module Modules class FileReader

data/lib/logstash/modules/kibana_base.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/json"
 
 module LogStash module Modules class KibanaBase

data/lib/logstash/modules/kibana_client.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require "logstash/json"
 require "manticore"
 

data/lib/logstash/modules/kibana_config.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-
 require_relative "file_reader"
 require_relative "kibana_settings"
 require_relative "kibana_dashboards"

data/lib/logstash/modules/kibana_dashboards.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require_relative "kibana_base"
 
 module LogStash module Modules class KibanaDashboards < KibanaBase

data/lib/logstash/modules/kibana_importer.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-
 module LogStash module Modules class KibanaImporter
   include LogStash::Util::Loggable
 

data/lib/logstash/modules/kibana_resource.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require_relative "resource_base"
 
 module LogStash module Modules class KibanaResource

data/lib/logstash/modules/kibana_settings.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require_relative "kibana_base"
 
 module LogStash module Modules class KibanaSettings < KibanaBase

data/lib/logstash/modules/logstash_config.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require_relative "file_reader"
 require "logstash/settings"
 

data/lib/logstash/modules/resource_base.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/json"
 require_relative "file_reader"
 

data/lib/logstash/modules/scaffold.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require "logstash/util/loggable"
 require "erb"
 
 require_relative "elasticsearch_config"

data/lib/logstash/modules/settings_merger.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/util"
-require "logstash/util/loggable"
 
 module LogStash module Modules module SettingsMerger
   include LogStash::Util::Loggable

data/lib/logstash/namespace.rb

@@ -1,15 +1,2 @@
-# encoding: utf-8
-module LogStash
-  module Inputs; end
-  module Outputs; end
-  module Filters; end
-  module Search; end
-  module Config; end
-  module File; end
-  module Web; end
-  module Util; end
-  module PluginMixins; end
-  module PluginManager; end
-  module Api; end
-  module Modules; end
-end # module LogStash
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly require it.

data/lib/logstash/outputs/base.rb

@@ -1,8 +1,6 @@
 # encoding: utf-8
 require "logstash/event"
-require "logstash/logging"
 require "logstash/plugin"
-require "logstash/namespace"
 require "logstash/config/mixin"
 require "concurrent/atomic/atomic_fixnum"
 
@@ -24,7 +22,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
   # when we no longer support the :legacy type
   # This is hacky, but it can only be herne
   config :workers, :type => :number, :default => 1
-  
+
   # Set or return concurrency type
   def self.concurrency(type=nil)
     if type
@@ -68,7 +66,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
     # If we're running with a single thread we must enforce single-threaded concurrency by default
     # Maybe in a future version we'll assume output plugins are threadsafe
     @single_worker_mutex = Mutex.new
-    
+
     @receives_encoded = self.methods.include?(:multi_receive_encoded)
   end
 
@@ -108,7 +106,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
-    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their
     #               parent plugin's
     @codec.execution_context = context
     context

data/lib/logstash/patches/clamp.rb

@@ -48,6 +48,12 @@ module Clamp
         end
       end
 
+      def define_appender_for(option)
+        define_method(option.append_method) do |value|
+          LogStash::SETTINGS.get_value(option.attribute_name) << value
+        end
+      end
+
       def define_deprecated_accessors_for(option, opts, &block)
         define_deprecated_writer_for(option, opts, &block)
       end

data/lib/logstash/pipeline.rb

@@ -2,67 +2,34 @@
 require "thread"
 require "stud/interval"
 require "concurrent"
-require "logstash/namespace"
-require "logstash/errors"
 require "logstash-core/logstash-core"
 require "logstash/event"
 require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
-require "logstash/shutdown_watcher"
-require "logstash/pipeline_reporter"
-require "logstash/instrument/metric"
-require "logstash/instrument/namespaced_metric"
-require "logstash/instrument/null_metric"
-require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
-require "logstash/util/dead_letter_queue_manager"
 require "logstash/filter_delegator"
-require "logstash/queue_factory"
-require "logstash/plugins/plugin_factory"
 require "logstash/compiler"
-require "logstash/execution_context"
-require "securerandom"
 
-java_import org.logstash.common.DeadLetterQueueFactory
-java_import org.logstash.common.SourceWithMetadata
-java_import org.logstash.common.io.DeadLetterQueueWriter
-java_import org.logstash.config.ir.ConfigCompiler
-
-module LogStash; class BasePipeline
+module LogStash; class BasePipeline < AbstractPipeline
   include LogStash::Util::Loggable
 
-  attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :execution_context, :ephemeral_id
-  attr_reader :pipeline_config
+  attr_reader :inputs, :filters, :outputs
 
   def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @logger = self.logger
-    @mutex = Mutex.new
-    @ephemeral_id = SecureRandom.uuid
-
-    @pipeline_config = pipeline_config
-    @config_str = pipeline_config.config_string
-    @settings = pipeline_config.settings
-    @config_hash = Digest::SHA1.hexdigest(@config_str)
-
-    @lir = ConfigCompiler.configToPipelineIR(
-      @config_str, @settings.get_value("config.support_escapes")
-    )
-
-    @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
+    super pipeline_config, namespaced_metric, @logger
 
     @inputs = nil
     @filters = nil
     @outputs = nil
     @agent = agent
 
-    @dlq_writer = dlq_writer
-
     @plugin_factory = LogStash::Plugins::PluginFactory.new(
       # use NullMetric if called in the BasePipeline context otherwise use the @metric value
-      @lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, @metric || Instrument::NullMetric.new),
-      LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
+      lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, metric),
+      LogStash::Plugins::ExecutionContextFactory.new(@agent, self, dlq_writer),
       FilterDelegator
     )
     grammar = LogStashConfigParser.new
@@ -85,39 +52,10 @@ module LogStash; class BasePipeline
     end
   end
 
-  def dlq_writer
-    if settings.get_value("dead_letter_queue.enable")
-      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"), settings.get_value("dead_letter_queue.max_bytes"))
-    else
-      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
-    end
-  end
-
-  def close_dlq_writer
-    @dlq_writer.close
-    if settings.get_value("dead_letter_queue.enable")
-      DeadLetterQueueFactory.release(pipeline_id)
-    end
-  end
-
-  def compile_lir
-    org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
-      self.config_str, @settings.get_value("config.support_escapes")
-    )
-  end
-
-  def plugin(plugin_type, name, line, column, *args)
-    @plugin_factory.plugin(plugin_type, name, line, column, *args)
-  end
-
   def reloadable?
     configured_as_reloadable? && reloadable_plugins?
   end
 
-  def configured_as_reloadable?
-    settings.get("pipeline.reloadable")
-  end
-
   def reloadable_plugins?
     non_reloadable_plugins.empty?
   end
@@ -128,6 +66,11 @@ module LogStash; class BasePipeline
 
   private
 
+
+  def plugin(plugin_type, name, line, column, *args)
+    @plugin_factory.plugin(plugin_type, name, line, column, *args)
+  end
+
   def default_logging_keys(other_keys = {})
     { :pipeline_id => pipeline_id }.merge(other_keys)
   end
@@ -138,51 +81,19 @@ module LogStash; class Pipeline < BasePipeline
     :worker_threads,
     :events_consumed,
     :events_filtered,
-    :reporter,
     :started_at,
-    :thread,
-    :settings,
-    :metric,
-    :filter_queue_client,
-    :input_queue_client,
-    :queue
+    :thread
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
   def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
-    @settings = pipeline_config.settings
-    # This needs to be configured before we call super which will evaluate the code to make
-    # sure the metric instance is correctly send to the plugins to make the namespace scoping work
-    @metric = if namespaced_metric
-      settings.get("metric.collect") ? namespaced_metric : Instrument::NullMetric.new(namespaced_metric.collector)
-    else
-      Instrument::NullMetric.new
-    end
-
-    @ephemeral_id = SecureRandom.uuid
-    @settings = settings
-    @reporter = PipelineReporter.new(@logger, self)
-    @worker_threads = []
-
     super
 
-    begin
-      @queue = LogStash::QueueFactory.create(settings)
-    rescue => e
-      @logger.error("Logstash failed to create queue", default_logging_keys("exception" => e.message, "backtrace" => e.backtrace))
-      raise e
-    end
+    @worker_threads = []
 
-    @input_queue_client = @queue.write_client
-    @filter_queue_client = @queue.read_client
     @signal_queue = java.util.concurrent.LinkedBlockingQueue.new
-    # Note that @inflight_batches as a central mechanism for tracking inflight
-    # batches will fail if we have multiple read clients here.
-    @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
-    @filter_queue_client.set_pipeline_metric(
-        metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
-    )
-    @drain_queue =  @settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
+
+    @drain_queue =  settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
 
 
     @events_filtered = java.util.concurrent.atomic.LongAdder.new
@@ -202,14 +113,14 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   def safe_pipeline_worker_count
-    default = @settings.get_default("pipeline.workers")
-    pipeline_workers = @settings.get("pipeline.workers") #override from args "-w 8" or config
+    default = settings.get_default("pipeline.workers")
+    pipeline_workers = settings.get("pipeline.workers") #override from args "-w 8" or config
     safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
     plugins = unsafe_filters.collect { |f| f.config_name }
 
     return pipeline_workers if unsafe_filters.empty?
 
-    if @settings.set?("pipeline.workers")
+    if settings.set?("pipeline.workers")
       if pipeline_workers > 1
         @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads", default_logging_keys(:worker_threads => pipeline_workers, :filters => plugins))
       end
@@ -236,9 +147,9 @@ module LogStash; class Pipeline < BasePipeline
     collect_dlq_stats
 
     @logger.info("Starting pipeline", default_logging_keys(
-      "pipeline.workers" => @settings.get("pipeline.workers"),
-      "pipeline.batch.size" => @settings.get("pipeline.batch.size"),
-      "pipeline.batch.delay" => @settings.get("pipeline.batch.delay")))
+      "pipeline.workers" => settings.get("pipeline.workers"),
+      "pipeline.batch.size" => settings.get("pipeline.batch.size"),
+      "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
 
     @finished_execution = Concurrent::AtomicBoolean.new(false)
 
@@ -307,12 +218,6 @@ module LogStash; class Pipeline < BasePipeline
     return 0
   end # def run
 
-  def close
-    @filter_queue_client.close
-    @queue.close
-    close_dlq_writer
-  end
-
   def transition_to_running
     @running.make_true
   end
@@ -329,10 +234,6 @@ module LogStash; class Pipeline < BasePipeline
     @running.false?
   end
 
-  def system?
-    settings.get_value("pipeline.system")
-  end
-
   # register_plugin simply calls the plugin #register method and catches & logs any error
   # @param plugin [Plugin] the plugin to register
   # @return [Plugin] the registered plugin
@@ -361,8 +262,8 @@ module LogStash; class Pipeline < BasePipeline
       maybe_setup_out_plugins
 
       pipeline_workers = safe_pipeline_worker_count
-      batch_size = @settings.get("pipeline.batch.size")
-      batch_delay = @settings.get("pipeline.batch.delay")
+      batch_size = settings.get("pipeline.batch.size")
+      batch_delay = settings.get("pipeline.batch.delay")
 
       max_inflight = batch_size * pipeline_workers
 
@@ -370,10 +271,10 @@ module LogStash; class Pipeline < BasePipeline
       config_metric.gauge(:workers, pipeline_workers)
       config_metric.gauge(:batch_size, batch_size)
       config_metric.gauge(:batch_delay, batch_delay)
-      config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
-      config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
+      config_metric.gauge(:config_reload_automatic, settings.get("config.reload.automatic"))
+      config_metric.gauge(:config_reload_interval, settings.get("config.reload.interval"))
       config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
-      config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
+      config_metric.gauge(:dead_letter_queue_path, dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
 
       if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
         @logger.warn("CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})", default_logging_keys)
@@ -403,19 +304,15 @@ module LogStash; class Pipeline < BasePipeline
     end
   end
 
-  def dlq_enabled?
-    @settings.get("dead_letter_queue.enable")
-  end
-
   # Main body of what a worker thread does
   # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
-    @filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
+    filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
     output_events_map = Hash.new { |h, k| h[k] = [] }
     while true
       signal = @signal_queue.poll || NO_SIGNAL
 
-      batch = @filter_queue_client.read_batch.to_java # metrics are started in read_batch
+      batch = filter_queue_client.read_batch.to_java # metrics are started in read_batch
       batch_size = batch.filteredSize
       if batch_size > 0
         @events_consumed.add(batch_size)
@@ -424,7 +321,7 @@ module LogStash; class Pipeline < BasePipeline
       flush_filters_to_batch(batch, :final => false) if signal.flush?
       if batch.filteredSize > 0
         output_batch(batch, output_events_map)
-        @filter_queue_client.close_batch(batch)
+        filter_queue_client.close_batch(batch)
       end
       # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
       break if (@worker_shutdown.get && !draining_queue?)
@@ -432,11 +329,11 @@ module LogStash; class Pipeline < BasePipeline
 
     # we are shutting down, queue is drained if it was required, now  perform a final flush.
     # for this we need to create a new empty batch to contain the final flushed events
-    batch = @filter_queue_client.to_java.newBatch
-    @filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
+    batch = filter_queue_client.to_java.newBatch
+    filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
     flush_filters_to_batch(batch, :final => true)
     output_batch(batch, output_events_map)
-    @filter_queue_client.close_batch(batch)
+    filter_queue_client.close_batch(batch)
   end
 
   def filter_batch(batch)
@@ -444,7 +341,7 @@ module LogStash; class Pipeline < BasePipeline
       #these are both original and generated events
       batch.merge(e) unless e.cancelled?
     end
-    @filter_queue_client.add_filtered_metrics(batch.filtered_size)
+    filter_queue_client.add_filtered_metrics(batch.filtered_size)
     @events_filtered.add(batch.filteredSize)
   rescue Exception => e
     # Plugins authors should manage their own exceptions in the plugin code
@@ -476,7 +373,7 @@ module LogStash; class Pipeline < BasePipeline
       events.clear
     end
 
-    @filter_queue_client.add_output_metrics(batch.filtered_size)
+    filter_queue_client.add_output_metrics(batch.filtered_size)
   end
 
   def wait_inputs
@@ -508,8 +405,7 @@ module LogStash; class Pipeline < BasePipeline
   def inputworker(plugin)
     Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
     begin
-      input_queue_client = wrapped_write_client(plugin.id.to_sym)
-      plugin.run(input_queue_client)
+      plugin.run(wrapped_write_client(plugin.id.to_sym))
     rescue => e
       if plugin.stop?
         @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
@@ -663,41 +559,10 @@ module LogStash; class Pipeline < BasePipeline
       .each {|t| t.delete("status") }
   end
 
-  def collect_dlq_stats
-    if dlq_enabled?
-      dlq_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :dlq])
-      dlq_metric.gauge(:queue_size_in_bytes, @dlq_writer.get_current_queue_size)
-    end
-  end
-
-  def collect_stats
-    pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
-    pipeline_metric.gauge(:type, settings.get("queue.type"))
-    if @queue.is_a?(LogStash::WrappedAckedQueue) && @queue.queue.is_a?(LogStash::AckedQueue)
-      queue = @queue.queue
-      dir_path = queue.dir_path
-      file_store = Files.get_file_store(Paths.get(dir_path))
-
-      pipeline_metric.namespace([:capacity]).tap do |n|
-        n.gauge(:page_capacity_in_bytes, queue.page_capacity)
-        n.gauge(:max_queue_size_in_bytes, queue.max_size_in_bytes)
-        n.gauge(:max_unread_events, queue.max_unread_events)
-        n.gauge(:queue_size_in_bytes, queue.persisted_size_in_bytes)
-      end
-      pipeline_metric.namespace([:data]).tap do |n|
-        n.gauge(:free_space_in_bytes, file_store.get_unallocated_space)
-        n.gauge(:storage_type, file_store.type)
-        n.gauge(:path, dir_path)
-      end
-
-      pipeline_metric.gauge(:events, queue.unread_count)
-    end
-  end
-
   def clear_pipeline_metrics
     # TODO(ph): I think the metric should also proxy that call correctly to the collector
     # this will simplify everything since the null metric would simply just do a noop
-    collector = @metric.collector
+    collector = metric.collector
 
     unless collector.nil?
       # selectively reset metrics we don't wish to keep after reloading
@@ -713,8 +578,8 @@ module LogStash; class Pipeline < BasePipeline
   # We want to hide most of what's in here
   def inspect
     {
-      :pipeline_id => @pipeline_id,
-      :settings => @settings.inspect,
+      :pipeline_id => pipeline_id,
+      :settings => settings.inspect,
       :ready => @ready,
       :running => @running,
       :flushing => @flushing
@@ -737,13 +602,6 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   def draining_queue?
-    @drain_queue ? !@filter_queue_client.empty? : false
-  end
-
-  def wrapped_write_client(plugin_id)
-    #need to ensure that metrics are initialized one plugin at a time, else a race condition can exist.
-    @mutex.synchronize do
-      LogStash::WrappedWriteClient.new(@input_queue_client, @pipeline_id.to_s.to_sym, metric, plugin_id)
-    end
+    @drain_queue ? !filter_queue_client.empty? : false
   end
 end; end