logstash-core 6.0.1-java → 6.1.0-java

data/lib/logstash/json.rb

@@ -1,36 +1,12 @@
 # encoding: utf-8
 require "logstash/environment"
-require "logstash/errors"
-if LogStash::Environment.jruby?
-  require "jrjackson"
-  require "logstash/java_integration"
-else
-  require  "oj"
-end
+require "jrjackson"
+require "logstash/java_integration"
 
 module LogStash
   module Json
-    class ParserError < LogStash::Error; end
-    class GeneratorError < LogStash::Error; end
-
     extend self
 
-    ### MRI
-
-    def mri_load(data, options = {})
-      Oj.load(data)
-    rescue Oj::ParseError => e
-      raise LogStash::Json::ParserError.new(e.message)
-    end
-
-    def mri_dump(o)
-      Oj.dump(o, :mode => :compat, :use_to_json => true)
-    rescue => e
-      raise LogStash::Json::GeneratorError.new(e.message)
-    end
-
-    ### JRuby
-
     def jruby_load(data, options = {})
       # TODO [guyboertje] remove these comments in 5.0
       # options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
@@ -52,9 +28,8 @@ module LogStash
       raise LogStash::Json::GeneratorError.new(e.message)
     end
 
-    prefix = LogStash::Environment.jruby? ? "jruby" : "mri"
-    alias_method :load, "#{prefix}_load".to_sym
-    alias_method :dump, "#{prefix}_dump".to_sym
+    alias_method :load, "jruby_load".to_sym
+    alias_method :dump, "jruby_dump".to_sym
 
   end
 end

data/lib/logstash/output_delegator.rb

@@ -44,11 +44,12 @@ module LogStash class OutputDelegator
   end
 
   def multi_receive(events)
-    @in_counter.increment(events.length)
+    count = events.size
+    @in_counter.increment(count)
     start_time = java.lang.System.nano_time
     @strategy.multi_receive(events)
     @time_metric.increment((java.lang.System.nano_time - start_time) / 1_000_000)
-    @out_counter.increment(events.length)
+    @out_counter.increment(count)
   end
 
   def do_close

data/lib/logstash/patches/bugfix_jruby_2558.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require "logstash/environment"
 
-if LogStash::Environment.windows? && LogStash::Environment.jruby?
+if LogStash::Environment.windows?
   require "socket"
   module JRubyBug2558SocketPeerAddrBugFix
     def peeraddr(*args)

data/lib/logstash/pipeline.rb

@@ -22,6 +22,7 @@ require "logstash/util/dead_letter_queue_manager"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
 require "logstash/queue_factory"
+require "logstash/plugins/plugin_factory"
 require "logstash/compiler"
 require "logstash/execution_context"
 require "securerandom"
@@ -29,6 +30,7 @@ require "securerandom"
 java_import org.logstash.common.DeadLetterQueueFactory
 java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
+java_import org.logstash.config.ir.ConfigCompiler
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
@@ -46,16 +48,12 @@ module LogStash; class BasePipeline
     @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
-    @lir = compile_lir
-
-    # Every time #plugin is invoked this is incremented to give each plugin
-    # a unique id when auto-generating plugin ids
-    @plugin_counter ||= 0
+    @lir = ConfigCompiler.configToPipelineIR(
+      @config_str, @settings.get_value("config.support_escapes")
+    )
 
     @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
 
-    # A list of plugins indexed by id
-    @plugins_by_id = {}
     @inputs = nil
     @filters = nil
     @outputs = nil
@@ -63,6 +61,12 @@ module LogStash; class BasePipeline
 
     @dlq_writer = dlq_writer
 
+    @plugin_factory = LogStash::Plugins::PluginFactory.new(
+      # use NullMetric if called in the BasePipeline context otherwise use the @metric value
+      @lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, @metric || Instrument::NullMetric.new),
+      @logger, LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
+      FilterDelegator
+    )
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
     raise(ConfigurationError, grammar.failure_reason) if parsed_config.nil?
@@ -101,61 +105,13 @@ module LogStash; class BasePipeline
   end
 
   def compile_lir
-    sources_with_metadata = [
-      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
-    ]
-    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
+    org.logstash.config.ir.ConfigCompiler.configToPipelineIR(
+      self.config_str, @settings.get_value("config.support_escapes")
+    )
   end
 
   def plugin(plugin_type, name, line, column, *args)
-    @plugin_counter += 1
-
-    # Collapse the array of arguments into a single merged hash
-    args = args.reduce({}, &:merge)
-
-    if plugin_type == "codec"
-      id = SecureRandom.uuid # codecs don't really use their IDs for metrics, so we can use anything here
-    else
-      # Pull the ID from LIR to keep IDs consistent between the two representations
-      id = lir.graph.vertices.filter do |v| 
-        v.source_with_metadata && 
-        v.source_with_metadata.line == line && 
-        v.source_with_metadata.column == column
-      end.findFirst.get.id
-    end
-
-    args["id"] = id # some code pulls the id out of the args
-
-    if !id
-      raise ConfigurationError, "Could not determine ID for #{plugin_type}/#{plugin_name}"
-    end
-
-    raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
-    @plugins_by_id[id] = true
-
-    # use NullMetric if called in the BasePipeline context otherwise use the @metric value
-    metric = @metric || Instrument::NullMetric.new
-
-    pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
-    # Scope plugins of type 'input' to 'inputs'
-    type_scoped_metric = pipeline_scoped_metric.namespace("#{plugin_type}s".to_sym)
-
-    klass = Plugin.lookup(plugin_type, name)
-
-    execution_context = ExecutionContext.new(self, @agent, id, klass.config_name, @dlq_writer)
-
-    if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
-    elsif plugin_type == "filter"
-      FilterDelegator.new(@logger, klass, type_scoped_metric, execution_context, args)
-    else # input
-      input_plugin = klass.new(args)
-      scoped_metric = type_scoped_metric.namespace(id.to_sym)
-      scoped_metric.gauge(:name, input_plugin.config_name)
-      input_plugin.metric = scoped_metric
-      input_plugin.execution_context = execution_context
-      input_plugin
-    end
+    @plugin_factory.plugin(plugin_type, name, line, column, *args)
   end
 
   def reloadable?
@@ -241,7 +197,6 @@ module LogStash; class Pipeline < BasePipeline
     @ready = Concurrent::AtomicBoolean.new(false)
     @running = Concurrent::AtomicBoolean.new(false)
     @flushing = Concurrent::AtomicReference.new(false)
-    @force_shutdown = Concurrent::AtomicBoolean.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
   end # def initialize
 
@@ -434,10 +389,11 @@ module LogStash; class Pipeline < BasePipeline
       end
 
       pipeline_workers.times do |t|
-        @worker_threads << Thread.new do
-          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
-          worker_loop(batch_size, batch_delay)
+        thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
+          _pipeline.worker_loop(_b_size, _b_delay)
         end
+        thread.name="[#{pipeline_id}]>worker#{t}"
+        @worker_threads << thread
       end
 
       # inputs should be started last, after all workers
@@ -466,25 +422,24 @@ module LogStash; class Pipeline < BasePipeline
     shutdown_requested = false
 
     @filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
-
+    output_events_map = Hash.new { |h, k| h[k] = [] }
     while true
       signal = @signal_queue.poll || NO_SIGNAL
       shutdown_requested |= signal.shutdown? # latch on shutdown signal
 
       batch = @filter_queue_client.read_batch # metrics are started in read_batch
-      if batch.size > 0
-        @events_consumed.increment(batch.size)
+      batch_size = batch.size
+      if batch_size > 0
+        @events_consumed.increment(batch_size)
         filter_batch(batch)
       end
       flush_filters_to_batch(batch, :final => false) if signal.flush?
       if batch.size > 0
-        output_batch(batch)
-        unless @force_shutdown.true? # ack the current batch
-          @filter_queue_client.close_batch(batch)
-        end
+        output_batch(batch, output_events_map)
+        @filter_queue_client.close_batch(batch)
       end
       # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
-      break if (shutdown_requested && !draining_queue?) || @force_shutdown.true?
+      break if (shutdown_requested && !draining_queue?)
     end
 
     # we are shutting down, queue is drained if it was required, now  perform a final flush.
@@ -492,8 +447,7 @@ module LogStash; class Pipeline < BasePipeline
     batch = @filter_queue_client.new_batch
     @filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
     flush_filters_to_batch(batch, :final => true)
-    return if @force_shutdown.true? # Do not ack the current batch
-    output_batch(batch)
+    output_batch(batch, output_events_map)
     @filter_queue_client.close_batch(batch)
   end
 
@@ -502,7 +456,7 @@ module LogStash; class Pipeline < BasePipeline
       #these are both original and generated events
       batch.merge(e) unless e.cancelled?
     end
-    @filter_queue_client.add_filtered_metrics(batch)
+    @filter_queue_client.add_filtered_metrics(batch.filtered_size)
     @events_filtered.increment(batch.size)
   rescue Exception => e
     # Plugins authors should manage their own exceptions in the plugin code
@@ -518,27 +472,23 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   # Take an array of events and send them to the correct output
-  def output_batch(batch)
+  def output_batch(batch, output_events_map)
     # Build a mapping of { output_plugin => [events...]}
-    output_events_map = Hash.new { |h, k| h[k] = [] }
     batch.each do |event|
       # We ask the AST to tell us which outputs to send each event to
       # Then, we stick it in the correct bin
-
-      # output_func should never return anything other than an Array but we have lots of legacy specs
-      # that monkeypatch it and return nil. We can deprecate  "|| []" after fixing these specs
-      (output_func(event) || []).each do |output|
+      output_func(event).each do |output|
         output_events_map[output].push(event)
       end
     end
     # Now that we have our output to event mapping we can just invoke each output
     # once with its list of events
     output_events_map.each do |output, events|
-      return if @force_shutdown.true?
       output.multi_receive(events)
+      events.clear
     end
 
-    @filter_queue_client.add_output_metrics(batch)
+    @filter_queue_client.add_output_metrics(batch.filtered_size)
   end
 
   def wait_inputs
@@ -618,10 +568,6 @@ module LogStash; class Pipeline < BasePipeline
     @logger.info("Pipeline terminated", "pipeline.id" => @pipeline_id)
   end # def shutdown
 
-  def force_shutdown!
-    @force_shutdown.make_true
-  end
-
   def wait_for_workers
     @logger.debug("Closing inputs", default_logging_keys)
     @worker_threads.map(&:join)
@@ -668,7 +614,6 @@ module LogStash; class Pipeline < BasePipeline
     flushers = options[:final] ? @shutdown_flushers : @periodic_flushers
 
     flushers.each do |flusher|
-      return if @force_shutdown.true?
       flusher.call(options, &block)
     end
   end
@@ -710,8 +655,6 @@ module LogStash; class Pipeline < BasePipeline
   # @param options [Hash]
   def flush_filters_to_batch(batch, options = {})
     flush_filters(options) do |event|
-      return if @force_shutdown.true?
-
       unless event.cancelled?
         @logger.debug? and @logger.debug("Pushing flushed events", default_logging_keys(:event => event))
         batch.merge(event)

data/lib/logstash/pipeline_action/create.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/pipeline_action/base"
 require "logstash/pipeline"
+require "logstash/java_pipeline"
 require "logstash/converge_result"
 require "logstash/util/loggable"
 
@@ -32,8 +33,13 @@ module LogStash module PipelineAction
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
     def execute(agent, pipelines)
-      pipeline = LogStash::Pipeline.new(@pipeline_config, @metric, agent)
-      
+      pipeline =
+        if @pipeline_config.settings.get_value("pipeline.java_execution")
+          LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
+        else
+          LogStash::Pipeline.new(@pipeline_config, @metric, agent)
+        end
+
       status = pipeline.start # block until the pipeline is correctly started or crashed
 
       if status

data/lib/logstash/pipeline_action/reload.rb

@@ -27,7 +27,12 @@ module LogStash module PipelineAction
       end
 
       begin
-        pipeline_validator = LogStash::BasePipeline.new(@pipeline_config)
+        pipeline_validator =
+          if @pipeline_config.settings.get_value("pipeline.java_execution")
+            LogStash::JavaBasePipeline.new(@pipeline_config)
+          else
+            LogStash::BasePipeline.new(@pipeline_config)
+          end
       rescue => e
         return LogStash::ConvergeResult::FailedAction.from_exception(e)
       end

data/lib/logstash/pipeline_reporter.rb

@@ -87,7 +87,8 @@ module LogStash; class PipelineReporter
   def worker_states(batch_map)
     pipeline.worker_threads.map.with_index do |thread, idx|
       status = thread.status || "dead"
-      inflight_count = batch_map[thread] ? batch_map[thread].size : 0
+      batch = batch_map[thread]
+      inflight_count = batch ? batch.size : 0
       {
         :status => status,
         :alive => thread.alive?,

data/lib/logstash/pipeline_settings.rb

@@ -14,6 +14,7 @@ module LogStash
       "dead_letter_queue.enable",
       "dead_letter_queue.max_bytes",
       "metric.collect",
+      "pipeline.java_execution",
       "path.config",
       "path.dead_letter_queue",
       "path.queue",

data/lib/logstash/plugins/plugin_factory.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+
+module LogStash
+  module Plugins
+
+    class ExecutionContextFactory
+
+      def initialize(agent, pipeline, dlq_writer)
+        @agent = agent
+        @pipeline = pipeline
+        @dlq_writer = dlq_writer
+      end
+
+      def create(id, klass_cfg_name)
+        ExecutionContext.new(@pipeline, @agent, id, klass_cfg_name, @dlq_writer)
+      end
+    end
+
+    class PluginMetricFactory
+
+      def initialize(pipeline_id, metric)
+        @pipeline_id = pipeline_id.to_s.to_sym
+        @metric = metric
+      end
+
+      def create(plugin_type)
+        @metric.namespace([:stats, :pipelines, @pipeline_id, :plugins])
+          .namespace("#{plugin_type}s".to_sym)
+      end
+    end
+
+    class PluginFactory
+      include org.logstash.config.ir.compiler.RubyIntegration::PluginFactory
+
+      def initialize(lir, metric_factory, logger, exec_factory, filter_class)
+        @lir = lir
+        @plugins_by_id = {}
+        @metric_factory = metric_factory
+        @logger = logger
+        @exec_factory = exec_factory
+        @filter_class = filter_class
+      end
+
+      def buildOutput(name, line, column, *args)
+        plugin("output", name, line, column, *args)
+      end
+
+      def buildFilter(name, line, column, *args)
+        plugin("filter", name, line, column, *args)
+      end
+
+      def buildInput(name, line, column, *args)
+        plugin("input", name, line, column, *args)
+      end
+
+      def buildCodec(name, *args)
+        plugin("codec", name, 0, 0, *args)
+      end
+
+      def plugin(plugin_type, name, line, column, *args)
+        # Collapse the array of arguments into a single merged hash
+        args = args.reduce({}, &:merge)
+
+        if plugin_type == "codec"
+          id = SecureRandom.uuid # codecs don't really use their IDs for metrics, so we can use anything here
+        else
+          # Pull the ID from LIR to keep IDs consistent between the two representations
+          id = @lir.graph.vertices.filter do |v|
+            v.source_with_metadata &&
+              v.source_with_metadata.line == line &&
+              v.source_with_metadata.column == column
+          end.findFirst.get.id
+        end
+        args["id"] = id # some code pulls the id out of the args
+
+        raise ConfigurationError, "Could not determine ID for #{plugin_type}/#{plugin_name}" unless id
+        raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
+
+        @plugins_by_id[id] = true
+        # Scope plugins of type 'input' to 'inputs'
+        type_scoped_metric = @metric_factory.create(plugin_type)
+        klass = Plugin.lookup(plugin_type, name)
+        execution_context = @exec_factory.create(id, klass.config_name)
+
+        if plugin_type == "output"
+          OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
+        elsif plugin_type == "filter"
+          @filter_class.new(@logger, klass, type_scoped_metric, execution_context, args)
+        else # input or codec plugin
+          plugin_instance = klass.new(args)
+          scoped_metric = type_scoped_metric.namespace(id.to_sym)
+          scoped_metric.gauge(:name, plugin_instance.config_name)
+          plugin_instance.metric = scoped_metric
+          plugin_instance.execution_context = execution_context
+          plugin_instance
+        end
+      end
+    end
+  end
+end