logstash-core 2.1.3-java → 2.2.0-java

data/lib/logstash/pipeline_reporter.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require 'ostruct'
+
+module LogStash; class PipelineReporter
+  attr_reader :logger, :pipeline
+
+  # This is an immutable copy of the pipeline state,
+  # It is a proxy to a hash to allow us to add methods dynamically to the hash
+  class Snapshot
+    def initialize(data)
+      @data = data
+    end
+
+    def to_hash
+      @data
+    end
+
+    def to_simple_hash
+      {"inflight_count" => inflight_count, "stalling_thread_info" => format_threads_by_plugin}
+    end
+
+    def to_str
+      to_simple_hash.to_s
+    end
+    alias_method :to_s, :to_str
+
+    def method_missing(meth)
+      @data[meth]
+    end
+
+    def format_threads_by_plugin
+      stalled_plugins = {}
+      stalling_threads_info.each do |thr|
+        key = (thr.delete("plugin") || "other")
+        stalled_plugins[key] ||= []
+        stalled_plugins[key] << thr
+      end
+      stalled_plugins
+    end
+  end
+
+  def initialize(logger,pipeline)
+    @logger = logger
+    @pipeline = pipeline
+  end
+
+  # The main way of accessing data from the reporter,,
+  # this provides a (more or less) consistent snapshot of what's going on in the
+  # pipeline with some extra decoration
+  def snapshot
+    Snapshot.new(self.to_hash)
+  end
+
+  def to_hash
+    pipeline.inflight_batches_synchronize do |batch_map|
+      worker_states_snap = worker_states(batch_map) # We only want to run this once
+      inflight_count = worker_states_snap.map {|s| s[:inflight_count] }.reduce(0, :+)
+
+      {
+        :events_filtered => events_filtered,
+        :events_consumed => events_consumed,
+        :worker_count => pipeline.worker_threads.size,
+        :inflight_count => inflight_count,
+        :worker_states => worker_states_snap,
+        :output_info => output_info,
+        :thread_info => pipeline.plugin_threads_info,
+        :stalling_threads_info => pipeline.stalling_threads_info
+      }
+    end
+  end
+
+  private
+
+  def events_filtered
+    pipeline.events_filtered.value
+  end
+
+  def events_consumed
+    pipeline.events_consumed.value
+  end
+
+  def plugin_threads
+    pipeline.plugin_threads
+  end
+
+  # Not threadsafe! must be called within an `inflight_batches_synchronize` block
+  def worker_states(batch_map)
+      pipeline.worker_threads.map.with_index do |thread,idx|
+        status = thread.status || "dead"
+        inflight_count = batch_map[thread] ? batch_map[thread].size : 0
+        {
+          :status => status,
+          :alive => thread.alive?,
+          :index => idx,
+          :inflight_count => inflight_count
+        }
+    end
+  end
+
+  def output_info
+    pipeline.outputs.map do |output_delegator|
+      is_multi_worker = output_delegator.worker_count > 1
+
+      {
+        :type => output_delegator.config_name,
+        :config => output_delegator.config,
+        :is_multi_worker => is_multi_worker,
+        :events_received => output_delegator.events_received,
+        :workers => output_delegator.workers,
+        :busy_workers => output_delegator.busy_workers
+      }
+    end
+  end
+end end

data/lib/logstash/plugin.rb

@@ -24,7 +24,7 @@ class LogStash::Plugin
 
   public
   def initialize(params=nil)
-    @params = params
+    @params = LogStash::Util.deep_clone(params)
     @logger = Cabin::Channel.get(LogStash)
   end
 

data/lib/logstash/{shutdown_controller.rb → shutdown_watcher.rb}

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 module LogStash
-  class ShutdownController
+  class ShutdownWatcher
 
     CHECK_EVERY = 1 # second
     REPORT_EVERY = 5 # checks
@@ -34,8 +34,8 @@ module LogStash
     end
 
     def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
-      controller = self.new(pipeline, cycle_period, report_every, abort_threshold)
-      Thread.new(controller) { |controller| controller.start }
+      watcher = self.new(pipeline, cycle_period, report_every, abort_threshold)
+      Thread.new(watcher) { |watcher| watcher.start }
     end
 
     def logger
@@ -47,10 +47,10 @@ module LogStash
       cycle_number = 0
       stalled_count = 0
       Stud.interval(@cycle_period) do
-        @reports << Report.from_pipeline(@pipeline)
+        @reports << pipeline_report_snapshot
         @reports.delete_at(0) if @reports.size > @report_every # expire old report
         if cycle_number == (@report_every - 1) # it's report time!
-          logger.warn(@reports.last.to_hash)
+          logger.warn(@reports.last)
 
           if shutdown_stalled?
             logger.error("The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.") if stalled_count == 0
@@ -69,6 +69,10 @@ module LogStash
       end
     end
 
+    def pipeline_report_snapshot
+      @pipeline.reporter.snapshot
+    end
+
     # A pipeline shutdown is stalled if
     # * at least REPORT_EVERY reports have been created
     # * the inflight event count is in monotonically increasing
@@ -78,7 +82,7 @@ module LogStash
       return false unless @reports.size == @report_every #
       # is stalled if inflight count is either constant or increasing
       stalled_event_count = @reports.each_cons(2).all? do |prev_report, next_report|
-        prev_report.inflight_count["total"] <= next_report.inflight_count["total"]
+        prev_report.inflight_count <= next_report.inflight_count
       end
       if stalled_event_count
         @reports.each_cons(2).all? do |prev_report, next_report|
@@ -93,35 +97,4 @@ module LogStash
       exit(-1)
     end
   end
-
-  class Report
-
-    attr_reader :inflight_count, :stalling_threads
-
-    def self.from_pipeline(pipeline)
-      new(pipeline.inflight_count, pipeline.stalling_threads)
-    end
-
-    def initialize(inflight_count, stalling_threads)
-      @inflight_count = inflight_count
-      @stalling_threads = format_threads_by_plugin(stalling_threads)
-    end
-
-    def to_hash
-      {
-        "INFLIGHT_EVENT_COUNT" => @inflight_count,
-        "STALLING_THREADS" => @stalling_threads
-      }
-    end
-
-    def format_threads_by_plugin(stalling_threads)
-      stalled_plugins = {}
-      stalling_threads.each do |thr|
-        key = (thr.delete("plugin") || "other")
-        stalled_plugins[key] ||= []
-        stalled_plugins[key] << thr
-      end
-      stalled_plugins
-    end
-  end
 end

data/lib/logstash/util.rb

@@ -183,4 +183,21 @@ module LogStash::Util
       o
     end
   end
+
+  def self.deep_clone(o)
+    case o
+    when Hash
+      o.inject({}) {|h, (k,v)| h[k] = deep_clone(v); h }
+    when Array
+      o.map {|v| deep_clone(v) }
+    when Fixnum, Symbol, IO, TrueClass, FalseClass, NilClass
+      o
+    when LogStash::Codecs::Base
+      o.clone
+    when String
+      o.clone #need to keep internal state e.g. frozen
+    else
+      Marshal.load(Marshal.dump(o))
+    end
+  end
 end # module LogStash::Util

data/lib/logstash/util/decorators.rb

@@ -19,13 +19,16 @@ module LogStash::Util
         value.each do |v|
           v = event.sprintf(v)
           if event.include?(field)
-            event[field] = Array(event[field])
-            event[field] << v
+            # note below that the array field needs to be updated then reassigned to the event.
+            # this is important because a construct like event[field] << v will not work
+            # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+            a = Array(event[field])
+            a << v
+            event[field] = a
           else
             event[field] = v
           end
-          @logger.debug? and @logger.debug("#{pluginname}: adding value to field",
-                                         :field => field, :value => value)
+          @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value)
         end
       end
     end
@@ -34,9 +37,13 @@ module LogStash::Util
     def add_tags(tags, event, pluginname)
       tags.each do |tag|
         tag = event.sprintf(tag)
-        @logger.debug? and @logger.debug("#{pluginname}: adding tag",
-                                       :tag => tag)
-        (event["tags"] ||= []) << tag
+        @logger.debug? and @logger.debug("#{pluginname}: adding tag", :tag => tag)
+        # note below that the tags array field needs to be updated then reassigned to the event.
+        # this is important because a construct like event["tags"] << tag will not work
+        # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+        tags = event["tags"] || []
+        tags << tag
+        event["tags"] = tags
       end
     end
 

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -6,8 +6,8 @@ require "logstash/util"
 module LogStash module Util class WorkerThreadsDefaultPrinter
 
   def initialize(settings)
-    @setting = settings.fetch('filter-workers', 0)
-    @default = settings.fetch('default-filter-workers', 0)
+    @setting = settings.fetch(:pipeline_workers, 0)
+    @default = settings.fetch(:default_pipeline_workers, 0)
   end
 
   def visit(collector)
@@ -17,12 +17,12 @@ module LogStash module Util class WorkerThreadsDefaultPrinter
 
   def visit_setting(collector)
     return if @setting == 0
-    collector.push("User set filter workers: #{@setting}")
+    collector.push("User set pipeline workers: #{@setting}")
   end
 
   def visit_default(collector)
     return if @default == 0
-    collector.push "Default filter workers: #{@default}"
+    collector.push "Default pipeline workers: #{@default}"
   end
 
 end end end

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+module LogStash; module Util
+  class WrappedSynchronousQueue
+    java_import java.util.concurrent.SynchronousQueue
+    java_import java.util.concurrent.TimeUnit
+
+    def initialize()
+      @queue = java.util.concurrent.SynchronousQueue.new()
+    end
+
+    # Push an object to the queue if the queue is full
+    # it will block until the object can be added to the queue.
+    #
+    # @param [Object] Object to add to the queue
+    def push(obj)
+      @queue.put(obj)
+    end
+    alias_method(:<<, :push)
+
+    # Offer an object to the queue, wait for the specified amout of time.
+    # If adding to the queue was successfull it wil return true, false otherwise.
+    #
+    # @param [Object] Object to add to the queue
+    # @param [Integer] Time in milliseconds to wait before giving up
+    # @return [Boolean] True if adding was successfull if not it return false
+    def offer(obj, timeout_ms)
+      @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
+    end
+
+    # Blocking
+    def take
+      @queue.take()
+    end
+
+    # Block for X millis
+    def poll(millis)
+      @queue.poll(millis, TimeUnit::MILLISECONDS)
+    end
+  end
+end end

data/lib/logstash/version.rb

@@ -1,6 +1,14 @@
 # encoding: utf-8
-# The version of logstash.
-LOGSTASH_VERSION = "2.1.3"
 
+# The version of the logstash package (not the logstash-core gem version).
+#
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
+
+# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb
+#       there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why
+#       the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but
+#       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
+#       fixed.
+
+LOGSTASH_VERSION = "2.2.0"

data/locales/en.yml

@@ -37,7 +37,7 @@ en:
       output-worker-unsupported-with-message: >-
         %{plugin} output plugin: setting 'workers => %{worker_count}' is not
         supported by this plugin. I will continue working as if you had not set
-        this setting.
+        this setting. Reason: %{message}
     plugin:
       deprecated_milestone: >-
         %{plugin} plugin is using the 'milestone' method to declare the version
@@ -155,8 +155,13 @@ en:
           the empty string for the '-e' flag.
         configtest: |+
           Check configuration for valid syntax and then exit.
-        filterworkers: |+
-          Sets the number of filter workers to run.
+        pipeline-workers: |+
+          Sets the number of pipeline workers to run.
+        pipeline-batch-size: |+
+          Size of batches the pipeline is to work in.
+        pipeline-batch-delay: |+
+          When creating pipeline batches, how long to wait while polling
+          for the next event.
         log: |+
           Write logstash internal logs to the given
           file. Without this flag, logstash will emit

data/logstash-core.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'logstash/version'
+require 'logstash-core/version'
 
 Gem::Specification.new do |gem|
   gem.authors       = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"]
@@ -11,11 +11,13 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
   gem.license       = "Apache License (2.0)"
 
-  gem.files         = Dir.glob(["logstash-core.gemspec", "lib/logstash-core.rb", "lib/logstash/**/*.rb", "spec/**/*.rb", "locales/*"])
+  gem.files         = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*"])
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
-  gem.version       = LOGSTASH_VERSION.gsub(/-/, '.')
+  gem.version       = LOGSTASH_CORE_VERSION
+
+  gem.add_runtime_dependency "logstash-core-event", "~> 2.2.0"
 
   gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/logstash/output_delegator_spec.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe LogStash::OutputDelegator do
+  let(:logger) { double("logger") }
+  let(:events) { 7.times.map { LogStash::Event.new }}
+  let(:default_worker_count) { 1 }
+
+  subject { described_class.new(logger, out_klass, default_worker_count) }
+
+  context "with a plain output plugin" do
+    let(:out_klass) { double("output klass") }
+    let(:out_inst) { double("output instance") }
+
+    before do
+      allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
+      allow(out_klass).to receive(:threadsafe?).and_return(false)
+      allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      allow(out_inst).to receive(:register)
+      allow(out_inst).to receive(:multi_receive)
+      allow(logger).to receive(:debug).with(any_args)
+    end
+
+    it "should initialize cleanly" do
+      expect { subject }.not_to raise_error
+    end
+
+    context "after having received a batch of events" do
+      before do
+        subject.multi_receive(events)
+      end
+
+      it "should pass the events through" do
+        expect(out_inst).to have_received(:multi_receive).with(events)
+      end
+
+      it "should increment the number of events received" do
+        expect(subject.events_received).to eql(events.length)
+      end
+    end
+
+    it "should register all workers on register" do
+      expect(out_inst).to receive(:register)
+      subject.register
+    end
+
+    it "should close all workers when closing" do
+      expect(out_inst).to receive(:do_close)
+      subject.do_close
+    end
+
+    describe "concurrency and worker support" do
+      describe "non-threadsafe outputs that allow workers" do
+        let(:default_worker_count) { 3 }
+
+        before do
+          allow(out_klass).to receive(:threadsafe?).and_return(false)
+          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+        end
+
+        it "should instantiate multiple workers" do
+          expect(subject.workers.length).to eql(default_worker_count)
+        end
+
+        it "should send received events to the worker" do
+          expect(out_inst).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+      end
+
+      describe "threadsafe outputs" do
+        before do
+          allow(out_klass).to receive(:threadsafe?).and_return(true)
+          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+        end
+
+        it "should return true when threadsafe? is invoked" do
+          expect(subject.threadsafe?).to eql(true)
+        end
+
+        it "should define a threadsafe_worker" do
+          expect(subject.send(:threadsafe_worker)).to eql(out_inst)
+        end
+
+        it "should utilize threadsafe_multi_receive" do
+          expect(subject.send(:threadsafe_worker)).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+
+        it "should not utilize the worker queue" do
+          expect(subject.send(:worker_queue)).not_to receive(:pop)
+          subject.multi_receive(events)
+        end
+
+        it "should send received events to the worker" do
+          expect(out_inst).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+      end
+    end
+  end
+
+  # This may seem suspiciously similar to the class in outputs/base_spec
+  # but, in fact, we need a whole new class because using this even once
+  # will immutably modify the base class
+  class LogStash::Outputs::NOOPDelLegacyNoWorkers < ::LogStash::Outputs::Base
+    LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
+
+    def register
+      workers_not_supported(LEGACY_WORKERS_NOT_SUPPORTED_REASON)
+    end
+  end
+
+  describe "legacy output workers_not_supported" do
+    let(:default_worker_count) { 2 }
+    let(:out_klass) { LogStash::Outputs::NOOPDelLegacyNoWorkers }
+
+    before do
+      allow(logger).to receive(:debug).with(any_args)
+    end
+
+    it "should only setup one worker" do
+      expect(subject.worker_count).to eql(1)
+    end
+  end
+end