logstash-core 2.0.1.snapshot1-java → 2.1.0-java

data/lib/logstash/util.rb

@@ -24,6 +24,41 @@ module LogStash::Util
     end
   end # def set_thread_name
 
+  def self.set_thread_plugin(plugin)
+    Thread.current[:plugin] = plugin
+  end
+
+  def self.get_thread_id(thread)
+    if RUBY_ENGINE == "jruby"
+      JRuby.reference(thread).native_thread.id
+    else
+      raise Exception.new("Native thread IDs aren't supported outside of JRuby")
+    end
+  end
+
+  def self.thread_info(thread)
+    backtrace = thread.backtrace.map do |line|
+      line.gsub(LogStash::Environment::LOGSTASH_HOME, "[...]")
+    end
+
+    blocked_on = case backtrace.first
+                 when /in `push'/ then "blocked_on_push"
+                 when /(?:pipeline|base).*pop/ then "waiting_for_events"
+                 else nil
+                 end
+
+    {
+      "thread_id" => get_thread_id(thread),
+      "name" => thread[:name],
+      "plugin" => (thread[:plugin] ? thread[:plugin].debug_info : nil),
+      "backtrace" => backtrace,
+      "blocked_on" => blocked_on,
+      "status" => thread.status,
+      "current_call" => backtrace.first
+    }
+  end
+
+
   # Merge hash 'src' into 'dst' nondestructively
   #
   # Duplicate keys will become array values

data/lib/logstash/util/defaults_printer.rb

@@ -20,7 +20,7 @@ module LogStash module Util class DefaultsPrinter
     @printers.each do |printer|
       printer.visit(collector)
     end
-    "Default settings used: " + collector.join(', ')
+    "Settings: " + collector.join(', ')
   end
 
   private

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -6,11 +6,23 @@ require "logstash/util"
 module LogStash module Util class WorkerThreadsDefaultPrinter
 
   def initialize(settings)
-    @setting = settings.fetch('filter-workers', 1)
+    @setting = settings.fetch('filter-workers', 0)
+    @default = settings.fetch('default-filter-workers', 0)
   end
 
   def visit(collector)
-    collector.push "Filter workers: #{@setting}"
+    visit_setting(collector)
+    visit_default(collector)
+  end
+
+  def visit_setting(collector)
+    return if @setting == 0
+    collector.push("User set filter workers: #{@setting}")
+  end
+
+  def visit_default(collector)
+    return if @default == 0
+    collector.push "Default filter workers: #{@default}"
   end
 
 end end end

data/lib/logstash/version.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # The version of logstash.
-LOGSTASH_VERSION = "2.0.1.snapshot1"
+LOGSTASH_VERSION = "2.1.0"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

data/locales/en.yml

@@ -187,3 +187,8 @@ en:
         debug: |+
           Most verbose logging. This causes 'debug'
           level logs to be emitted.
+        unsafe_shutdown: |+
+          Force logstash to exit during shutdown even
+          if there are still inflight events in memory.
+          By default, logstash will refuse to quit until all
+          received events have been pushed to the outputs.

data/logstash-core.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
-  gem.add_runtime_dependency "concurrent-ruby", "~> 0.9.1"
+  gem.add_runtime_dependency "concurrent-ruby", "0.9.2"
   gem.add_runtime_dependency "jruby-openssl", ">= 0.9.11" # Required to support TLSv1.2
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
@@ -35,6 +35,7 @@ Gem::Specification.new do |gem|
 
   # filetools and rakelib
   gem.add_runtime_dependency "minitar", "~> 0.5.4"
+  gem.add_runtime_dependency "rubyzip", "~> 1.1.7"
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
   if RUBY_PLATFORM == 'java'

data/spec/core/event_spec.rb

@@ -496,4 +496,23 @@ describe LogStash::Event do
       subject{LogStash::Event.new(LogStash::Json.load(LogStash::Json.dump(event_hash)))}
     end
   end
+
+
+  describe "#to_s" do
+    let(:timestamp) { LogStash::Timestamp.new }
+    let(:event1) { LogStash::Event.new({ "@timestamp" => timestamp, "host" => "foo", "message" => "bar"}) }
+    let(:event2) { LogStash::Event.new({ "host" => "bar", "message" => "foo"}) }
+
+    it "should cache only one template" do
+      LogStash::StringInterpolation::CACHE.clear
+      expect {
+        event1.to_s
+        event2.to_s
+      }.to change { LogStash::StringInterpolation::CACHE.size }.by(1)
+    end
+
+    it "return the string containing the timestamp, the host and the message" do
+      expect(event1.to_s).to eq("#{timestamp.to_iso8601} #{event1["host"]} #{event1["message"]}")
+    end
+  end
 end

data/spec/core/pipeline_spec.rb

@@ -53,20 +53,127 @@ class DummyOutput < LogStash::Outputs::Base
   end
 end
 
+class DummyFilter < LogStash::Filters::Base
+  config_name "dummyfilter"
+  milestone 2
+
+  def register() end
+
+  def filter(event) end
+
+  def threadsafe?() false; end
+
+  def close() end
+end
+
+class DummySafeFilter < LogStash::Filters::Base
+  config_name "dummysafefilter"
+  milestone 2
+
+  def register() end
+
+  def filter(event) end
+
+  def threadsafe?() true; end
+
+  def close() end
+end
+
 class TestPipeline < LogStash::Pipeline
-  attr_reader :outputs
+  attr_reader :outputs, :filter_threads, :settings, :logger
 end
 
 describe LogStash::Pipeline do
+  let(:worker_thread_count)     { 8 }
+  let(:safe_thread_count)       { 1 }
+  let(:override_thread_count)   { 42 }
+
+  describe "defaulting the filter workers based on thread safety" do
+    before(:each) do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummysafefilter").and_return(DummySafeFilter)
+      allow(LogStash::Config::CpuCoreStrategy).to receive(:fifty_percent).and_return(worker_thread_count)
+    end
+
+    context "when there are some not threadsafe filters" do
+      let(:test_config_with_filters) {
+        <<-eos
+        input {
+          dummyinput {}
+        }
+
+        filter {
+          dummyfilter {}
+        }
+
+        output {
+          dummyoutput {}
+        }
+        eos
+      }
+
+      context "when there is no command line -w N set" do
+        it "starts one filter thread" do
+          msg = "Defaulting filter worker threads to 1 because there are some" +
+                " filters that might not work with multiple worker threads"
+          pipeline = TestPipeline.new(test_config_with_filters)
+          expect(pipeline.logger).to receive(:warn).with(msg,
+            {:count_was=>worker_thread_count, :filters=>["dummyfilter"]})
+          pipeline.run
+          expect(pipeline.filter_threads.size).to eq(safe_thread_count)
+        end
+      end
+
+      context "when there is command line -w N set" do
+        it "starts multiple filter thread" do
+          msg = "Warning: Manual override - there are filters that might" +
+                " not work with multiple worker threads"
+          pipeline = TestPipeline.new(test_config_with_filters)
+          expect(pipeline.logger).to receive(:warn).with(msg,
+            {:worker_threads=> override_thread_count, :filters=>["dummyfilter"]})
+          pipeline.configure("filter-workers", override_thread_count)
+          pipeline.run
+          expect(pipeline.filter_threads.size).to eq(override_thread_count)
+        end
+      end
+    end
+
+    context "when there are threadsafe filters only" do
+      let(:test_config_with_filters) {
+        <<-eos
+        input {
+          dummyinput {}
+        }
+
+        filter {
+          dummysafefilter {}
+        }
 
-context "close" do
+        output {
+          dummyoutput {}
+        }
+        eos
+      }
 
-  before(:each) do
-    allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
-    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
-    allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+      it "starts multiple filter threads" do
+        pipeline = TestPipeline.new(test_config_with_filters)
+        pipeline.run
+        expect(pipeline.filter_threads.size).to eq(worker_thread_count)
+      end
+    end
   end
 
+  context "close" do
+    before(:each) do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+
     let(:test_config_without_output_workers) {
       <<-eos
       input {
@@ -191,6 +298,5 @@ context "close" do
         expect(subject[2]["foo"]).to eq("bar")
       end
     end
-
   end
 end

data/spec/core/shutdown_controller_spec.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/shutdown_controller"
+
+describe LogStash::ShutdownController do
+
+  let(:check_every) { 0.01 }
+  let(:check_threshold) { 100 }
+  subject { LogStash::ShutdownController.new(pipeline, check_every) }
+  let(:pipeline) { double("pipeline") }
+  report_count = 0
+
+  before :each do
+    allow(LogStash::Report).to receive(:from_pipeline).and_wrap_original do |m, *args|
+      report_count += 1
+      m.call(*args)
+    end
+  end
+
+  after :each do
+    report_count = 0
+  end
+
+  context "when pipeline is stalled" do
+    let(:increasing_count) { (1..5000).to_a.map {|i| { "total" => i } } }
+    before :each do
+      allow(pipeline).to receive(:inflight_count).and_return(*increasing_count)
+      allow(pipeline).to receive(:stalling_threads) { { } }
+    end
+
+    describe ".unsafe_shutdown = true" do
+      let(:abort_threshold) { subject.abort_threshold }
+      let(:report_every) { subject.report_every }
+
+      before :each do
+        subject.class.unsafe_shutdown = true
+      end
+
+      it "should force the shutdown" do
+        expect(subject).to receive(:force_exit).once
+        subject.start
+      end
+
+      it "should do exactly \"abort_threshold\" stall checks" do
+        allow(subject).to receive(:force_exit)
+        expect(subject).to receive(:shutdown_stalled?).exactly(abort_threshold).times.and_call_original
+        subject.start
+      end
+
+      it "should do exactly \"abort_threshold\"*\"report_every\" stall checks" do
+        allow(subject).to receive(:force_exit)
+        expect(LogStash::Report).to receive(:from_pipeline).exactly(abort_threshold*report_every).times.and_call_original
+        subject.start
+      end
+    end
+
+    describe ".unsafe_shutdown = false" do
+
+      before :each do
+        subject.class.unsafe_shutdown = false
+      end
+
+      it "shouldn't force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+  end
+
+  context "when pipeline is not stalled" do
+    let(:decreasing_count) { (1..5000).to_a.reverse.map {|i| { "total" => i } } }
+    before :each do
+      allow(pipeline).to receive(:inflight_count).and_return(*decreasing_count)
+      allow(pipeline).to receive(:stalling_threads) { { } }
+    end
+
+    describe ".unsafe_shutdown = true" do
+
+      before :each do
+        subject.class.unsafe_shutdown = true
+      end
+
+      it "should force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+
+    describe ".unsafe_shutdown = false" do
+
+      before :each do
+        subject.class.unsafe_shutdown = false
+      end
+
+      it "shouldn't force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+  end
+end

data/spec/license_spec.rb

@@ -12,6 +12,7 @@ describe "Project licenses" do
     Regexp.union([ /mit/,
                    /apache*/,
                    /bsd/,
+                   /artistic 2.*/,
                    /ruby/,
                    /lgpl/])
   }

data/spec/plugin_manager/install_spec.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'pluginmanager/main'
+
+describe LogStash::PluginManager::Install do
+  let(:cmd) { LogStash::PluginManager::Install.new("install") }
+
+  before(:each) do
+    expect(cmd).to receive(:validate_cli_options!).and_return(nil)
+  end
+
+  context "when validating plugins" do
+    let(:sources) { ["https://rubygems.org", "http://localhost:9292"] }
+
+    before(:each) do
+      expect(cmd).to receive(:plugins_gems).and_return([["dummy", nil]])
+      expect(cmd).to receive(:install_gems_list!).and_return(nil)
+      expect(cmd).to receive(:remove_unused_locally_installed_gems!).and_return(nil)
+      cmd.verify = true
+    end
+
+    it "should load all the sources defined in the Gemfile" do
+      expect(cmd.gemfile.gemset).to receive(:sources).and_return(sources)
+      expect(LogStash::PluginManager).to receive(:logstash_plugin?).with("dummy", nil, {:rubygems_source => sources}).and_return(true)
+      cmd.execute
+    end
+  end
+end

data/spec/plugin_manager/update_spec.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'pluginmanager/main'
+
+describe LogStash::PluginManager::Update do
+  let(:cmd)     { LogStash::PluginManager::Update.new("update") }
+  let(:sources) { cmd.gemfile.gemset.sources }
+
+  before(:each) do
+    expect(cmd).to receive(:find_latest_gem_specs).and_return({})
+    allow(cmd).to receive(:warn_local_gems).and_return(nil)
+    expect(cmd).to receive(:display_updated_plugins).and_return(nil)
+    expect_any_instance_of(LogStash::Bundler).to receive(:invoke!).with(:clean => true)
+  end
+
+  it "pass all gem sources to the bundle update command" do
+    sources = cmd.gemfile.gemset.sources
+    expect_any_instance_of(LogStash::Bundler).to receive(:invoke!).with(:update => [], :rubygems_source => sources)
+    cmd.execute
+  end
+
+  context "when skipping validation" do
+    let(:cmd)    { LogStash::PluginManager::Update.new("update") }
+    let(:plugin) { OpenStruct.new(:name => "dummy", :options => {} ) }
+
+    before(:each) do
+      expect(cmd.gemfile).to receive(:find).with(plugin).and_return(plugin)
+      expect(cmd.gemfile).to receive(:save).and_return(nil)
+      expect(cmd).to receive(:plugins_to_update).and_return([plugin])
+      expect_any_instance_of(LogStash::Bundler).to receive(:invoke!).with(:update => [plugin], :rubygems_source => sources).and_return(nil)
+    end
+
+    it "skips version verification when ask for it" do
+      cmd.verify = false
+      expect(cmd).to_not receive(:validates_version)
+      cmd.execute
+    end
+  end
+end