logstash-core 5.0.0.alpha6.snapshot1-java → 5.0.0.alpha6.snapshot2-java

data/lib/logstash/shutdown_watcher.rb

@@ -2,6 +2,7 @@
 
 module LogStash
   class ShutdownWatcher
+    include LogStash::Util::Loggable
 
     CHECK_EVERY = 1 # second
     REPORT_EVERY = 5 # checks
@@ -25,14 +26,6 @@ module LogStash
       @unsafe_shutdown
     end
 
-    def self.logger=(logger)
-      @logger = logger
-    end
-
-    def self.logger
-      @logger ||= Cabin::Channel.get(LogStash)
-    end
-
     def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
       controller = self.new(pipeline, cycle_period, report_every, abort_threshold)
       Thread.new(controller) { |controller| controller.start }

data/lib/logstash/util/decorators.rb

@@ -6,10 +6,9 @@ module LogStash::Util
 
   # Decorators provides common manipulation on the event data.
   module Decorators
+    include LogStash::Util::Loggable
     extend self
 
-    @logger = Cabin::Channel.get(LogStash)
-
     # fields is a hash of field => value
     # where both `field` and `value` can use sprintf syntax.
     def add_fields(fields,event, pluginname)
@@ -28,7 +27,7 @@ module LogStash::Util
           else
             event.set(field, v)
           end
-          @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value)
+          self.logger.debug? and self.logger.debug("#{pluginname}: adding value to field", "field" => field, "value" => value)
         end
       end
     end
@@ -37,7 +36,7 @@ module LogStash::Util
     def add_tags(tags, event, pluginname)
       tags.each do |tag|
         tag = event.sprintf(tag)
-        @logger.debug? and @logger.debug("#{pluginname}: adding tag", :tag => tag)
+        self.logger.debug? and self.logger.debug("#{pluginname}: adding tag", "tag" => tag)
         # note below that the tags array field needs to be updated then reassigned to the event.
         # this is important because a construct like event["tags"] << tag will not work
         # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140

data/lib/logstash/util/java_version.rb

@@ -1,11 +1,6 @@
 # encoding: utf-8
-require 'cabin'
 
 module LogStash::Util::JavaVersion
-  def self.logger
-    @logger ||= Cabin::Channel.get(LogStash)
-  end
-  
   # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
   def self.version
     return nil unless LogStash::Environment.jruby?

data/lib/logstash/util/loggable.rb

@@ -1,29 +1,19 @@
 # encoding: utf-8
+require "logstash/logging/logger"
 require "logstash/namespace"
-require "cabin"
 
 module LogStash module Util
   module Loggable
-    class << self
-      def logger=(new_logger)
-        @logger = new_logger
+    def self.included(klass)
+      def klass.logger
+        ruby_name = self.name || self.class.name || self.class.to_s
+        log4j_name = ruby_name.gsub('::', '.').downcase
+        @logger ||= LogStash::Logging::Logger.new(log4j_name)
       end
 
       def logger
-        @logger ||= Cabin::Channel.get(LogStash)
+        self.class.logger
       end
     end
-
-    def self.included(base)
-      class << base
-        def logger
-          Loggable.logger
-        end
-      end
-    end
-
-    def logger
-      Loggable.logger
-    end
   end
 end; end

data/lib/logstash/util/safe_uri.rb

@@ -6,6 +6,7 @@ require "logstash/util"
 # logged, you don't accidentally print the password itself.
 class LogStash::Util::SafeURI
   PASS_PLACEHOLDER = "xxxxxx".freeze
+  HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
   
   extend Forwardable
   
@@ -17,6 +18,7 @@ class LogStash::Util::SafeURI
   def initialize(arg)    
     @uri = case arg
            when String
+             arg = "//#{arg}" if HOSTNAME_PORT_REGEX.match(arg)
              URI.parse(arg)
            when URI
              arg

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -218,16 +218,16 @@ module LogStash; module Util
           if event.nil?
             # queue poll timed out
             next
-          elsif event == LogStash::SHUTDOWN
+          elsif event.is_a?(LogStash::SignalEvent)
             # We MUST break here. If a batch consumes two SHUTDOWN events
             # then another worker may have its SHUTDOWN 'stolen', thus blocking
             # the pipeline.
-            @shutdown_signal_received = true
-            break
-          elsif event == LogStash::FLUSH
+            @shutdown_signal_received = event.shutdown?
+
             # See comment above
             # We should stop doing work after flush as well.
-            @flush_signal_received = true
+            @flush_signal_received = event.flush?
+
             break
           else
             @originals[event] = true

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0.alpha6.snapshot1"
+LOGSTASH_VERSION = "5.0.0.alpha6.snapshot2"

data/locales/en.yml

@@ -86,10 +86,10 @@ en:
     runner:
       short-help: |-
         usage:
-          bin/logstash -f CONFIG_PATH [-t] [-r] [--quiet|verbose|debug] [-w COUNT] [-l LOG]
-          bin/logstash -e CONFIG_STR [-t] [--quiet|verbose|debug] [-w COUNT] [-l LOG]
-          bin/logstash -i SHELL [--quiet|verbose|debug]
-          bin/logstash -V [--verbose|debug]
+          bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
+          bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
+          bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
+          bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
           bin/logstash --help
       invalid-configuration: >-
         The given configuration is invalid. Reason: %{error}
@@ -228,10 +228,12 @@ en:
           and NAME is the name of the plugin.
         log_level: |+
           Set the log level for logstash. Possible values are:
-            - quiet => :error
-            - verbose => :info
-            - debug => :debug
-            - warn => :warn
+            - fatal
+            - error
+            - warn
+            - info
+            - debug
+            - trace
         unsafe_shutdown: |+
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.
@@ -252,3 +254,12 @@ en:
         log_format: |+
           Specify if Logstash should write its own logs in JSON form (one
           event per line) or in plain text (using Ruby's Object#inspect)
+        debug: |+
+          Set the log level to debug.
+          DEPRECATED: use --log.level=debug instead.
+        verbose: |+
+          Set the log level to info.
+          DEPRECATED: use --log.level=verbose instead.
+        quiet: |+
+          Set the log level to info.
+          DEPRECATED: use --log.level=quiet instead.

data/logstash-core.gemspec

@@ -14,12 +14,11 @@ Gem::Specification.new do |gem|
   gem.files         = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru"])
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
-  gem.require_paths = ["lib"]
+  gem.require_paths = ["lib", "vendor/jars"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha6.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha6.snapshot2"
 
-  gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
@@ -59,4 +58,6 @@ Gem::Specification.new do |gem|
     # https://github.com/rubinius/rubinius/issues/2632#issuecomment-26954565
     gem.add_runtime_dependency "racc"
   end
+
+  gem.add_runtime_dependency 'jar-dependencies', '~> 0.3.4'
 end

data/spec/api/lib/rack_app_spec.rb

@@ -6,7 +6,7 @@ describe LogStash::Api::RackApp do
 
   class DummyApp
     class RaisedError < StandardError; end
-    
+
     def call(env)
       case env["PATH_INFO"]
       when "/good-page"
@@ -21,7 +21,7 @@ describe LogStash::Api::RackApp do
     end
   end
 
-  let(:logger) { Cabin::Channel.get }
+  let(:logger) { double("logger") }
 
   describe LogStash::Api::RackApp::ApiErrorHandler do
     let(:app) do
@@ -49,7 +49,7 @@ describe LogStash::Api::RackApp do
         allow(logger).to receive(:error).with(any_args)
         get "/raise-error"
       end
-      
+
       it "should return a 500 error" do
         expect(last_response.status).to eql(500)
       end
@@ -74,13 +74,15 @@ describe LogStash::Api::RackApp do
         run DummyApp.new
       end
     end
-    
+
     it "should log good requests as info" do
+      expect(logger).to receive(:info?).and_return(true)
       expect(logger).to receive(:info).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
       get "/good-page"
     end
 
     it "should log 5xx requests as warnings" do
+      expect(logger).to receive(:warn?).and_return(true)
       expect(logger).to receive(:warn).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
       get "/service-unavailable"
     end

data/spec/logstash/config/loader_spec.rb

@@ -3,7 +3,9 @@ require "spec_helper"
 require "logstash/config/loader"
 
 describe LogStash::Config::Loader do
-  subject { described_class.new(Cabin::Channel.get) }
+  let(:logger) { double("logger") }
+  subject { described_class.new(logger) }
+
   context "when local" do
     before { expect(subject).to receive(:local_config).with(path) }
 

data/spec/logstash/config/mixin_spec.rb

@@ -178,21 +178,15 @@ describe LogStash::Config::Mixin do
       end
     end
 
-    shared_examples("safe URI") do            
+    shared_examples("safe URI") do |options|
+      options ||= {}
+      
       subject { klass.new("uri" => uri_str) }
 
       it "should be a SafeURI object" do
         expect(subject.uri).to(be_a(LogStash::Util::SafeURI))
       end
 
-      it "should make password values hidden with #to_s" do
-        expect(subject.uri.to_s).to eql(uri_hidden)
-      end
-
-      it "should make password values hidden with #inspect" do
-        expect(subject.uri.inspect).to eql(uri_hidden)
-      end
-
       it "should correctly copy URI types" do
         clone = subject.class.new(subject.params)
         expect(clone.uri.to_s).to eql(uri_hidden)
@@ -206,6 +200,18 @@ describe LogStash::Config::Mixin do
         expect(subject.original_params['uri']).to(be_a(LogStash::Util::SafeURI))
       end
 
+      if !options[:exclude_password_specs]
+        describe "passwords" do
+          it "should make password values hidden with #to_s" do
+            expect(subject.uri.to_s).to eql(uri_hidden)
+          end
+
+          it "should make password values hidden with #inspect" do
+            expect(subject.uri.inspect).to eql(uri_hidden)
+          end
+        end
+      end
+
       context "attributes" do
         [:scheme, :user, :password, :hostname, :path].each do |attr|
           it "should make #{attr} available" do
@@ -215,6 +221,19 @@ describe LogStash::Config::Mixin do
       end
     end
 
+    context "with a host:port combination" do
+      let(:scheme) { nil }
+      let(:user) { nil }
+      let(:password) { nil }
+      let(:hostname) { "myhostname" }
+      let(:port) { 1234 }
+      let(:path) { "" }
+      let(:uri_str) { "#{hostname}:#{port}" }
+      let(:uri_hidden) { "//#{hostname}:#{port}" }
+
+      include_examples("safe URI", :exclude_password_specs => true)
+    end
+
     context "with a username / password" do
       let(:scheme) { "myscheme" }
       let(:user) { "myuser" }

data/spec/logstash/output_delegator_spec.rb

@@ -6,8 +6,9 @@ describe LogStash::OutputDelegator do
   let(:logger) { double("logger") }
   let(:events) { 7.times.map { LogStash::Event.new }}
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
+  let(:metric) { LogStash::Instrument::NullMetric.new }
 
-  subject { described_class.new(logger, out_klass, LogStash::Instrument::NullMetric.new, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
+  subject { described_class.new(logger, out_klass, metric, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
 
   context "with a plain output plugin" do
     let(:out_klass) { double("output klass") }
@@ -15,10 +16,13 @@ describe LogStash::OutputDelegator do
     let(:concurrency) { :single }
 
     before(:each) do
+      # use the same metric instance
+      allow(metric).to receive(:namespace).with(any_args).and_return(metric)
+
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:name).and_return("example")
       allow(out_klass).to receive(:concurrency).with(any_args).and_return concurrency
-      allow(out_klass).to receive(:config_name)
+      allow(out_klass).to receive(:config_name).and_return("dummy_plugin")
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)
@@ -32,6 +36,11 @@ describe LogStash::OutputDelegator do
       expect { subject }.not_to raise_error
     end
 
+    it "should push the name of the plugin to the metric" do
+      expect(metric).to receive(:gauge).with(:name, out_klass.config_name)
+      described_class.new(logger, out_klass, metric, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
+    end
+
     context "after having received a batch of events" do
       before do
         subject.register
@@ -88,6 +97,10 @@ describe LogStash::OutputDelegator do
             expect(out_klass).to have_received(:new).with(plugin_args)
           end
 
+          it "should set the metric on the instance" do
+            expect(out_inst).to have_received(:metric=).with(metric)
+          end
+
           [[:register], [:do_close], [:multi_receive, [[]] ] ].each do |method, args|
             context "strategy objects" do
               before do

data/spec/logstash/outputs/base_spec.rb

@@ -34,7 +34,6 @@ class LogStash::Outputs::NOOPMultiReceiveEncoded < ::LogStash::Outputs::Base
   end
 end
 
-
 describe "LogStash::Outputs::Base#new" do
   let(:params) { {} }  
   subject(:instance) { klass.new(params.dup) }

data/spec/logstash/pipeline_reporter_spec.rb

@@ -4,6 +4,7 @@ require "logstash/pipeline"
 require "logstash/pipeline_reporter"
 
 class DummyOutput < LogStash::Outputs::Base
+
   config_name "dummyoutput"
   milestone 2
 

data/spec/logstash/pipeline_spec.rb

@@ -80,7 +80,7 @@ class DummySafeFilter < LogStash::Filters::Base
 end
 
 class TestPipeline < LogStash::Pipeline
-  attr_reader :outputs, :settings, :logger
+  attr_reader :outputs, :settings
 end
 
 describe LogStash::Pipeline do
@@ -130,7 +130,7 @@ describe LogStash::Pipeline do
         let(:logger) { double("pipeline logger").as_null_object }
 
         before do
-          expect(Cabin::Channel).to receive(:get).with(LogStash).and_return(logger).at_least(:once)
+          expect(TestPipeline).to receive(:logger).and_return(logger)
           allow(logger).to receive(:debug?).and_return(true)
         end
 
@@ -641,6 +641,18 @@ describe LogStash::Pipeline do
         plugin_name = dummy_output_id.to_sym
         expect(collected_metric[:stats][:pipelines][:main][:plugins][:outputs][plugin_name][:events][:out].value).to eq(number_of_events)
       end
+
+      it "populates the name of the output plugin" do
+        plugin_name = dummy_output_id.to_sym
+        expect(collected_metric[:stats][:pipelines][:main][:plugins][:outputs][plugin_name][:name].value).to eq(DummyOutput.config_name)
+      end
+
+      it "populates the name of the filter plugin" do
+        [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
+          plugin_name = "multiline_#{id}".to_sym
+          expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:name].value).to eq(LogStash::Filters::Multiline.config_name)
+        end
+      end
     end
   end
 

data/spec/logstash/plugin_spec.rb

@@ -76,7 +76,7 @@ describe LogStash::Plugin do
         .with(plugin_name)
         .and_return(double(:version => Gem::Version.new('1.0.0')))
 
-      expect_any_instance_of(Cabin::Channel).not_to receive(:info)
+      expect_any_instance_of(LogStash::Logging::Logger).not_to receive(:info)
       subject.validate({})
     end
 
@@ -85,7 +85,7 @@ describe LogStash::Plugin do
         .with(plugin_name)
         .and_return(double(:version => Gem::Version.new('0.9.1')))
 
-      expect_any_instance_of(Cabin::Channel).to receive(:info)
+      expect_any_instance_of(LogStash::Logging::Logger).to receive(:info)
         .with(/Using version 0.9.x/)
 
       subject.validate({})
@@ -96,7 +96,7 @@ describe LogStash::Plugin do
         .with(plugin_name)
         .and_return(double(:version => Gem::Version.new('0.1.1')))
 
-      expect_any_instance_of(Cabin::Channel).to receive(:info)
+      expect_any_instance_of(LogStash::Logging::Logger).to receive(:info)
         .with(/Using version 0.1.x/)
       subject.validate({})
     end
@@ -110,7 +110,7 @@ describe LogStash::Plugin do
         .with(plugin_name)
         .and_return(double(:version => Gem::Version.new('0.1.1')))
 
-      expect_any_instance_of(Cabin::Channel).to receive(:info)
+      expect_any_instance_of(LogStash::Logging::Logger).to receive(:info)
         .once
         .with(/Using version 0.1.x/)
 
@@ -119,7 +119,7 @@ describe LogStash::Plugin do
     end
 
     it "warns the user if we can't find a defined version" do
-      expect_any_instance_of(Cabin::Channel).to receive(:warn)
+      expect_any_instance_of(LogStash::Logging::Logger).to receive(:warn)
         .once
         .with(/plugin doesn't have a version/)
 
@@ -128,7 +128,7 @@ describe LogStash::Plugin do
 
 
     it 'logs a warning if the plugin use the milestone option' do
-      expect_any_instance_of(Cabin::Channel).to receive(:debug)
+      expect_any_instance_of(LogStash::Logging::Logger).to receive(:debug)
         .with(/stromae plugin is using the 'milestone' method/)
 
       class LogStash::Filters::Stromae < LogStash::Filters::Base
@@ -278,6 +278,7 @@ describe LogStash::Plugin do
     [LogStash::Inputs::Base, LogStash::Filters::Base, LogStash::Outputs::Base].each do |base|
       let(:plugin) do
         Class.new(base) do
+          #include LogStash::Util::Loggable
           config_name "testing"
 
           def register