logstash-core 2.3.4.snapshot1-java → 2.4.0-java

data/lib/logstash/pipeline.rb

@@ -392,7 +392,7 @@ module LogStash; class Pipeline
   def shutdown_workers
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
-      @logger.debug("Pushing shutdown", :thread => t)
+      @logger.debug("Pushing shutdown", :thread => t.inspect)
       @input_queue.push(LogStash::SHUTDOWN)
     end
 

data/lib/logstash/plugin.rb

@@ -2,6 +2,7 @@
 require "logstash/namespace"
 require "logstash/logging"
 require "logstash/config/mixin"
+require "logstash/instrument/null_metric"
 require "cabin"
 require "concurrent"
 
@@ -26,13 +27,14 @@ class LogStash::Plugin
   def initialize(params=nil)
     @params = LogStash::Util.deep_clone(params)
     @logger = Cabin::Channel.get(LogStash)
+    @metric_plugin = LogStash::Instrument::NullMetric.new
   end
 
   # close is called during shutdown, after the plugin worker
   # main task terminates
   public
   def do_close
-    @logger.debug("closing", :plugin => self)
+    @logger.debug("closing", :plugin => self.class.name)
     close
   end
 
@@ -47,6 +49,14 @@ class LogStash::Plugin
     return "#{self.class.name}: #{@params}"
   end
 
+  # This is a shim to make sure that plugin
+  # that record metric still work with 2.4
+  #
+  # https://github.com/elastic/logstash/issues/5539
+  def metric
+    @metric_plugin
+  end
+
   public
   def inspect
     if !@params.nil?

data/lib/logstash/util/safe_uri.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+# This class exists to quietly wrap a password string so that, when printed or
+# logged, you don't accidentally print the password itself.
+class LogStash::Util::SafeURI
+  PASS_PLACEHOLDER = "xxxxxx".freeze
+  HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
+  
+  extend Forwardable
+  
+  def_delegators :@uri, :coerce, :query=, :route_from, :port=, :default_port, :select, :normalize!, :absolute?, :registry=, :path, :password, :hostname, :merge, :normalize, :host, :component_ary, :userinfo=, :query, :set_opaque, :+, :merge!, :-, :password=, :parser, :port, :set_host, :set_path, :opaque=, :scheme, :fragment=, :set_query, :set_fragment, :userinfo, :hostname=, :set_port, :path=, :registry, :opaque, :route_to, :set_password, :hierarchical?, :set_user, :set_registry, :set_userinfo, :fragment, :component, :user=, :set_scheme, :absolute, :host=, :relative?, :scheme=, :user
+  
+  attr_reader :uri
+  
+  public
+  def initialize(arg)    
+    @uri = case arg
+           when String
+             arg = "//#{arg}" if HOSTNAME_PORT_REGEX.match(arg)
+             URI.parse(arg)
+           when URI
+             arg
+           else
+             raise ArgumentError, "Expected a string or URI, got a #{arg.class} creating a URL"
+           end
+  end
+
+  def to_s
+    sanitized.to_s
+  end
+
+  def inspect
+    sanitized.to_s
+  end
+
+  def sanitized
+    return uri unless uri.password # nothing to sanitize here!
+    
+    safe = uri.clone
+    safe.password = PASS_PLACEHOLDER
+    safe
+  end
+
+  def ==(other)
+    other.is_a?(::LogStash::Util::SafeURI) ? @uri == other.uri : false
+  end
+end
+

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.3.4.snapshot1"
+LOGSTASH_VERSION = "2.4.0"

data/locales/en.yml

@@ -225,3 +225,7 @@ en:
           Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
           in plaintext passwords appearing in your logs!
+        log-in-json: |+
+          Specify that Logstash should write its own logs in JSON form - one
+          event per line. If false, Logstash will log using Ruby's
+          Object#inspect (not easy to machine-parse)

data/logstash-core.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event", "2.3.4.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event", "2.4.0"
+  # gem.add_runtime_dependency "logstash-core-event-java", "2.4.0.dev"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -26,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
   gem.add_runtime_dependency "concurrent-ruby", "0.9.2"
-  gem.add_runtime_dependency "jruby-openssl", "0.9.13" # Required to support TLSv1.2
+  gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan

data/spec/logstash/agent_spec.rb

@@ -255,7 +255,7 @@ describe LogStash::Agent do
     it "should fail with single invalid dir path" do
       expect(File).to receive(:directory?).and_return(false)
       expect(LogStash::Environment).not_to receive(:add_plugin_path)
-      expect{subject.configure_plugin_paths(single_path)}.to raise_error(LogStash::ConfigurationError)
+      expect{subject.configure_plugin_paths(single_path)}.to raise_error(Clamp::UsageError)
     end
 
     it "should add multiple valid dir path to the environment" do
@@ -293,15 +293,15 @@ describe LogStash::Agent do
     it "should fail with single invalid dir path" do
       expect(File).to receive(:directory?).and_return(false)
       expect(LogStash::Environment).not_to receive(:add_plugin_path)
-      expect{subject.configure_plugin_paths(single_path)}.to raise_error(LogStash::ConfigurationError)
+      expect{subject.configure_plugin_paths(single_path)}.to raise_error(Clamp::UsageError)
     end
   end
 
   describe "--config-test" do
     let(:cli_args) { ["-t", "-e", pipeline_string] }
+    let(:pipeline_string) { "input { } filter { } output { }" }
 
     context "with a good configuration" do
-      let(:pipeline_string) { "input { } filter { } output { }" }
       it "should exit successfuly" do
         expect(subject.run(cli_args)).to eq(0)
       end
@@ -313,6 +313,11 @@ describe LogStash::Agent do
         expect(subject.run(cli_args)).to eq(1)
       end
     end
+
+    it "requests the config loader to format_config" do
+      expect(subject.config_loader).to receive(:format_config)
+      subject.run(cli_args)
+    end
   end
 
   describe "pipeline settings" do

data/spec/logstash/codecs/base_spec.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "spec_helper"
+
+DATA_DOUBLE = "data".freeze
+
+# use a dummy NOOP output to test Outputs::Base
+class LogStash::Codecs::NOOPAsync < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_async"
+
+  def encode(event)
+    @last_result = @on_event.call(event, DATA_DOUBLE)
+  end
+end
+
+class LogStash::Codecs::NOOPSync < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_sync"
+
+  def encode_sync(event)
+    DATA_DOUBLE
+  end
+end
+
+class LogStash::Codecs::NOOPMulti < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_multi"
+
+  def encode_sync(event)
+    DATA_DOUBLE
+  end
+end
+
+describe LogStash::Codecs::Base do
+  let(:params) { {} }
+  subject(:instance) { klass.new(params.dup) }
+  let(:event) { double("event") }
+  let(:encoded_data) { DATA_DOUBLE }
+  let(:encoded_tuple) { [event, encoded_data] }
+
+  describe "encoding" do
+    shared_examples "encoder types" do |codec_class|
+      let(:klass) { codec_class }
+      
+      describe "#{codec_class}" do
+        describe "multi_encode" do
+          it "should return an array of [event,data] tuples" do
+            expect(instance.multi_encode([event,event])).to eq([encoded_tuple, encoded_tuple])
+          end
+        end
+        
+        describe "#encode" do
+          before do
+            @result = nil
+            instance.on_event do |event, data|
+              @result = [event, data]
+            end
+            instance.encode(event)
+          end
+          
+          it "should yield the correct result" do
+            expect(@result).to eq(encoded_tuple)
+          end
+        end
+      end
+    end
+
+    include_examples("encoder types", LogStash::Codecs::NOOPAsync)
+    include_examples("encoder types", LogStash::Codecs::NOOPSync)
+    include_examples("encoder types", LogStash::Codecs::NOOPMulti)
+  end
+end
+
+                          

data/spec/logstash/config/mixin_spec.rb

@@ -68,6 +68,74 @@ describe LogStash::Config::Mixin do
     end
   end
 
+  context "when validating lists of items" do
+    let(:klass) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "multiuri"
+        config :uris, :validate => :uri, :list => true
+        config :strings, :validate => :string, :list => true
+        config :required_strings, :validate => :string, :list => true, :required => true
+      end
+    end
+
+    let(:uris) { ["http://example.net/1", "http://example.net/2"] }
+    let(:safe_uris) { uris.map {|str| ::LogStash::Util::SafeURI.new(str) } }
+    let(:strings) { ["I am a", "modern major general"] }
+    let(:required_strings) { ["required", "strings"] }
+
+    subject { klass.new("uris" => uris, "strings" => strings, "required_strings" => required_strings) }
+
+    it "a URI list should return an array of URIs" do
+      expect(subject.uris).to match_array(safe_uris)
+    end
+
+    it "a string list should return an array of strings" do
+      expect(subject.strings).to match_array(strings)
+    end
+
+    context "with a scalar value" do
+      let(:strings) { "foo" }
+
+      it "should return the scalar value as a single element array" do
+        expect(subject.strings).to match_array([strings])
+      end
+    end
+
+    context "with an empty list" do
+      let(:strings) { [] }
+
+      it "should return nil" do
+        expect(subject.strings).to be_nil
+      end
+    end
+
+    describe "with required => true" do
+      context "and a single element" do
+        let(:required_strings) { ["foo"] }
+
+        it "should return the single value" do
+          expect(subject.required_strings).to eql(required_strings)
+        end
+      end
+
+      context "with an empty list" do
+        let (:required_strings) { [] }
+
+        it "should raise a configuration error" do
+          expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
+        end        
+      end
+
+      context "with no value specified" do
+        let (:required_strings) { nil }
+
+        it "should raise a configuration error" do
+          expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
+        end
+      end          
+    end
+  end
+
   context "when validating :password" do
     let(:klass) do
       Class.new(LogStash::Filters::Base)  do
@@ -102,6 +170,95 @@ describe LogStash::Config::Mixin do
     end
   end
 
+  context "when validating :uri" do
+    let(:klass) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "fakeuri"
+        config :uri, :validate => :uri
+      end
+    end
+
+    shared_examples("safe URI") do |options|
+      options ||= {}
+      
+      subject { klass.new("uri" => uri_str) }
+
+      it "should be a SafeURI object" do
+        expect(subject.uri).to(be_a(LogStash::Util::SafeURI))
+      end
+
+      it "should correctly copy URI types" do
+        clone = subject.class.new(subject.params)
+        expect(clone.uri.to_s).to eql(uri_hidden)
+      end
+
+      it "should make the real URI object availale under #uri" do
+        expect(subject.uri.uri).to be_a(::URI)
+      end
+
+      it "should obfuscate original_params" do
+        expect(subject.original_params['uri']).to(be_a(LogStash::Util::SafeURI))
+      end
+
+      if !options[:exclude_password_specs]
+        describe "passwords" do
+          it "should make password values hidden with #to_s" do
+            expect(subject.uri.to_s).to eql(uri_hidden)
+          end
+
+          it "should make password values hidden with #inspect" do
+            expect(subject.uri.inspect).to eql(uri_hidden)
+          end
+        end
+      end
+
+      context "attributes" do
+        [:scheme, :user, :password, :hostname, :path].each do |attr|
+          it "should make #{attr} available" do
+            expect(subject.uri.send(attr)).to eql(self.send(attr))
+          end
+        end
+      end
+    end
+
+    context "with a host:port combination" do
+      let(:scheme) { nil }
+      let(:user) { nil }
+      let(:password) { nil }
+      let(:hostname) { "myhostname" }
+      let(:port) { 1234 }
+      let(:path) { "" }
+      let(:uri_str) { "#{hostname}:#{port}" }
+      let(:uri_hidden) { "//#{hostname}:#{port}" }
+
+      include_examples("safe URI", :exclude_password_specs => true)
+    end
+
+    context "with a username / password" do
+      let(:scheme) { "myscheme" }
+      let(:user) { "myuser" }
+      let(:password) { "fancypants" }
+      let(:hostname) { "myhostname" }
+      let(:path) { "/my/path" }
+      let(:uri_str) { "#{scheme}://#{user}:#{password}@#{hostname}#{path}" }
+      let(:uri_hidden) { "#{scheme}://#{user}:#{LogStash::Util::SafeURI::PASS_PLACEHOLDER}@#{hostname}#{path}" }
+
+      include_examples("safe URI")
+    end
+
+    context "without a username / password" do
+      let(:scheme) { "myscheme" }
+      let(:user) { nil }
+      let(:password) { nil }
+      let(:hostname) { "myhostname" }
+      let(:path) { "/my/path" }
+      let(:uri_str) { "#{scheme}://#{hostname}#{path}" }
+      let(:uri_hidden) { "#{scheme}://#{hostname}#{path}" }
+
+      include_examples("safe URI")
+    end
+  end
+
   describe "obsolete settings" do
     let(:plugin_class) do
       Class.new(LogStash::Inputs::Base) do

data/spec/logstash/instrument/null_metric_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require "logstash/instrument/null_metric"
+
+describe LogStash::Instrument::NullMetric do
+  let(:key) { "galaxy" }
+
+  describe "#increment" do
+    it "allows to increment a key with no amount" do
+      expect { subject.increment(key, 100) }.not_to raise_error
+    end
+
+    it "allow to increment a key" do
+      expect { subject.increment(key) }.not_to raise_error
+    end
+  end
+
+  describe "#decrement" do
+    it "allows to decrement a key with no amount" do
+      expect { subject.decrement(key, 100) }.not_to raise_error
+    end
+
+    it "allow to decrement a key" do
+      expect { subject.decrement(key) }.not_to raise_error
+    end
+  end
+
+  describe "#gauge" do
+    it "allows to set a value" do
+      expect { subject.gauge(key, "pluto") }.not_to raise_error
+    end
+  end
+
+  describe "#report_time" do
+    it "allow to record time" do
+      expect { subject.report_time(key, 1000) }.not_to raise_error
+    end
+  end
+
+  describe "#time" do
+    it "allow to record time with a block given" do
+      expect do
+        subject.time(key) { 1+1 }
+      end.not_to raise_error
+    end
+
+    it "when using a block it return the generated value" do
+      expect(subject.time(key) { 1+1 }).to eq(2)
+    end
+
+    it "allow to record time with no block given" do
+      expect do
+        clock = subject.time(key)
+        clock.stop
+      end.not_to raise_error
+    end
+  end
+
+  describe "#namespace" do
+    it "return a NullMetric" do
+      expect(subject.namespace(key)).to be_kind_of LogStash::Instrument::NullMetric
+    end
+  end
+end