logstash-core 1.5.2.2-java → 1.5.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 725ce14df0b0a4b69d131e0237637d1d7e57e079
-  data.tar.gz: 90f891a0291d4618d256e5373ba94467a3d3fa90
+  metadata.gz: d44a36ab493256fbc2305fd3747b94dc9dd45a2b
+  data.tar.gz: dc976fcaf28589a88b2e0183a1365f3ea096effc
 SHA512:
-  metadata.gz: db9d44d6a2ba46ef2e585533d24bc4c1fab43c69e2908a4ff3085294be3bb67979a333be90ae0e9112cfde4955e30519afa71cd58f318057f0704303ce71d9a7
-  data.tar.gz: fdb61a0b127ef81f07faea4cad94583ff175a380446307d7e6059b4b44498e9f94141c931cfd858ce9588b3f86d1c073e50385b097af60dfa40be2b6e0c3298b
+  metadata.gz: 44d5d3b875f34ce9a038a31b18b60f7d2b1d9879c43dca7d14c07b16b00c558b303ac49e61024236a72fdd794e6a1c07d64fe54d7049e05a54dd4f900af392a3
+  data.tar.gz: c75f08d03c18e6aa17310b58293676be495004b34ec695c8132099ba718682e1596666a125d9b2c9c49f03c3db655974e991542420920f21f3556795619299d0

data/lib/logstash/agent.rb

@@ -39,6 +39,11 @@ class LogStash::Agent < Clamp::Command
   option ["-V", "--version"], :flag,
     I18n.t("logstash.agent.flag.version")
 
+ option ["-p", "--pluginpath"] , "PATH",
+   I18n.t("logstash.agent.flag.pluginpath"),
+   :multivalued => true,
+   :attribute_name => :plugin_paths
+
   option ["-t", "--configtest"], :flag,
     I18n.t("logstash.agent.flag.configtest"),
     :attribute_name => :config_test
@@ -204,6 +209,7 @@ class LogStash::Agent < Clamp::Command
   # Log file stuff, plugin path checking, etc.
   def configure
     configure_logging(log_file)
+    configure_plugin_paths(plugin_paths)
   end # def configure
 
   # Point logging at a specific path.
@@ -229,7 +235,6 @@ class LogStash::Agent < Clamp::Command
       else
         @logger.level = :warn
       end
-
     end
 
     if log_file
@@ -254,6 +259,15 @@ class LogStash::Agent < Clamp::Command
     # http://jira.codehaus.org/browse/JRUBY-7003
   end # def configure_logging
 
+  # add the given paths for ungemified/bare plugins lookups
+  # @param paths [String, Array<String>] plugins path string or list of path strings to add
+  def configure_plugin_paths(paths)
+    Array(paths).each do |path|
+      fail(I18n.t("logstash.agent.configuration.plugin_path_missing", :path => path)) unless File.directory?(path)
+      LogStash::Environment.add_plugin_path(path)
+    end
+  end
+
   def load_config(path)
     begin
       uri = URI.parse(path)

data/lib/logstash/environment.rb

@@ -99,6 +99,14 @@ module LogStash
       I18n.reload!
       fail "No locale? This is a bug." if I18n.available_locales.empty?
     end
+
+    # add path for bare/ungemified plugins lookups. the path must be the base path that will include
+    # the dir structure 'logstash/TYPE/NAME.rb' where TYPE is 'inputs' 'filters', 'outputs' or 'codecs'
+    # and NAME is the name of the plugin
+    # @param path [String] plugins path to add
+    def add_plugin_path(path)
+      $LOAD_PATH << path
+    end
   end
 end
 

data/lib/logstash/outputs/base.rb

@@ -33,7 +33,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
   # Note that this setting may not be useful for all outputs.
   config :workers, :validate => :number, :default => 1
 
-  attr_reader :worker_plugins
+  attr_reader :worker_plugins, :worker_queue
 
   public
   def workers_not_supported(message=nil)

data/lib/logstash/patches.rb

@@ -1,3 +1,4 @@
 require "logstash/patches/bugfix_jruby_2558"
 require "logstash/patches/cabin"
 require "logstash/patches/profile_require_calls"
+require "logstash/patches/stronger_openssl_defaults"

data/lib/logstash/patches/stronger_openssl_defaults.rb

@@ -0,0 +1,62 @@
+
+require "openssl"
+
+# :nodoc:
+class OpenSSL::SSL::SSLContext
+  # Wrap SSLContext.new to a stronger default settings.
+  class << self
+    alias_method :orig_new, :new
+    def new(*args)
+      c = orig_new(*args)
+
+      # MRI nor JRuby seem to actually invoke `SSLContext#set_params` by
+      # default, which makes the default ciphers (and other settings) not
+      # actually defaults. Oops!
+      # To force this, and force our (hopefully more secure) defaults on
+      # all things using openssl in Ruby, we will invoke set_params
+      # on all new SSLContext objects.
+      c.set_params
+      c
+    end
+  end
+
+  # This cipher selection comes from https://wiki.mozilla.org/Security/Server_Side_TLS
+  MOZILLA_INTERMEDIATE_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
+
+  # Returns the value that should be used for the default SSLContext options
+  #
+  # This is a method instead of a constant because some constants (like
+  # OpenSSL::SSL::OP_NO_COMPRESSION) may not be available in all Ruby
+  # versions/platforms.
+  def self.__default_options
+    # ruby-core is refusing to patch ruby's default openssl settings to be more
+    # secure, so let's fix that here. The next few lines setting options and
+    # ciphers come from jmhodges' proposed patch
+    ssloptions = OpenSSL::SSL::OP_ALL
+ 
+    # TODO(sissel): JRuby doesn't have this. Maybe work on a fix?
+    if defined?(OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS)
+      ssloptions &= ~OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS
+    end
+
+    # TODO(sissel): JRuby doesn't have this. Maybe work on a fix?
+    if defined?(OpenSSL::SSL::OP_NO_COMPRESSION)
+      ssloptions |= OpenSSL::SSL::OP_NO_COMPRESSION
+    end
+
+    # Disable SSLv2 and SSLv3. They are insecure and highly discouraged.
+    ssloptions |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
+    ssloptions |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
+    ssloptions
+  end
+
+  # Overwriting the DEFAULT_PARAMS const idea from here: https://www.ruby-lang.org/en/news/2014/10/27/changing-default-settings-of-ext-openssl/
+  remove_const(:DEFAULT_PARAMS) if const_defined?(:DEFAULT_PARAMS)
+  DEFAULT_PARAMS = {
+    :ssl_version => "SSLv23",
+    :verify_mode => OpenSSL::SSL::VERIFY_PEER,
+    :ciphers => MOZILLA_INTERMEDIATE_CIPHERS,
+    :options => __default_options # Not a constant because it's computed at start-time.
+  }
+
+end

data/lib/logstash/pipeline.rb

@@ -8,6 +8,7 @@ require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
+require "logstash/util/reporter"
 
 class LogStash::Pipeline
 
@@ -252,6 +253,8 @@ class LogStash::Pipeline
   #
   # This method is intended to be called from another thread
   def shutdown
+    InflightEventsReporter.logger = @logger
+    InflightEventsReporter.start(@input_to_filter, @filter_to_output, @outputs)
     @input_threads.each do |thread|
       # Interrupt all inputs
       @logger.info("Sending shutdown signal to input thread", :thread => thread)

data/lib/logstash/plugin.rb

@@ -111,10 +111,10 @@ class LogStash::Plugin
 
   public
   def inspect
-    if !@config.nil?
-      description = @config \
-        .select { |k,v| !v.nil? && (v.respond_to?(:empty?) && !v.empty?) } \
-        .collect { |k,v| "#{k}=>#{v.inspect}" }
+    if !@params.nil?
+      description = @params
+        .reject { |k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?) }
+        .collect { |k, v| "#{k}=>#{v.inspect}" }
       return "<#{self.class.name} #{description.join(", ")}>"
     else
       return "<#{self.class.name} --->"

data/lib/logstash/util/plugin_version.rb

@@ -22,8 +22,14 @@ module LogStash::Util
 
     def self.find_version!(name)
       begin
-        specification = Gem::Specification.find_by_name(name)
-        new(specification.version)
+        spec = Gem::Specification.find_by_name(name)
+        if spec.nil?
+          # Checking for nil? is a workaround for situations where find_by_name
+          # is not able to find the real spec, as for example with pre releases
+          # of plugins
+          spec = Gem::Specification.find_all_by_name(name).first
+        end
+        new(spec.version)
       rescue Gem::LoadError
         # Rescuing the LoadError and raise a Logstash specific error.
         # Likely we can't find the gem in the current GEM_PATH
@@ -39,5 +45,11 @@ module LogStash::Util
     def <=>(other)
       version <=> other.version
     end
+
+    private
+
+    def self.build_from_spec(spec)
+      new(spec.version)
+    end
   end
 end

data/lib/logstash/util/reporter.rb

@@ -0,0 +1,27 @@
+class InflightEventsReporter
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.start(input_to_filter, filter_to_output, outputs)
+    Thread.new do
+      loop do
+        sleep 5
+        report(input_to_filter, filter_to_output, outputs)
+      end
+    end
+  end
+
+  def self.report(input_to_filter, filter_to_output, outputs)
+    report = {
+      "input_to_filter" => input_to_filter.size,
+      "filter_to_output" => filter_to_output.size,
+      "outputs" => []
+    }
+    outputs.each do |output|
+      next unless output.worker_queue && output.worker_queue.size > 0
+      report["outputs"] << [output.inspect, output.worker_queue.size]
+    end
+    @logger.warn ["INFLIGHT_EVENTS_REPORT", Time.now.iso8601, report]
+  end
+end

data/lib/logstash/util/unicode_trimmer.rb

@@ -1,4 +1,4 @@
-module UnicodeTrimmer
+module LogStash::Util::UnicodeTrimmer
   # The largest possible unicode chars are 4 bytes
   # http://stackoverflow.com/questions/9533258/what-is-the-maximum-number-of-bytes-for-a-utf-8-encoded-character
   # http://tools.ietf.org/html/rfc3629

data/lib/logstash/version.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # The version of logstash.
-LOGSTASH_VERSION = "1.5.2"
+LOGSTASH_VERSION = "1.5.3"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

data/logstash-core.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
-  gem.version       = "1.5.2.2"
+  gem.version       = LOGSTASH_VERSION.gsub(/-/, '.')
 
   gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -37,7 +37,7 @@ Gem::Specification.new do |gem|
 
   if RUBY_PLATFORM == 'java'
     gem.platform = RUBY_PLATFORM
-    gem.add_runtime_dependency "jrjackson", "~> 0.2.8" #(Apache 2.0 license)
+    gem.add_runtime_dependency "jrjackson", "~> 0.2.9" #(Apache 2.0 license)
   else
     gem.add_runtime_dependency "oj" #(MIT-style license)
   end

data/spec/core/environment_spec.rb

@@ -38,7 +38,18 @@ describe LogStash::Environment do
         allow(Dir).to receive(:glob).and_return([])
         expect { subject.load_runtime_jars! }.to raise_error
       end
+    end
+  end
+
+  context "add_plugin_path" do
+    let(:path) { "/some/path" }
+
+    before(:each) { expect($LOAD_PATH).to_not include(path) }
+    after(:each) { $LOAD_PATH.delete(path) }
 
+    it "should add the path to $LOAD_PATH" do
+      expect{subject.add_plugin_path(path)}.to change{$LOAD_PATH.size}.by(1)
+      expect($LOAD_PATH).to include(path)
     end
   end
 end

data/spec/core/plugin_spec.rb

@@ -29,6 +29,22 @@ describe LogStash::Plugin do
     expect(LogStash::Plugin.lookup("filter", "lady_gaga")).to eq(LogStash::Filters::LadyGaga)
   end
 
+  describe "#inspect" do
+    class LogStash::Filters::MyTestFilter < LogStash::Filters::Base
+      config_name "param1"
+      config :num, :validate => :number, :default => 20
+      config :str, :validate => :string, :default => "test"
+    end
+    subject { LogStash::Filters::MyTestFilter.new("num" => 1, "str" => "hello") }
+
+    it "should print the class of the filter" do
+      expect(subject.inspect).to match(/^<LogStash::Filters::MyTestFilter/)
+    end
+    it "should list config options and values" do
+      expect(subject.inspect).to match(/num=>1, str=>"hello"/)
+    end
+  end
+
   context "when validating the plugin version" do
     let(:plugin_name) { 'logstash-filter-stromae' }
     subject do

data/spec/coverage_helper.rb

@@ -2,11 +2,20 @@
 # running coverage analysis
 module CoverageHelper
 
-  SKIP_LIST = ["lib/bootstrap/rspec.rb", "lib/logstash/util/prctl.rb"]
+  ##
+  # Skip list used to avoid loading certain patterns within
+  # the logstash directories, this patterns are excluded becuause
+  # of potential problems or because they are going to be loaded
+  # in another way.
+  ##
+  SKIP_LIST = Regexp.union([
+    /^lib\/bootstrap\/rspec.rb$/,
+    /^lib\/logstash\/util\/prctl.rb$/
+  ])
 
   def self.eager_load
     Dir.glob("lib/**/*.rb") do |file|
-      next if SKIP_LIST.include?(file)
+      next if file =~ SKIP_LIST
       require file
     end
   end

data/spec/logstash/agent_spec.rb

@@ -22,7 +22,7 @@ describe LogStash::Agent do
         end
       end
     end
-    
+
     context "when remote" do
       context 'supported scheme' do
         let(:path) { "http://test.local/superconfig.conf" }
@@ -34,4 +34,28 @@ describe LogStash::Agent do
       end
     end
   end
+
+  context "--pluginpath" do
+    let(:single_path) { "/some/path" }
+    let(:multiple_paths) { ["/some/path1", "/some/path2"] }
+
+    it "should add single valid dir path to the environment" do
+      expect(File).to receive(:directory?).and_return(true)
+      expect(LogStash::Environment).to receive(:add_plugin_path).with(single_path)
+      subject.configure_plugin_paths(single_path)
+    end
+
+    it "should fail with single invalid dir path" do
+      expect(File).to receive(:directory?).and_return(false)
+      expect(LogStash::Environment).not_to receive(:add_plugin_path)
+      expect{subject.configure_plugin_paths(single_path)}.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should add multiple valid dir path to the environment" do
+      expect(File).to receive(:directory?).exactly(multiple_paths.size).times.and_return(true)
+      multiple_paths.each{|path| expect(LogStash::Environment).to receive(:add_plugin_path).with(path)}
+      subject.configure_plugin_paths(multiple_paths)
+    end
+  end
 end
+

data/spec/logstash/patches_spec.rb

@@ -0,0 +1,25 @@
+require "logstash/patches"
+
+describe "OpenSSL defaults" do
+  subject { OpenSSL::SSL::SSLContext.new }
+
+  # OpenSSL::SSL::SSLContext#ciphers returns an array of 
+  # [ [ ciphername, version, bits, alg_bits ], [ ... ], ... ]
+ 
+  # List of cipher names
+  let(:ciphers) { subject.ciphers.map(&:first) }
+
+  # List of cipher encryption bit strength. 
+  let(:encryption_bits) { subject.ciphers.map { |_, _, _, a| a } }
+
+  it "should not include any export ciphers" do
+    # SSLContext#ciphers returns an array of [ciphername, tlsversion, key_bits, alg_bits]
+    # Let's just check the cipher names
+    expect(ciphers).not_to be_any { |name| name =~ /EXPORT/ || name =~ /^EXP/ }
+  end
+
+  it "should not include any weak ciphers (w/ less than 128 bits in encryption algorithm)" do
+    # SSLContext#ciphers returns an array of [ciphername, tlsversion, key_bits, alg_bits]
+    expect(encryption_bits).not_to be_any { |bits| bits < 128 }
+  end
+end

data/spec/util/plugin_version_spec.rb

@@ -1,18 +1,33 @@
 require "spec_helper"
 require "logstash/util/plugin_version"
 
-describe LogStash::Util::PluginVersion do
+describe "LogStash::Util::PluginVersion" do
+
   subject { LogStash::Util::PluginVersion }
 
   context "#find_version!" do
+
+    let(:gem)     { "bundler" }
+
     it 'raises an PluginNoVersionError if we cant find the plugin in the gem path' do
       dummy_name ='this-character-doesnt-exist-in-the-marvel-universe'
       expect { subject.find_version!(dummy_name) }.to raise_error(LogStash::PluginNoVersionError)
     end
 
     it 'returns the version of the gem' do
-      expect { subject.find_version!('bundler') }.not_to raise_error
+      expect { subject.find_version!(gem) }.not_to raise_error
     end
+
+    context "with a pre release gem" do
+
+      it 'return the version of the gem' do
+        # Gem::Specification.find_by_name return nil if the gem is not activated, as for
+        # example the pre release ones.
+        expect(Gem::Specification).to receive(:find_by_name).and_return(nil)
+        expect { subject.find_version!(gem) }.not_to raise_error
+      end
+    end
+
   end
 
   context "#new" do

data/spec/util/unicode_trimmer_spec.rb

@@ -9,19 +9,21 @@ RSpec.configure do |config|
 end
 
 describe "truncating unicode strings correctly" do
+  subject { LogStash::Util::UnicodeTrimmer }
+
   context "with extra bytes before the snip" do
     let(:ustr) { "Testing «ταБЬℓσ»: 1<2 & 4+1>3, now 20% off!" }
 
     it "should truncate to exact byte boundaries when possible" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, 21).bytesize).to eql(21)
+      expect(subject.trim_bytes(ustr, 21).bytesize).to eql(21)
     end
 
     it "should truncate below the bytesize when splitting a byte" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, 20).bytesize).to eql(18)
+      expect(subject.trim_bytes(ustr, 20).bytesize).to eql(18)
     end
 
     it "should not truncate the string when the bytesize is already OK" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
+      expect(subject.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
     end
   end
 
@@ -29,15 +31,15 @@ describe "truncating unicode strings correctly" do
     let(:ustr) { ": 1<2 & 4+1>3, now 20% off! testing «ταБЬℓσ»" }
 
     it "should truncate to exact byte boundaries when possible" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, 21).bytesize).to eql(21)
+      expect(subject.trim_bytes(ustr, 21).bytesize).to eql(21)
     end
 
     it "should truncate below the bytesize when splitting a byte" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, 52).bytesize).to eql(51)
+      expect(subject.trim_bytes(ustr, 52).bytesize).to eql(51)
     end
 
     it "should not truncate the string when the bytesize is already OK" do
-      expect(UnicodeTrimmer.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
+      expect(subject.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
     end
   end
 
@@ -47,7 +49,7 @@ describe "truncating unicode strings correctly" do
     let(:expected_range) { (size - 4)..size }
 
     stress_it "should be near the boundary of requested size" do
-      expect(expected_range).to include(UnicodeTrimmer.trim_bytes(text, size).bytesize)
+      expect(expected_range).to include(subject.trim_bytes(text, size).bytesize)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 1.5.2.2
+  version: 1.5.3
 platform: java
 authors:
 - Jordan Sissel
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cabin
@@ -158,12 +158,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.8
+        version: 0.2.9
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.8
+        version: 0.2.9
   prerelease: false
   type: :runtime
 description: The core components of logstash, the scalable log and event management tool
@@ -201,6 +201,7 @@ files:
 - lib/logstash/patches/cabin.rb
 - lib/logstash/patches/profile_require_calls.rb
 - lib/logstash/patches/rubygems.rb
+- lib/logstash/patches/stronger_openssl_defaults.rb
 - lib/logstash/pipeline.rb
 - lib/logstash/plugin.rb
 - lib/logstash/program.rb
@@ -219,6 +220,7 @@ files:
 - lib/logstash/util/password.rb
 - lib/logstash/util/plugin_version.rb
 - lib/logstash/util/prctl.rb
+- lib/logstash/util/reporter.rb
 - lib/logstash/util/require-helper.rb
 - lib/logstash/util/retryable.rb
 - lib/logstash/util/socket_peer.rb
@@ -242,6 +244,7 @@ files:
 - spec/lib/logstash/java_integration_spec.rb
 - spec/license_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/patches_spec.rb
 - spec/outputs/base_spec.rb
 - spec/spec_helper.rb
 - spec/util/accessors_spec.rb
@@ -293,6 +296,7 @@ test_files:
 - spec/lib/logstash/java_integration_spec.rb
 - spec/license_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/patches_spec.rb
 - spec/outputs/base_spec.rb
 - spec/spec_helper.rb
 - spec/util/accessors_spec.rb