logstash-core 5.6.5-java → 5.6.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d30cf0d0e9fd2059e3e39c1b1d66b69cd3ab027
-  data.tar.gz: 0588a9533169bcef1457548abcd613c6b16997a9
+  metadata.gz: ac3afa459e4fb90932a6222ae0cdb6e1c511c98a
+  data.tar.gz: 781766e9a85797a32c5dd2277a51de3f5b3b11d6
 SHA512:
-  metadata.gz: f94f9f48e66e099b55b00318662da76f5adda111ccb581954a7cbd3f94454422129973602c773645daa1e0996bd2edde1256ceee5ec4b5ddf2f922290864958b
-  data.tar.gz: f4d9d880baa2832e7caa7ee5e621419d221e96e913b51215cd0ae521bf3cf97e37540eb136f7f1da280e6b1eb67f36d69ee95cb04bed76f933b2030c5d2d8dcf
+  metadata.gz: 8059e6050e85dcb936a0d66130541be7c1e5942f0f5893089098a93be0062b9d73f4f3268607fbbc8548c3520d8bb363a6aefc6dca72054eebb99f4de11475d0
+  data.tar.gz: de2b67e382baacbdc8fcd354c27763cf3b01dc8793327f29e0c3c4f70d0d1d224e2eba95057123ca79fb0137a2d0a3ff0d12856e0bbee637f2e0adf45d586887

data/lib/logstash-core_jars.rb

@@ -14,13 +14,13 @@ rescue LoadError
 end
 
 if defined? Jars
-  require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' )
-  require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
-  require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' )
-  require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.1' )
-  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' )
+  require_jar 'org.apache.logging.log4j', 'log4j-core', '2.6.2'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1'
+  require_jar 'org.apache.logging.log4j', 'log4j-api', '2.6.2'
+  require_jar 'org.slf4j', 'slf4j-api', '1.7.21'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1'
+  require_jar 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2'
+  require_jar 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.1'
+  require_jar 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1'
 end

data/lib/logstash/config/mixin.rb

@@ -60,30 +60,6 @@ module LogStash::Config::Mixin
     # store the plugin type, turns LogStash::Inputs::Base into 'input'
     @plugin_type = self.class.ancestors.find { |a| a.name =~ /::Base$/ }.config_name
 
-    # warn about deprecated variable use
-    params.each do |name, value|
-      opts = self.class.get_config[name]
-      if opts && opts[:deprecated]
-        extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
-        extra.gsub!("%PLUGIN%", self.class.config_name)
-        self.logger.warn("You are using a deprecated config setting " +
-                     "#{name.inspect} set in #{self.class.config_name}. " +
-                     "Deprecated settings will continue to work, " +
-                     "but are scheduled for removal from logstash " +
-                     "in the future. #{extra} If you have any questions " +
-                     "about this, please visit the #logstash channel " +
-                     "on freenode irc.", :name => name, :plugin => self)
-
-      end
-      if opts && opts[:obsolete]
-        extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
-        extra.gsub!("%PLUGIN%", self.class.config_name)
-        raise LogStash::ConfigurationError,
-          I18n.t("logstash.runner.configuration.obsolete", :name => name,
-                 :plugin => self.class.config_name, :extra => extra)
-      end
-    end
-
     # Set defaults from 'config :foo, :default => somevalue'
     self.class.get_config.each do |name, opts|
       next if params.include?(name.to_s)
@@ -109,17 +85,46 @@ module LogStash::Config::Mixin
       params[name.to_s] = deep_replace(value)
     end
 
-
     if !self.class.validate(params)
       raise LogStash::ConfigurationError,
         I18n.t("logstash.runner.configuration.invalid_plugin_settings")
     end
 
+    # now that we know the parameters are valid, we can obfuscate the original copy
+    # of the parameters before storing them as an instance variable
+    self.class.secure_params!(original_params)
+    @original_params = original_params
+
+    # warn about deprecated variable use
+    original_params.each do |name, value|
+      opts = self.class.get_config[name]
+      if opts && opts[:deprecated]
+        extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
+        extra.gsub!("%PLUGIN%", self.class.config_name)
+        self.logger.warn("You are using a deprecated config setting " +
+                     "#{name.inspect} set in #{self.class.config_name}. " +
+                     "Deprecated settings will continue to work, " +
+                     "but are scheduled for removal from logstash " +
+                     "in the future. #{extra} If you have any questions " +
+                     "about this, please visit the #logstash channel " +
+                     "on freenode irc.", :name => name, :plugin => self)
+
+      end
+
+      if opts && opts[:obsolete]
+        extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
+        extra.gsub!("%PLUGIN%", self.class.config_name)
+        raise LogStash::ConfigurationError,
+          I18n.t("logstash.runner.configuration.obsolete", :name => name,
+                 :plugin => self.class.config_name, :extra => extra)
+      end
+    end
+
     # We remove any config options marked as obsolete,
     # no code should be associated to them and their values should not bleed
     # to the plugin context.
     #
-    # This need to be done after fetching the options from the parents classed
+    # This need to be done after fetching the options from the parents class
     params.reject! do |name, value|
       opts = self.class.get_config[name]
       opts.include?(:obsolete)
@@ -134,11 +139,6 @@ module LogStash::Config::Mixin
       instance_variable_set("@#{key}", value)
     end
 
-    # now that we know the parameters are valid, we can obfuscate the original copy
-    # of the parameters before storing them as an instance variable
-    self.class.secure_params!(original_params)
-    @original_params = original_params
-
     @config = params
   end # def config_init
 

data/lib/logstash/dependency_report.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+Thread.abort_on_exception = true
+Encoding.default_external = Encoding::UTF_8
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+require "clamp"
+require "logstash/namespace"
+require "rubygems"
+require "jars/gemspec_artifacts"
+
+class LogStash::DependencyReport < Clamp::Command
+  option [ "--csv" ], "OUTPUT_PATH", "The path to write the dependency report in csv format.",
+    :required => true, :attribute_name => :output_path
+
+  def execute
+    require "csv"
+    CSV.open(output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
+      puts "Finding gem dependencies"
+      gems.each { |d| csv << d }
+      puts "Finding java/jar dependencies"
+      jars.each { |d| csv << d }
+    end
+
+    # Copy in COPYING.csv which is a best-effort, hand-maintained file of dependency license information.
+    File.open(output_path, "a+") do |file|
+      extra = File.join(File.dirname(__FILE__), "..", "..", "..", "COPYING.csv")
+      file.write(IO.read(extra))
+    end
+    nil
+  end
+
+  def gems
+    # @mgreau requested `logstash-*` dependencies be removed from this list: 
+    # https://github.com/elastic/logstash/pull/8837#issuecomment-351859433
+    Gem::Specification.reject { |g| g.name =~ /^logstash-/ }.collect do |gem|
+      licenses = ("UNKNOWN" if gem.licenses.empty?) || (gem.licenses.map { |l| SPDX.map(l) }.join("|") if !gem.licenses.empty?)
+      [gem.name, gem.version.to_s, gem.homepage, licenses]
+    end
+  end
+
+  def jars
+    jars = []
+    # For any gems with jar dependencies,
+    #   Look at META-INF/MANIFEST.MF for any jars in each gem
+    #   Note any important details.
+    Gem::Specification.select { |g| g.requirements && g.requirements.any? { |r| r =~ /^jar / } }.collect do |gem|
+
+      # Where is the gem installed
+      root = gem.full_gem_path
+
+      Dir.glob(File.join(root, "**", "*.jar")).collect do |path|
+        jar = java.util.jar.JarFile.new(path)
+        manifest = jar.getManifest
+
+        pom_entries = jar.entries.select { |t| t.getName.start_with?("META-INF/maven/") && t.getName.end_with?("/pom.properties") }
+
+        # Some jar files have multiple maven pom.properties files. It is unclear how to know what is correct?
+        # TODO(sissel): Maybe we should use all pom.properties files? None of the pom.properties/pom.xml files have license information, though.
+        # TODO(sissel): In some cases, there are META-INF/COPYING and
+        #   META-INF/NOTICE.txt files? Can we use these somehow? There is no
+        #   common syntax for parsing these files, though...
+        pom_map = if pom_entries.count == 1
+          pom_in = jar.getInputStream(pom_entries.first)
+          pom_content = pom_in.available.times.collect { pom_in.read }.pack("C*")
+          # Split non-comment lines by `key=val` into a map { key => val }
+          Hash[pom_content.split(/\r?\n/).grep(/^[^#]/).map { |line| line.split("=", 2) }]
+        else
+          {}
+        end
+
+        next if manifest.nil?
+        # convert manifest attributes to a map w/ keys .to_s
+        # without this, the attribute keys will be `Object#inspect` values
+        # like #<Java::JavaUtilJar::Attributes::Name0xabcdef0>
+        attributes = Hash[manifest.getMainAttributes.map { |k,v| [k.to_s, v] }]
+
+        begin
+          # Prefer the maven/pom groupId when it is available.
+          artifact = pom_map.fetch("artifactId", attributes.fetch("Implementation-Title"))
+          group = pom_map.fetch("groupId", attributes.fetch("Implementation-Vendor-Id"))
+          jars << [
+            group + ":" + artifact,
+            attributes.fetch("Bundle-Version"),
+            attributes.fetch("Bundle-DocURL"),
+            SPDX.map(attributes.fetch("Bundle-License")),
+          ]
+        rescue KeyError => e
+          # The jar is missing a required manifest field, it may not have any useful manifest data.
+          # Ignore it and move on.
+        end
+      end
+    end
+    jars.uniq.sort
+  end
+
+  module SPDX
+    # This is a non-exhaustive, best effort list of licenses as they map to SPDX identifiers.
+    ALIASES = {
+      "Apache-2.0" => [
+        "Apache 2",
+        "apache-2.0",
+        "Apache 2.0",
+        "Apache License (2.0)",
+        "Apache License 2.0",
+        "https://www.apache.org/licenses/LICENSE-2.0.txt",
+        "http://www.apache.org/licenses/LICENSE-2.0.txt",
+      ],
+      "Artistic-2.0" => [
+        "Artistic 2.0"
+      ],
+      "BSD-2-Clause" => [
+        "2-clause BSDL",
+        "2-clause"
+      ],
+      "GPL-2.0" => [
+        "GPL-2"
+      ]
+    }
+
+    # Get a map of name => spdx
+    MAP_APACHE2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[0]]
+    MAP_ARTISTIC2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[1]]
+    MAP_BSD = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[2]]
+    MAP_GPL2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[3]]
+
+    module_function
+    def map(value)
+      MAP_APACHE2[value] ||  MAP_ARTISTIC2[value] || MAP_BSD[value] ||  MAP_GPL2[value] || value
+    end
+  end
+end

data/lib/logstash/dependency_report_runner.rb

@@ -0,0 +1,17 @@
+require_relative "../../../lib/bootstrap/environment"
+
+if $0 == __FILE__
+  begin
+    LogStash::Bundler.setup!({:without => [:build, :development]})
+  rescue => Bundler::GemfileNotFound
+    $stderr.puts("No Gemfile found. Maybe you need to run `rake artifact:tar`?")
+    raise
+  end
+
+  require "logstash/namespace"
+  require_relative "../../../lib/bootstrap/patches/jar_dependencies"
+  require "logstash/dependency_report"
+
+  exit_status = LogStash::DependencyReport.run 
+  exit(exit_status || 0)
+end

data/spec/logstash/config/mixin_spec.rb

@@ -3,6 +3,34 @@ require "spec_helper"
 require "logstash/config/mixin"
 
 describe LogStash::Config::Mixin do
+  context "when encountering a deprecated option" do
+    let(:password) { "sekret" }
+    let(:double_logger) { double("logger").as_null_object }
+
+    subject do 
+      Class.new(LogStash::Filters::Base) do
+        include LogStash::Config::Mixin
+        config_name "test_deprecated"
+        milestone 1
+        config :old_opt, :validate => :string, :deprecated => "this is old school"
+        config :password, :validate => :password
+      end.new({
+        "old_opt" => "whut",
+        "password" => password
+      })
+    end
+
+    it "should not log the password" do
+      expect(LogStash::Logging::Logger).to receive(:new).with(anything).and_return(double_logger)
+      expect(double_logger).to receive(:warn) do |arg1,arg2|
+          message = 'You are using a deprecated config setting "old_opt" set in test_deprecated. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. this is old school If you have any questions about this, please visit the #logstash channel on freenode irc.'
+          expect(arg1).to eq(message)
+          expect(arg2[:plugin].to_s).to include('"password"=><password>')
+        end.once
+      subject
+    end
+  end
+
   context "when validating :bytes successfully" do
     subject do
       local_num_bytes = num_bytes # needs to be locally scoped :(

data/spec/logstash/timestamp_spec.rb

@@ -30,6 +30,10 @@ describe LogStash::Timestamp do
       expect{LogStash::Timestamp.new("foobar")}.to raise_error
     end
 
+    it "should copy itself correctly on clone and dup" do
+      expect(LogStash::Timestamp.now.clone.to_java).not_to be_nil
+      expect(LogStash::Timestamp.now.dup.to_java).not_to be_nil
+    end
   end
 
 end

data/versions-gem-copy.yml

@@ -1,7 +1,7 @@
 ---
-logstash: 5.6.5
-logstash-core: 5.6.5
-logstash-core-plugin-api: 2.1.12
+logstash: 5.6.6
+logstash-core: 5.6.6
+logstash-core-plugin-api: 2.1.29
 
 # Note: this file is copied to the root of logstash-core because its gemspec needs it when
 #       bundler evaluates the gemspec via bin/logstash

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.6.5
+  version: 5.6.6
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-04 00:00:00.000000000 Z
+date: 2018-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -373,6 +373,8 @@ files:
 - lib/logstash/config/modules_common.rb
 - lib/logstash/config/source/modules.rb
 - lib/logstash/config/string_escape.rb
+- lib/logstash/dependency_report.rb
+- lib/logstash/dependency_report_runner.rb
 - lib/logstash/elasticsearch_client.rb
 - lib/logstash/environment.rb
 - lib/logstash/errors.rb