logstash-core 6.1.1-java → 6.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7552e565f3c3c48330156cbbe377242f94224743eb8fbe7570da6d9dcd0a2de0
-  data.tar.gz: 52ca8909ad7532ae6b835e291925b88637b7da30a857c97f36e28fc089372334
+  metadata.gz: b00c8adeb8f8cfd17b847fbbeed77d2367ae24573d495ff2e23f08f02a7a7178
+  data.tar.gz: 121f070df4f77d854f6c9839e7adb4cbd8064f32053d24e93ef4c0694242f423
 SHA512:
-  metadata.gz: fe8ab5dcfcb6e4a88ccfa4b1b879c509d1f2152cbd27d02973bfe0d94da5d61641d45d4452d827e17e6669608ebd0491bfb98cbd2e5871dc258d333de903c522
-  data.tar.gz: 5bd7b2485ec5ce44a7cf8e601d8171c498e55c341a1cbf3384fa85f71f68e0df8a9d916a42dfd90e14cdccf5d921e9fd6049a5aa45d7727fde771d1500f3fedc
+  metadata.gz: 10a10a11213c8132deb3a3a87b1c1de3e383b642c3b8bca2280d85936010808fcf2d748565bb8bb43f4c67c82a4641d797709c165b606148e723805084069620
+  data.tar.gz: a2f39c7e42131ba604af2862a1046d33f1ed80e3284a9fcd54c749c7916c067905217505412c1e42976a4fdb5fca32bd65d8949f8b93920e74ed049b2cf1a771

data/lib/logstash-core_jars.rb

@@ -15,14 +15,14 @@ rescue LoadError
 end
 
 if defined? Jars
-  require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' )
-  require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
-  require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' )
-  require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' )
-  require_jar( 'org.codehaus.janino', 'commons-compiler', '3.0.7' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' )
-  require_jar( 'org.codehaus.janino', 'janino', '3.0.7' )
+  require_jar 'org.apache.logging.log4j', 'log4j-core', '2.6.2'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1'
+  require_jar 'org.apache.logging.log4j', 'log4j-api', '2.6.2'
+  require_jar 'org.slf4j', 'slf4j-api', '1.7.21'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1'
+  require_jar 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2'
+  require_jar 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1'
+  require_jar 'org.codehaus.janino', 'commons-compiler', '3.0.7'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1'
+  require_jar 'org.codehaus.janino', 'janino', '3.0.7'
 end

data/lib/logstash/compiler/lscl.rb

@@ -262,7 +262,10 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
         java_f_branch = f_branch && javaify_sexpr(f_branch)
 
         if java_t_branch || java_f_branch
-          jdsl.iIf(condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop)
+          # We use the condition as the source with metadata because it hashes correctly
+          # It's hard to use the 'real' source due to the re-branching from if / elsif into if/else only
+          # branches. We should come back and improve this at some point if that makes a difference
+          jdsl.iIf(condition.source_with_metadata, condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop)
         else
           jdsl.noop()
         end
@@ -297,7 +300,8 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
 
       all_elements = [first_element, *rest_elements]
 
-      if all_elements.size == 1
+
+      res = if all_elements.size == 1
         elem = all_elements.first
         if elem.is_a?(Selector)
           eventValue = elem.recursive_select(SelectorElement).first.expr
@@ -310,6 +314,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
       else
         join_conditions(all_elements)
       end
+      res
     end
 
     def precedence(op)
@@ -339,13 +344,13 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
 
       case op
       when :and
-        return jdsl.eAnd(left, right);
+        return jdsl.eAnd(source_meta, left, right);
       when :nand
-        return jdsl.eNand(left, right);
+        return jdsl.eNand(source_meta, left, right);
       when :or
-        return jdsl.eOr(left, right);
+        return jdsl.eOr(source_meta, left, right);
       when :xor
-        return jdsl.eXor(left, right);
+        return jdsl.eXor(source_meta, left, right);
       else
         raise "Unknown op #{jop}"
       end
@@ -394,7 +399,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
       while elem = stack.pop
         if elem.is_a?(::Method)
           right, left = working_stack.pop, working_stack.pop
-          working_stack << elem.call(left, right)
+          working_stack << elem.call(source_meta, left, right)
         else
           working_stack << elem
         end
@@ -472,13 +477,13 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
       # Handle string rvalues, they just get turned into regexps
       # Maybe we really shouldn't handle these anymore...
       if regexp.class == org.logstash.config.ir.expression.ValueExpression
-        regexp = jdsl.eRegex(regexp.get)
+        regexp = jdsl.eRegex(source_meta, regexp.get)
       end
       
       raise "Expected a selector in #{text_value}!" unless selector
       raise "Expected a regexp in #{text_value}!" unless regexp
 
-      operator_method.call(source_meta, selector, regexp);
+      operator_method.call(source_meta, selector, regexp)
     end
   end
 
@@ -512,9 +517,9 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
     
     def expr
       if self.text_value == '!~'
-        jdsl.method(:eRegexNeq)
+        jdsl.java_method(:eRegexNeq, [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.ValueExpression])
       elsif self.text_value == '=~'
-        jdsl.method(:eRegexEq)
+        jdsl.java_method(:eRegexEq, [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.ValueExpression])
       else
         raise "Unknown regex operator #{self.text_value}"
       end

data/lib/logstash/compiler/lscl/helpers.rb

@@ -49,9 +49,10 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
       org.logstash.config.ir.DSL
     end
     
-    AND_METHOD = jdsl.method(:eAnd)
-    NAND_METHOD = jdsl.method(:eNand)
-    OR_METHOD = jdsl.method(:eOr)
-    XOR_METHOD = jdsl.method(:eXor)
+    BOOLEAN_DSL_METHOD_SIGNATURE = [org.logstash.common.SourceWithMetadata, org.logstash.config.ir.expression.Expression, org.logstash.config.ir.expression.Expression]
+    AND_METHOD = jdsl.java_method(:eAnd, BOOLEAN_DSL_METHOD_SIGNATURE)
+    NAND_METHOD = jdsl.java_method(:eNand, BOOLEAN_DSL_METHOD_SIGNATURE)
+    OR_METHOD = jdsl.java_method(:eOr, BOOLEAN_DSL_METHOD_SIGNATURE)
+    XOR_METHOD = jdsl.java_method(:eXor, BOOLEAN_DSL_METHOD_SIGNATURE)
   end
 end; end; end; end; end

data/lib/logstash/dependency_report.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+Thread.abort_on_exception = true
+Encoding.default_external = Encoding::UTF_8
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+require "clamp"
+require "logstash/namespace"
+require "rubygems"
+require "jars/gemspec_artifacts"
+
+class LogStash::DependencyReport < Clamp::Command
+  option [ "--csv" ], "OUTPUT_PATH", "The path to write the dependency report in csv format.",
+    :required => true, :attribute_name => :output_path
+
+  def execute
+    require "csv"
+    CSV.open(output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
+      puts "Finding gem dependencies"
+      gems.each { |d| csv << d }
+      puts "Finding java/jar dependencies"
+      jars.each { |d| csv << d }
+    end
+
+    # Copy in COPYING.csv which is a best-effort, hand-maintained file of dependency license information.
+    File.open(output_path, "a+") do |file|
+      extra = File.join(File.dirname(__FILE__), "..", "..", "..", "COPYING.csv")
+      file.write(IO.read(extra))
+    end
+    nil
+  end
+
+  def gems
+    # @mgreau requested `logstash-*` dependencies be removed from this list: 
+    # https://github.com/elastic/logstash/pull/8837#issuecomment-351859433
+    Gem::Specification.reject { |g| g.name =~ /^logstash-/ }.collect do |gem|
+      licenses = ("UNKNOWN" if gem.licenses.empty?) || (gem.licenses.map { |l| SPDX.map(l) }.join("|") if !gem.licenses.empty?)
+      [gem.name, gem.version.to_s, gem.homepage, licenses]
+    end
+  end
+
+  def jars
+    jars = []
+    # For any gems with jar dependencies,
+    #   Look at META-INF/MANIFEST.MF for any jars in each gem
+    #   Note any important details.
+    Gem::Specification.select { |g| g.requirements && g.requirements.any? { |r| r =~ /^jar / } }.collect do |gem|
+
+      # Where is the gem installed
+      root = gem.full_gem_path
+
+      Dir.glob(File.join(root, "**", "*.jar")).collect do |path|
+        jar = java.util.jar.JarFile.new(path)
+        manifest = jar.getManifest
+
+        pom_entries = jar.entries.select { |t| t.getName.start_with?("META-INF/maven/") && t.getName.end_with?("/pom.properties") }
+
+        # Some jar files have multiple maven pom.properties files. It is unclear how to know what is correct?
+        # TODO(sissel): Maybe we should use all pom.properties files? None of the pom.properties/pom.xml files have license information, though.
+        # TODO(sissel): In some cases, there are META-INF/COPYING and
+        #   META-INF/NOTICE.txt files? Can we use these somehow? There is no
+        #   common syntax for parsing these files, though...
+        pom_map = if pom_entries.count == 1
+          pom_in = jar.getInputStream(pom_entries.first)
+          pom_content = pom_in.available.times.collect { pom_in.read }.pack("C*")
+          # Split non-comment lines by `key=val` into a map { key => val }
+          Hash[pom_content.split(/\r?\n/).grep(/^[^#]/).map { |line| line.split("=", 2) }]
+        else
+          {}
+        end
+
+        next if manifest.nil?
+        # convert manifest attributes to a map w/ keys .to_s
+        # without this, the attribute keys will be `Object#inspect` values
+        # like #<Java::JavaUtilJar::Attributes::Name0xabcdef0>
+        attributes = Hash[manifest.getMainAttributes.map { |k,v| [k.to_s, v] }]
+
+        begin
+          # Prefer the maven/pom groupId when it is available.
+          artifact = pom_map.fetch("artifactId", attributes.fetch("Implementation-Title"))
+          group = pom_map.fetch("groupId", attributes.fetch("Implementation-Vendor-Id"))
+          jars << [
+            group + ":" + artifact,
+            attributes.fetch("Bundle-Version"),
+            attributes.fetch("Bundle-DocURL"),
+            SPDX.map(attributes.fetch("Bundle-License")),
+          ]
+        rescue KeyError => e
+          # The jar is missing a required manifest field, it may not have any useful manifest data.
+          # Ignore it and move on.
+        end
+      end
+    end
+    jars.uniq.sort
+  end
+
+  module SPDX
+    # This is a non-exhaustive, best effort list of licenses as they map to SPDX identifiers.
+    ALIASES = {
+      "Apache-2.0" => [
+        "Apache 2",
+        "apache-2.0",
+        "Apache 2.0",
+        "Apache License (2.0)",
+        "Apache License 2.0",
+        "https://www.apache.org/licenses/LICENSE-2.0.txt",
+        "http://www.apache.org/licenses/LICENSE-2.0.txt",
+      ],
+      "Artistic-2.0" => [
+        "Artistic 2.0"
+      ],
+      "BSD-2-Clause" => [
+        "2-clause BSDL",
+        "2-clause"
+      ],
+      "GPL-2.0" => [
+        "GPL-2"
+      ]
+    }
+
+    # Get a map of name => spdx
+    MAP_APACHE2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[0]]
+    MAP_ARTISTIC2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[1]]
+    MAP_BSD = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[2]]
+    MAP_GPL2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[3]]
+
+    module_function
+    def map(value)
+      MAP_APACHE2[value] ||  MAP_ARTISTIC2[value] || MAP_BSD[value] ||  MAP_GPL2[value] || value
+    end
+  end
+end

data/lib/logstash/dependency_report_runner.rb

@@ -0,0 +1,17 @@
+require_relative "../../../lib/bootstrap/environment"
+
+if $0 == __FILE__
+  begin
+    LogStash::Bundler.setup!({:without => [:build, :development]})
+  rescue => Bundler::GemfileNotFound
+    $stderr.puts("No Gemfile found. Maybe you need to run `rake artifact:tar`?")
+    raise
+  end
+
+  require "logstash/namespace"
+  require_relative "../../../lib/bootstrap/patches/jar_dependencies"
+  require "logstash/dependency_report"
+
+  exit_status = LogStash::DependencyReport.run 
+  exit(exit_status || 0)
+end

data/spec/logstash/compiler/compiler_spec.rb

@@ -8,6 +8,10 @@ describe LogStash::Compiler do
     Java::OrgLogstashConfigIr::DSL
   end
 
+  def rand_meta
+    org.logstash.common.SourceWithMetadata.new("test", SecureRandom.uuid, 1, 1, SecureRandom.uuid)
+  end
+
   let(:source_protocol) { "test_proto" }
 
   let(:settings) { mock_settings({}) }
@@ -173,7 +177,7 @@ describe LogStash::Compiler do
         let(:plugin_source) { "generator {}" }
 
         it "should contain the plugin" do
-          expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator"))
+          expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator"))
         end
       end
 
@@ -190,7 +194,7 @@ describe LogStash::Compiler do
         end
 
         it "should contain the plugin" do
-          expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
+          expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
         end
       end
 
@@ -203,7 +207,7 @@ describe LogStash::Compiler do
         end
 
         it "should contain the plugin" do
-          expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
+          expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
         end
       end
 
@@ -216,7 +220,7 @@ describe LogStash::Compiler do
         end
 
         it "should contain the plugin" do
-          expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
+          expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator", expected_plugin_args))
         end
       end
 
@@ -245,7 +249,7 @@ describe LogStash::Compiler do
         end
 
         it "should merge the contents of the individual directives" do
-          expect(c_plugin).to ir_eql(j.iPlugin(FILTER, "grok", expected_plugin_args))
+          expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
         end
 
         describe "a filter plugin that has nested Hash directives" do
@@ -300,7 +304,7 @@ describe LogStash::Compiler do
           end
 
           it "should produce a nested ::Hash object" do
-            expect(c_plugin).to ir_eql(j.iPlugin(FILTER, "matryoshka", expected_plugin_args))
+            expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "matryoshka", expected_plugin_args))
           end
         end
       end
@@ -313,7 +317,7 @@ describe LogStash::Compiler do
         let(:source) { "input { generator {} }" }
 
         it "should contain the single input" do
-          expect(input).to ir_eql(j.iPlugin(INPUT, "generator"))
+          expect(input).to ir_eql(j.iPlugin(rand_meta, INPUT, "generator"))
         end
 
         it_should_behave_like("component source_with_metadata") do
@@ -326,8 +330,8 @@ describe LogStash::Compiler do
 
         it "should contain both inputs" do
           expect(input).to ir_eql(j.iComposeParallel(
-                                j.iPlugin(INPUT, "generator", {"count" => 1}),
-                                j.iPlugin(INPUT, "generator", {"count" => 2})
+                                j.iPlugin(rand_meta, INPUT, "generator", {"count" => 1}),
+                                j.iPlugin(rand_meta, INPUT, "generator", {"count" => 2})
                               ))
         end
       end
@@ -351,7 +355,7 @@ describe LogStash::Compiler do
       let (:compiled_section) { compiled[section] }
 
       def splugin(*args)
-        j.iPlugin(section_name_enum, *args)
+        j.iPlugin(rand_meta, section_name_enum, *args)
       end
 
       def compose(*statements)
@@ -635,6 +639,7 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                            rand_meta,
                                             j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                             splugin("grok")
                                           )
@@ -647,6 +652,7 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                           j.noop,
                                           splugin("fplugin"),
@@ -660,6 +666,7 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                           j.noop,
                                           j.noop
@@ -673,6 +680,7 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                           splugin("tplugin"),
                                           splugin("fplugin")
@@ -686,9 +694,11 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                           splugin("tplugin"),
                                           j.iIf(
+                                            rand_meta,
                                             j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
                                             splugin("eifplugin"),
                                             splugin("fplugin")
@@ -712,12 +722,15 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
                                           splugin("tplugin"),
                                           j.iIf(
+                                            rand_meta,
                                             j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
                                             splugin("eifplugin"),
                                             j.iIf(
+                                              rand_meta,
                                               j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")),
                                               splugin("eeifplugin"),
                                               splugin("fplugin")
@@ -744,15 +757,18 @@ describe LogStash::Compiler do
 
           it "should compile correctly" do
             expect(compiled_section).to ir_eql(j.iIf(
+                                          rand_meta,
                                           j.eEq(j.eEventValue("[foo]"), j.eEventValue("[bar]")),
-                                          j.iIf(j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
+                                          j.iIf(rand_meta, j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
                                                    splugin("aplugin"),
                                                    j.noop
                                                   ),
                                           j.iIf(
+                                            rand_meta,
                                             j.eEq(j.eEventValue("[bar]"), j.eEventValue("[baz]")),
                                             splugin("bplugin"),
                                             j.iIf(
+                                              rand_meta,
                                               j.eEq(j.eEventValue("[baz]"), j.eEventValue("[bot]")),
                                               splugin("cplugin"),
                                               splugin("dplugin")
@@ -773,7 +789,7 @@ describe LogStash::Compiler do
         let(:source) { "input { } filter { grok {} } output { }" }
 
         it "should contain the single filter" do
-          expect(filter).to ir_eql(j.iPlugin(FILTER, "grok"))
+          expect(filter).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok"))
         end
 
         it_should_behave_like("component source_with_metadata") do
@@ -791,7 +807,7 @@ describe LogStash::Compiler do
         let(:source) { "input { } output { stdout {} }" }
 
         it "should contain the single input" do
-          expect(output).to ir_eql(j.iPlugin(OUTPUT, "stdout"))
+          expect(output).to ir_eql(j.iPlugin(rand_meta, OUTPUT, "stdout"))
         end
 
         it_should_behave_like("component source_with_metadata") do

data/versions-gem-copy.yml

@@ -1,6 +1,6 @@
 ---
-logstash: 6.1.1
-logstash-core: 6.1.1
+logstash: 6.1.2
+logstash-core: 6.1.2
 logstash-core-plugin-api: 2.1.16
 
 # jruby must reference a *released* version of jruby which can be downloaded from the official download url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 6.1.1
+  version: 6.1.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-17 00:00:00.000000000 Z
+date: 2018-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -387,6 +387,8 @@ files:
 - lib/logstash/config/source_loader.rb
 - lib/logstash/config/string_escape.rb
 - lib/logstash/converge_result.rb
+- lib/logstash/dependency_report.rb
+- lib/logstash/dependency_report_runner.rb
 - lib/logstash/elasticsearch_client.rb
 - lib/logstash/environment.rb
 - lib/logstash/errors.rb