logstash-core 6.3.2-java → 6.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 999e61360e29d34d6dbf15eebfa5a852f0114cc034c876e65f0ee434aadbefd7
-  data.tar.gz: 0366ea08d19ed8f15316c6b48ab8262ac8f5f810be24308017b445624578ed5c
+  metadata.gz: 75e766866b8e8de1c59212aab21120296be4a2b3768e736e9fc7fae7851d6687
+  data.tar.gz: 6c9f88892074eaa5c909df9365f94e58e5ec6781c559268b72fdc3c47f59be57
 SHA512:
-  metadata.gz: 83bb9dc76db210ebf2a1d6f1094b2f0aaf2aa4e1fdeca7b97c9682b585138b67f45b1138357b096da241cf9145b787db83d615e62fdca072c80675c1769d4e92
-  data.tar.gz: c1d165d04b42ba541b42110f4e640cfa3f818a74c585a564df0c45bef69a43a2d6b137f04fb1fd57313994220ac80ec9d7ca862611d77dae6248da476b2ed2c4
+  metadata.gz: 3dc2bdc1806597be02fd49c75f7c2060543b5decbbb6c5657d680798d2092cabce49044fe8d08490a0ee37f507f4ff5bed5c71b21e72790a81b99e143a58c52d
+  data.tar.gz: 482cbc47fc99ff43101b28bf0f6fff6488fe1faeeea717a1934afd8dde672d4efd523a7f56749bc4f3822628029ac39b525f66e9d0a6c1207e7cc5292793effb

data/lib/logstash/agent.rb

@@ -1,18 +1,12 @@
 # encoding: utf-8
 require "logstash/environment"
-require "logstash/errors"
 require "logstash/config/cpu_core_strategy"
 require "logstash/instrument/collector"
-require "logstash/instrument/metric"
 require "logstash/instrument/periodic_pollers"
-require "logstash/instrument/collector"
-require "logstash/instrument/metric"
 require "logstash/pipeline"
 require "logstash/webserver"
-require "logstash/event_dispatcher"
 require "logstash/config/source_loader"
 require "logstash/pipeline_action"
-require "logstash/converge_result"
 require "logstash/state_resolver"
 require "stud/trap"
 require "uri"
@@ -51,6 +45,16 @@ class LogStash::Agent
     # Generate / load the persistent uuid
     id
 
+    # Set the global FieldReference parsing mode
+    parsing_mode = case setting('config.field_reference.parser')
+                   when 'COMPAT' then org.logstash.FieldReference::ParsingMode::COMPAT;
+                   when 'LEGACY' then org.logstash.FieldReference::ParsingMode::LEGACY;
+                   when 'STRICT' then org.logstash.FieldReference::ParsingMode::STRICT;
+                   else fail('Unsupported FieldReference parsing mode')
+                   end
+    logger.debug("Setting global FieldReference parsing mode: #{parsing_mode}")
+    org.logstash.FieldReference::set_parsing_mode(parsing_mode)
+
     # This is for backward compatibility in the tests
     if source_loader.nil?
       @source_loader = LogStash::Config::SourceLoader.new

data/lib/logstash/api/modules/logging.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 #
 java_import org.apache.logging.log4j.core.LoggerContext
+java_import java.lang.IllegalArgumentException
 
 module LogStash
   module Api
@@ -30,6 +31,9 @@ module LogStash
               raise ArgumentError, I18n.t("logstash.web_api.logging.unrecognized_option", :option => remaining.keys.first)
             end
             respond_with({"acknowledged" => true})
+          rescue IllegalArgumentException => e
+            status 400
+            respond_with({"error" => e.message})
           rescue ArgumentError => e
             status 400
             respond_with({"error" => e.message})

data/lib/logstash/api/service.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/instrument/collector"
-require "logstash/util/loggable"
 
 module LogStash
   module Api

data/lib/logstash/bootstrap_check/default_config.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/errors"
-require "logstash/logging"
 
 module LogStash module BootstrapCheck
   class DefaultConfig

data/lib/logstash/bootstrap_check/persisted_queue_config.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require 'logstash/errors'
 
 module LogStash
   module BootstrapCheck

data/lib/logstash/codecs/base.rb

@@ -1,8 +1,6 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/event"
 require "logstash/plugin"
-require "logstash/logging"
 
 # This is the base class for logstash codecs.
 module LogStash::Codecs; class Base < LogStash::Plugin
@@ -41,12 +39,12 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   # over the current API for shared plugins
   # It is best if the codec implements this directly
   def multi_encode(events)
-    if @has_encode_sync              
+    if @has_encode_sync
       events.map {|event| [event, self.encode_sync(event)]}
     else
       batch = Thread.current[:logstash_output_codec_batch] ||= []
       batch.clear
-      
+
       events.each {|event| self.encode(event) }
       batch
     end

data/lib/logstash/compiler.rb

@@ -1,4 +1,3 @@
-require 'logstash/util/loggable'
 require 'logstash/compiler/lscl/lscl_grammar'
 
 java_import org.logstash.config.ir.PipelineIR
@@ -15,10 +14,10 @@ module LogStash; class Compiler
     input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph
     output_graph = Graph.combine(*graph_sections.map {|s| s[:output] }).graph
 
-    filter_graph = graph_sections.reduce(nil) do |acc, s| 
+    filter_graph = graph_sections.reduce(nil) do |acc, s|
       filter_section = s[:filter]
 
-      if acc.nil? 
+      if acc.nil?
         filter_section
       else
         acc.chain(filter_section)

data/lib/logstash/compiler/lscl.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require 'logstash/errors'
 require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
 require "logstash/compiler/lscl/helpers"

data/lib/logstash/config/config_ast.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require 'logstash/errors'
 require "logstash/compiler/lscl/helpers"
 require "treetop"
 

data/lib/logstash/config/cpu_core_strategy.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/config/defaults"
 
 module LogStash module Config module CpuCoreStrategy

data/lib/logstash/config/defaults.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "concurrent"
 
 module LogStash module Config module Defaults

data/lib/logstash/config/file.rb

@@ -1,8 +1,6 @@
 # encoding: utf-8
-require "logstash/namespace"
 require "logstash/config/grammar"
 require "logstash/config/config_ast"
-require "logstash/errors"
 require "logger"
 
 class LogStash::Config::File

data/lib/logstash/config/mixin.rb

@@ -1,7 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/plugins/registry"
-require "logstash/logging"
 require "logstash/util/password"
 require "logstash/util/safe_uri"
 require "logstash/version"
@@ -33,21 +30,21 @@ LogStash::Environment.load_locale!
 # }
 #
 module LogStash::Config::Mixin
-  
+
   include LogStash::Util::SubstitutionVariables
-  
+
   attr_accessor :config
   attr_accessor :original_params
 
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
-  
+
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
     # Add the DSL methods to the 'base' given.
     base.extend(LogStash::Config::Mixin::DSL)
   end
-  
+
   def config_init(params)
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.

data/lib/logstash/config/modules_common.rb

@@ -1,11 +1,9 @@
 # encoding: utf-8
-require "logstash/util/loggable"
 require "logstash/elasticsearch_client"
 require "logstash/modules/kibana_client"
 require "logstash/modules/elasticsearch_importer"
 require "logstash/modules/kibana_importer"
 require "logstash/modules/settings_merger"
-require "logstash/errors"
 
 module LogStash module Config
   class ModulesCommon # extracted here for bwc with 5.x

data/lib/logstash/config/source/local.rb

@@ -1,8 +1,6 @@
 # encoding: utf-8
 require "logstash/config/source/base"
 require "logstash/config/pipeline_config"
-require "logstash/util/loggable"
-require "logstash/errors"
 require "uri"
 
 module LogStash module Config module Source
@@ -72,7 +70,7 @@ module LogStash module Config module Source
 
           config_string = ::File.read(file)
           config_string.force_encoding("UTF-8")
-          
+
           if config_string.valid_encoding?
             part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
             config_parts << part

data/lib/logstash/config/source/modules.rb

@@ -2,8 +2,6 @@
 require "logstash/config/source/base"
 require "logstash/config/modules_common"
 require "logstash/config/pipeline_config"
-require "logstash/util/loggable"
-require "logstash/errors"
 
 module LogStash module Config module Source
   class Modules < Base

data/lib/logstash/config/source/multi_local.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/config/source/local"
-require "logstash/util/loggable"
 require "logstash/pipeline_settings"
 
 module LogStash module Config module Source

data/lib/logstash/config/source_loader.rb

@@ -2,7 +2,6 @@
 require "logstash/config/source/local"
 require "logstash/config/source/modules"
 require "logstash/config/source/multi_local"
-require "logstash/errors"
 require "thread"
 require "set"
 
@@ -53,7 +52,7 @@ module LogStash module Config
         # This shouldn't happen with the settings object or with any external plugins.
         # but lets add a guard so we fail fast.
         @sources_lock.synchronize do
-          @logger.error "No source loaders matched! This shouldn't happen", :sources => @sources
+          logger.error "No source loaders matched! This shouldn't happen", :sources => @sources
         end
         raise LogStash::InvalidSourceLoaderSettingError, "Can't find an appropriate config loader with current settings"
       else

data/lib/logstash/dependency_report.rb

@@ -6,6 +6,8 @@ $DEBUGLIST = (ENV["DEBUG"] || "").split(",")
 require "clamp"
 require "rubygems"
 require "jars/gemspec_artifacts"
+require 'fileutils'
+require 'securerandom'
 
 class LogStash::DependencyReport < Clamp::Command
   option [ "--csv" ], "OUTPUT_PATH", "The path to write the dependency report in csv format.",
@@ -13,19 +15,30 @@ class LogStash::DependencyReport < Clamp::Command
 
   def execute
     require "csv"
-    CSV.open(output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
+
+    tmp_dir = java.lang.System.getProperty("java.io.tmpdir")
+    ruby_output_path = File.join(tmp_dir, SecureRandom.uuid)
+    # Write a CSV with just the ruby stuff
+    CSV.open(ruby_output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
       puts "Finding gem dependencies"
       gems.each { |d| csv << d }
-      puts "Finding java/jar dependencies"
+      puts "Finding gem embedded java/jar dependencies"
       jars.each { |d| csv << d }
     end
+    puts "Wrote temporary ruby deps CSV to #{ruby_output_path}"
 
-    # Copy in COPYING.csv which is a best-effort, hand-maintained file of dependency license information.
-    File.open(output_path, "a+") do |file|
-      extra = File.join(File.dirname(__FILE__), "..", "..", "..", "COPYING.csv")
-      file.write(IO.read(extra))
+    # Use gradle to find the rest and add to the ruby CSV
+    puts "Find gradle jar dependencies #{Dir.pwd}"
+    command = ["./gradlew", "generateLicenseReport", "-PlicenseReportInputCSV=#{ruby_output_path}", "-PlicenseReportOutputCSV=#{output_path}"]
+    puts "Executing #{command}"
+    system(*command)
+    if $?.exitstatus != 0
+      raise "Could not run gradle java deps! Exit status #{$?.exitstatus}"
     end
+
     nil
+  ensure
+    FileUtils.rm(ruby_output_path) if ruby_output_path
   end
 
   def gems

data/lib/logstash/elasticsearch_client.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
 require "elasticsearch"
 require "elasticsearch/transport/transport/http/manticore"
 

data/lib/logstash/environment.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 require "logstash-core/logstash-core"
-require "logstash/errors"
-require "logstash/java_integration"
 require "logstash/config/cpu_core_strategy"
 require "logstash/settings"
 require "logstash/util/cloud_setting_id"
@@ -26,6 +24,8 @@ module LogStash
     Setting::NullableString.new("config.string", nil, false),
            Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []),
            Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []),
+                    Setting.new("modules_list", Array, []),
+                    Setting.new("modules_variable_list", Array, []),
            Setting::Modules.new("cloud.id", LogStash::Util::CloudSettingId),
            Setting::Modules.new("cloud.auth",LogStash::Util::CloudSettingAuth),
            Setting::Boolean.new("modules_setup", false),
@@ -33,6 +33,7 @@ module LogStash
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds
            Setting::Boolean.new("config.support_escapes", false),
+            Setting::String.new("config.field_reference.parser", "COMPAT", true, %w(STRICT COMPAT LEGACY)),
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
            Setting::Boolean.new("pipeline.system", false),

data/lib/logstash/errors.rb

@@ -1,15 +1 @@
-# encoding: utf-8
-
-module LogStash
-  class EnvironmentError < Error; end
-  class ConfigurationError < Error; end
-  class PluginLoadingError < Error; end
-  class ShutdownSignal < StandardError; end
-  class PluginNoVersionError < Error; end
-  class BootstrapCheckError < Error; end
-
-  class Bug < Error; end
-  class ThisMethodWasRemoved < Bug; end
-  class ConfigLoadingError < Error; end
-  class InvalidSourceLoaderSettingError < Error; end
-end
+# Keeping this file for backwards compatibility with plugins that include it directly.

data/lib/logstash/event.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 
-require "logstash/namespace"
 require "logstash/json"
 
 # transient pipeline events for normal in-flow signaling as opposed to

data/lib/logstash/event_dispatcher.rb

@@ -1,40 +1 @@
-# encoding: utf-8
-module LogStash
-  class EventDispatcher
-    java_import "java.util.concurrent.CopyOnWriteArraySet"
-
-    attr_reader :emitter
-
-    def initialize(emitter)
-      @emitter = emitter
-      @listeners = CopyOnWriteArraySet.new
-    end
-
-    # This operation is slow because we use a CopyOnWriteArrayList
-    # But the majority of the addition will be done at bootstrap time
-    # So add_listener shouldn't be called often at runtime.
-    #
-    # On the other hand the notification could be called really often.
-    def add_listener(listener)
-      @listeners.add(listener)
-    end
-
-    # This operation is slow because we use a `CopyOnWriteArrayList` as the backend, instead of a
-    # ConcurrentHashMap, but since we are mostly adding stuff and iterating the `CopyOnWriteArrayList`
-    # should provide a better performance.
-    #
-    # See note on add_listener, this method shouldn't be called really often.
-    def remove_listener(listener)
-      @listeners.remove(listener)
-    end
-
-    def fire(method_name, *arguments)
-      @listeners.each do |listener|
-        if listener.respond_to?(method_name)
-          listener.send(method_name, emitter, *arguments)
-        end
-      end
-    end
-    alias_method :execute, :fire
-  end
-end
+# Keeping this file for backwards compatibility with plugins that include it directly.

data/lib/logstash/execution_context.rb

@@ -1,19 +1,2 @@
-# encoding: utf-8
-require "logstash/util/dead_letter_queue_manager"
-module LogStash
-  class ExecutionContext
-    attr_reader :pipeline, :agent, :dlq_writer
-
-    def initialize(pipeline, agent, plugin_id, plugin_type, dlq_writer)
-      @pipeline = pipeline
-      @agent = agent
-      @plugin_id = plugin_id
-      @plugin_type = plugin_type
-      @dlq_writer = LogStash::Util::PluginDeadLetterQueueWriter.new(dlq_writer, @plugin_id, @plugin_type)
-    end
-
-    def pipeline_id
-      @pipeline.pipeline_id
-    end
-  end
-end
+# The contents of this file have been ported to Java. It is included for for compatibility
+# with plugins that directly include it.