logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 58e1606b49e2fe865d648c113d2316ba19a4604d
-  data.tar.gz: 7c50f49a82f7bd2dec47546ddb51459e35d44c0c
+SHA256:
+  metadata.gz: 46f57cb18b555e501983b0d9d9ae61c3be63fd5925f444a4018adce705fe6b1f
+  data.tar.gz: 963cbd763704c8c84ca8bb5055412baa6b57d22d3e65ac03cd7f6b2434ec227a
 SHA512:
-  metadata.gz: c8d297ddd76d6f6c2167f934169a041c7b1df0b76a39a661ddc308405159407fa0800a3a41cf0bd8aeca2a1684ebd45482672f83985f442f500b6f4ce6e9edad
-  data.tar.gz: 8e5e40ab3a364110604e825a43f05a83ea1b8da136f50f8ee6ae475edb69d25c6f70e076c20175eabacf6dddb9032e4d111a1d251ed22d9e5f393ee3c06ef90f
+  metadata.gz: 93a8a8282730e5c5e687b5bf2bc48d0aab82fa5ec8398336114f3e4a3d73d6705dac0854827ab9bfd5c52173177c72db87da5e6bb9a1ca95b14a826c51ec2860
+  data.tar.gz: 61df6640d1d26a96b014c9eb7399982a9829465472f9f05f5b4882c4a8720a821c6e4eca9548286dcb907e792f32f91ab1ca90fdb3f9c96dfbdb1b45d9e0b6a0

data/gemspec_jars.rb

@@ -2,9 +2,11 @@
 # runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec
 # for the jar-dependencies requirements.
 
+gem.requirements << "jar org.apache.logging.log4j:log4j-slf4j-impl, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
-gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.4"
-gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.7.4"
-gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.7.4"
-gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.7.4"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.3"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.7.3"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.7.3"
+gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.7.3"
+gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.7.3"

data/lib/logstash-core/logstash-core.rb

@@ -8,8 +8,8 @@ end
 require "logstash-core_jars"
 
 # local dev setup
-classes_dir = File.expand_path("../../../build/classes/main", __FILE__)
-resources_dir = File.expand_path("../../../build/resources/main", __FILE__)
+classes_dir = File.expand_path("../../../out/production/classes", __FILE__)
+resources_dir = File.expand_path("../../../out/production/resources", __FILE__)
 
 if File.directory?(classes_dir) && File.directory?(resources_dir)
   # if in local dev setup, add target to classpath

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "6.0.0-alpha2"
+LOGSTASH_CORE_VERSION = "6.0.0-beta1"

data/lib/logstash-core_jars.rb

@@ -2,21 +2,25 @@
 begin
   require 'jar_dependencies'
 rescue LoadError
+  require 'com/fasterxml/jackson/core/jackson-databind/2.7.3/jackson-databind-2.7.3.jar'
   require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
-  require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.7.4/jackson-module-afterburner-2.7.4.jar'
   require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
-  require 'com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar'
-  require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar'
-  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.7.4/jackson-dataformat-cbor-2.7.4.jar'
-  require 'com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar'
+  require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
+  require 'com/fasterxml/jackson/core/jackson-annotations/2.7.3/jackson-annotations-2.7.3.jar'
+  require 'org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar'
+  require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.7.3/jackson-module-afterburner-2.7.3.jar'
+  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.7.3/jackson-dataformat-cbor-2.7.3.jar'
+  require 'com/fasterxml/jackson/core/jackson-core/2.7.3/jackson-core-2.7.3.jar'
 end
 
 if defined? Jars
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.3' )
   require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.4' )
   require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.4' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
-  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.7.4' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.4' )
+  require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.3' )
+  require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
+  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.3' )
+  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.7.3' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.3' )
 end

data/lib/logstash/agent.rb

@@ -25,7 +25,7 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :name, :settings, :webserver, :dispatcher
+  attr_reader :metric, :name, :settings, :webserver, :dispatcher, :ephemeral_id
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -37,6 +37,7 @@ class LogStash::Agent
     @logger = self.class.logger
     @settings = settings
     @auto_reload = setting("config.reload.automatic")
+    @ephemeral_id = SecureRandom.uuid
 
     # Do not use @pipelines directly. Use #with_pipelines which does locking
     @pipelines = {}
@@ -57,7 +58,8 @@ class LogStash::Agent
       @source_loader = source_loader
     end
 
-    @reload_interval = setting("config.reload.interval")
+    # Normalize time interval to seconds
+    @reload_interval = setting("config.reload.interval") / 1_000_000_000.0
 
     @collect_metric = setting("metric.collect")
 

data/lib/logstash/api/commands/default_metadata.rb

@@ -12,7 +12,7 @@ module LogStash
         end
 
         def host
-          Socket.gethostname
+          @@host ||= Socket.gethostname
         end
 
         def version

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -17,7 +17,12 @@ class HotThreadsReport
     report << '=' * STRING_SEPARATOR_LENGTH
     report << "\n"
     hash[:threads].each do |thread|
-      thread_report = "#{I18n.t("logstash.web_api.hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+      line_str = I18n.t("logstash.web_api.hot_threads.thread_title", 
+        :percent_of_cpu_time => thread[:percent_of_cpu_time], 
+        :thread_state => thread[:state], 
+        :thread_name => thread[:name],
+        :thread_id => thread[:thread_id])
+      thread_report = "#{line_str} \n"
       thread_report << "#{thread[:path]}\n" if thread[:path]
       thread[:traces].each do |trace|
         thread_report << "\t#{trace}\n"
@@ -31,9 +36,10 @@ class HotThreadsReport
 
   def to_hash
     hash = { :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
-    @thread_dump.each do |thread_name, _hash|
+    @thread_dump.each do |_hash|
       thread_name, thread_path = _hash["thread.name"].split(": ")
       thread = { :name => thread_name,
+                 :thread_id => _hash["thread.id"],
                  :percent_of_cpu_time => cpu_time_as_percent(_hash),
                  :state => _hash["thread.state"]
       }

data/lib/logstash/api/commands/node.rb

@@ -27,8 +27,8 @@ module LogStash
         def pipeline(pipeline_id)
           extract_metrics(
             [:stats, :pipelines, pipeline_id.to_sym, :config],
-            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
-          )
+            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval, :dead_letter_queue_enabled, :dead_letter_queue_path
+          ).reject{|_, v|v.nil?}
         rescue
           {}
         end

data/lib/logstash/api/commands/stats.rb

@@ -120,8 +120,8 @@ module LogStash
               },
               :reloads => stats[:reloads],
               :queue => stats[:queue]
-            }
-          end
+            }.merge(stats[:dlq] ? {:dead_letter_queue => stats[:dlq]} : {})
+            end
         end # module PluginsStats
       end
     end

data/lib/logstash/bootstrap_check/bad_ruby.rb

@@ -4,8 +4,8 @@ require "logstash/errors"
 module LogStash module BootstrapCheck
   class BadRuby
     def self.check(settings)
-      if RUBY_VERSION < "1.9.2"
-        raise LogStash::BootstrapCheckError, "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      if RUBY_VERSION < "2.0"
+        raise LogStash::BootstrapCheckError, "Ruby 2.0 or later is required. (You are running: " + RUBY_VERSION + ")"
       end
     end
   end

data/lib/logstash/bootstrap_check/default_config.rb

@@ -1,13 +1,12 @@
 # encoding: utf-8
 require "logstash/errors"
+require "logstash/logging"
 
 module LogStash module BootstrapCheck
   class DefaultConfig
     def self.check(settings)
       # currently none of the checks applies if there are multiple pipelines
-      if settings.get("config.reload.automatic") && settings.get_setting("config.string").set?
-        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.reload-with-config-string")
-      end
+      # See LogStash::Config::Source::Base for any further settings conflict checks
     end
   end
 end end

data/lib/logstash/compiler.rb

@@ -2,19 +2,18 @@ require 'logstash/util/loggable'
 require 'logstash/compiler/lscl/lscl_grammar'
 
 java_import org.logstash.config.ir.PipelineIR
-java_import org.logstash.config.ir.graph.Graph;
-java_import org.logstash.config.ir.graph.PluginVertex;
+java_import org.logstash.config.ir.graph.Graph
 
 module LogStash; class Compiler
   include ::LogStash::Util::Loggable
 
-  def self.compile_sources(*sources_with_metadata)
+  def self.compile_sources(sources_with_metadata, settings)
     graph_sections = sources_with_metadata.map do |swm|
-      self.compile_graph(swm)
+      self.compile_graph(swm, settings)
     end
 
-    input_graph = org.logstash.config.ir.graph.Graph.combine(*graph_sections.map {|s| s[:input] }).graph
-    output_graph = org.logstash.config.ir.graph.Graph.combine(*graph_sections.map {|s| s[:output] }).graph
+    input_graph = Graph.combine(*graph_sections.map {|s| s[:input] }).graph
+    output_graph = Graph.combine(*graph_sections.map {|s| s[:output] }).graph
 
     filter_graph = graph_sections.reduce(nil) do |acc, s| 
       filter_section = s[:filter]
@@ -28,10 +27,10 @@ module LogStash; class Compiler
 
     original_source = sources_with_metadata.map(&:text).join("\n")
 
-    org.logstash.config.ir.PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
+    PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
   end
 
-  def self.compile_ast(source_with_metadata)
+  def self.compile_ast(source_with_metadata, settings)
     if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata)
       raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}"
     end
@@ -43,14 +42,15 @@ module LogStash; class Compiler
       raise ConfigurationError, grammar.failure_reason
     end
 
+    config.process_escape_sequences = settings.get_value("config.support_escapes")
     config.compile(source_with_metadata)
   end
 
-  def self.compile_imperative(source_with_metadata)
-    compile_ast(source_with_metadata)
+  def self.compile_imperative(source_with_metadata, settings)
+    compile_ast(source_with_metadata, settings)
   end
 
-  def self.compile_graph(source_with_metadata)
-    Hash[compile_imperative(source_with_metadata).map {|section,icompiled| [section, icompiled.toGraph]}]
+  def self.compile_graph(source_with_metadata, settings)
+    Hash[compile_imperative(source_with_metadata, settings).map {|section,icompiled| [section, icompiled.toGraph]}]
   end
 end; end

data/lib/logstash/compiler/lscl.rb

@@ -2,10 +2,14 @@
 require 'logstash/errors'
 require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
+require "logstash/config/string_escape"
+
 java_import org.logstash.config.ir.DSL
 java_import org.logstash.common.SourceWithMetadata
 
 module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSCL; module AST
+  PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
+
   # Helpers for parsing LSCL files
   module Helpers
     def source_meta
@@ -46,10 +50,6 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
       [self.input.line_of(start), self.input.column_of(start)]
     end
 
-    def empty_source_meta()
-      org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, nil)
-    end
-
     def jdsl
       org.logstash.config.ir.DSL
     end
@@ -77,6 +77,10 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
 
   class Config < Node
     include Helpers
+
+    def process_escape_sequences=(val)
+      set_meta(PROCESS_ESCAPE_SEQUENCES, val)
+    end
     
     def compile(base_source_with_metadata=nil)
       # There is no way to move vars across nodes in treetop :(
@@ -99,11 +103,12 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
         end
       end
 
+      compiled_section_map = {}
       section_map.keys.each do |key|
-        section_map[key] = compose_for(key).call(empty_source_meta, *section_map[key])
+        compiled_section_map[key] = compose_for(key).call(*section_map[key])
       end
 
-      section_map
+      compiled_section_map
     end
   end
 
@@ -179,7 +184,12 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
   
   class String < Value
     def expr
-      jdsl.e_value(source_meta, text_value[1...-1])
+      value = if get_meta(PROCESS_ESCAPE_SEQUENCES)
+        ::LogStash::Config::StringEscape.process_escapes(text_value[1...-1])
+      else
+        text_value[1...-1]
+      end
+      jdsl.eValue(source_meta, value)
     end
   end
   

data/lib/logstash/compiler/treetop_monkeypatches.rb

@@ -10,6 +10,7 @@ class Treetop::Runtime::SyntaxNode
     @ast_metadata ||= {}
     @ast_metadata[key] = value
   end
+
   def compile
     return "" if elements.nil?
     return elements.collect(&:compile).reject(&:empty?).join("")

data/lib/logstash/config/config_ast.rb

@@ -5,6 +5,7 @@ require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
 
 module LogStash; module Config; module AST
+  PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
 
   def self.deferred_conditionals=(val)
     @deferred_conditionals = val
@@ -37,6 +38,11 @@ module LogStash; module Config; module AST
   end
 
   class Config < Node
+    def process_escape_sequences=(val)
+      set_meta(PROCESS_ESCAPE_SEQUENCES, val)
+    end
+
+
     def compile
       LogStash::Config::AST.deferred_conditionals = []
       LogStash::Config::AST.deferred_conditionals_index = 0
@@ -279,7 +285,11 @@ module LogStash; module Config; module AST
   end
   class String < Value
     def compile
-      return Unicode.wrap(text_value[1...-1])
+      if get_meta(PROCESS_ESCAPE_SEQUENCES)
+        Unicode.wrap(LogStash::Config::StringEscape.process_escapes(text_value[1...-1]))
+      else
+        Unicode.wrap(text_value[1...-1])
+      end
     end
   end
   class RegExp < Value

data/lib/logstash/config/mixin.rb

@@ -143,6 +143,9 @@ module LogStash::Config::Mixin
   end # def config_init
 
   module DSL
+
+    include LogStash::Util::EnvironmentVariables
+
     attr_accessor :flags
 
     # If name is given, set the name and return it.
@@ -388,6 +391,8 @@ module LogStash::Config::Mixin
       # (see LogStash::Inputs::File for example)
       result = nil
 
+      value = deep_replace(value)
+
       if validator.nil?
         return true, value
       elsif validator.is_a?(Array)

data/lib/logstash/config/modules_common.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require "logstash/elasticsearch_client"
+require "logstash/modules/kibana_client"
+require "logstash/modules/elasticsearch_importer"
+require "logstash/modules/kibana_importer"
+require "logstash/modules/settings_merger"
+require "logstash/errors"
+
+module LogStash module Config
+  class ModulesCommon # extracted here for bwc with 5.x
+    include LogStash::Util::Loggable
+
+    MODULES_MAX_PIPELINES = 1
+
+    def self.pipeline_configs(settings)
+      pipelines = []
+      plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
+
+      cli_settings = settings.get("modules.cli")
+      yml_settings = settings.get("modules")
+
+      modules_array = if !(cli_settings.empty? && yml_settings.empty?)
+            LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
+          elsif cli_settings.empty?
+             yml_settings
+          else
+            cli_settings
+          end
+
+      if modules_array.empty?
+        # no specifed modules
+        return pipelines
+      end
+      logger.debug("Specified modules", :modules_array => modules_array.to_s)
+
+      module_names = modules_array.collect {|module_hash| module_hash["name"]}
+      if module_names.size > MODULES_MAX_PIPELINES
+        error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', '))
+        raise LogStash::ConfigLoadingError, error_message
+      end
+
+      if module_names.length > module_names.uniq.length
+        duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
+      end
+
+      available_module_names = plugin_modules.map(&:module_name)
+      specified_and_available_names = module_names & available_module_names
+
+      if (specified_and_available_names).empty?
+        i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names}
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", i18n_opts)
+      end
+
+      specified_and_available_names.each do |module_name|
+        connect_fail_args = {}
+        begin
+          module_settings = settings.clone
+
+          module_hash = modules_array.find {|m| m["name"] == module_name}
+          current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
+
+          alt_name = "module-#{module_name}"
+          pipeline_id = alt_name
+          module_settings.set("pipeline.id", pipeline_id)
+          current_module.with_settings(module_hash)
+          config_test = settings.get("config.test_and_exit")
+          if !config_test
+            esclient = LogStash::ElasticsearchClient.build(module_hash)
+            kbnclient = LogStash::Modules::KibanaClient.new(module_hash)
+            esconnected = esclient.can_connect?
+            kbnconnected = kbnclient.can_connect?
+            if esconnected && kbnconnected
+              current_module.add_kibana_version(kbnclient.version_parts)
+              current_module.import(
+                  LogStash::Modules::ElasticsearchImporter.new(esclient),
+                  LogStash::Modules::KibanaImporter.new(kbnclient)
+                )
+            else
+              connect_fail_args[:module_name] = module_name
+              connect_fail_args[:elasticsearch_hosts] = esclient.host_settings
+              connect_fail_args[:kibana_hosts] = kbnclient.host_settings
+            end
+          end
+          config_string = current_module.config_string
+          pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => module_settings}
+        rescue => e
+          new_error = LogStash::ConfigLoadingError.new(I18n.t("logstash.modules.configuration.parse-failed", :error => e.message))
+          new_error.set_backtrace(e.backtrace)
+          raise new_error
+        end
+
+        if !connect_fail_args.empty?
+          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", connect_fail_args)
+        end
+      end
+      pipelines
+    end
+  end
+end end