logstash-core 5.4.3-java → 5.5.0-java

data/lib/logstash/modules/scaffold.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "erb"
+
+require_relative "elasticsearch_config"
+require_relative "kibana_config"
+require_relative "logstash_config"
+
+module LogStash module Modules class Scaffold
+  include LogStash::Util::Loggable
+
+  attr_reader :directory, :module_name, :logstash_configuration, :kibana_configuration, :elasticsearch_configuration
+
+  def initialize(name, directory)
+    @module_name = name
+    @directory = directory  # this is the 'configuration folder in the GEM root.'
+  end
+
+  def import(import_engine)
+    @elasticsearch_configuration.resources.each do |resource|
+      import_engine.put(resource)
+    end
+    @kibana_configuration.resources.each do |resource|
+      import_engine.put(resource)
+    end
+  end
+
+  def with_settings(module_settings)
+    @logstash_configuration = LogStashConfig.new(self, module_settings)
+    @kibana_configuration = KibanaConfig.new(self, module_settings)
+    @elasticsearch_configuration = ElasticsearchConfig.new(self, module_settings)
+    self
+  end
+
+  def config_string()
+    # settings should be set earlier by the caller.
+    # settings should be the subset from the YAML file with a structure like
+    # {"name" => "plugin name", "k1" => "v1", "k2" => "v2"}, etc.
+    return nil if @logstash_configuration.nil?
+    @logstash_configuration.config_string
+  end
+end end end # class LogStash::Modules::Scaffold
+

data/lib/logstash/namespace.rb

@@ -11,4 +11,5 @@ module LogStash
   module PluginMixins; end
   module PluginManager; end
   module Api; end
+  module Modules; end
 end # module LogStash

data/lib/logstash/outputs/base.rb

@@ -109,6 +109,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/pipeline.rb

@@ -18,11 +18,15 @@ require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
 require "logstash/instrument/wrapped_write_client"
+require "logstash/util/dead_letter_queue_manager"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
 require "logstash/queue_factory"
 require "logstash/execution_context"
 
+java_import org.logstash.common.DeadLetterQueueFactory
+java_import org.logstash.common.io.DeadLetterQueueWriter
+
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
 
@@ -43,7 +47,12 @@ module LogStash; class BasePipeline
     @inputs = nil
     @filters = nil
     @outputs = nil
-    @execution_context = LogStash::ExecutionContext.new(@pipeline_id)
+
+    if settings.get_value("dead_letter_queue.enable")
+      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"))
+    else
+      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
+    end
 
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
@@ -90,16 +99,18 @@ module LogStash; class BasePipeline
 
     klass = Plugin.lookup(plugin_type, name)
 
+    execution_context = ExecutionContext.new(self, id, klass.config_name, @dlq_writer)
+
     if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric, @execution_context, OutputDelegatorStrategyRegistry.instance, args)
+      OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
     elsif plugin_type == "filter"
-      FilterDelegator.new(@logger, klass, type_scoped_metric, @execution_context, args)
+      FilterDelegator.new(@logger, klass, type_scoped_metric, execution_context, args)
     else # input
       input_plugin = klass.new(args)
       scoped_metric = type_scoped_metric.namespace(id.to_sym)
       scoped_metric.gauge(:name, input_plugin.config_name)
       input_plugin.metric = scoped_metric
-      input_plugin.execution_context = @execution_context
+      input_plugin.execution_context = execution_context
       input_plugin
     end
   end
@@ -162,6 +173,7 @@ module LogStash; class Pipeline < BasePipeline
     )
     @drain_queue =  @settings.get_value("queue.drain")
 
+
     @events_filtered = Concurrent::AtomicFixnum.new(0)
     @events_consumed = Concurrent::AtomicFixnum.new(0)
 
@@ -239,6 +251,7 @@ module LogStash; class Pipeline < BasePipeline
   def close
     @filter_queue_client.close
     @queue.close
+    @dlq_writer.close
   end
 
   def transition_to_running
@@ -599,6 +612,7 @@ module LogStash; class Pipeline < BasePipeline
         n.gauge(:page_capacity_in_bytes, queue.page_capacity)
         n.gauge(:max_queue_size_in_bytes, queue.max_size_in_bytes)
         n.gauge(:max_unread_events, queue.max_unread_events)
+        n.gauge(:queue_size_in_bytes, queue.persisted_size_in_bytes)
       end
       pipeline_metric.namespace([:data]).tap do |n|
         n.gauge(:free_space_in_bytes, file_store.get_unallocated_space)

data/lib/logstash/plugin.rb

@@ -3,6 +3,7 @@ require "logstash/namespace"
 require "logstash/logging"
 require "logstash/config/mixin"
 require "logstash/instrument/null_metric"
+require "logstash/util/dead_letter_queue_manager"
 require "concurrent"
 require "securerandom"
 

data/lib/logstash/plugins/registry.rb

@@ -3,6 +3,7 @@ require "rubygems/package"
 require "logstash/util/loggable"
 require "logstash/plugin"
 require "logstash/plugins/hooks_registry"
+require "logstash/modules/scaffold"
 
 module LogStash module Plugins
   class Registry
@@ -109,6 +110,10 @@ module LogStash module Plugins
         .each { |specification| specification.register(hooks, LogStash::SETTINGS) }
     end
 
+    def plugins_with_type(type)
+      @registry.values.select { |specification| specification.type.to_sym == type.to_sym }.collect(&:klass)
+    end
+
     def load_available_plugins
       GemRegistry.logstash_plugins.each do |plugin_context|
         # When a plugin has a HOOK_FILE defined, its the responsibility of the plugin

data/lib/logstash/runner.rb

@@ -9,11 +9,13 @@ require "net/http"
 require "logstash/namespace"
 require "logstash-core/logstash-core"
 require "logstash/environment"
+require "logstash/modules/cli_parser"
 
 LogStash::Environment.load_locale!
 
 require "logstash/agent"
 require "logstash/config/defaults"
+require "logstash/config/modules_common"
 require "logstash/shutdown_watcher"
 require "logstash/patches/clamp"
 require "logstash/settings"
@@ -22,6 +24,17 @@ require "logstash/plugins/registry"
 
 java_import 'org.logstash.FileLockFactory'
 
+def register_local_modules(path)
+  modules_path = File.join(path, File::Separator, "modules")
+  Dir.foreach(modules_path) do |item|
+    # Ignore unix relative path ids
+    next if item == '.' or item == '..'
+    # Ignore non-directories
+    next if !File.directory?(File.join(modules_path, File::Separator, item))
+    LogStash::PLUGIN_REGISTRY.add(:modules, item, LogStash::Modules::Scaffold.new(item, File.join(modules_path, File::Separator, item, File::Separator, "configuration")))
+  end
+end
+
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
   # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
@@ -49,6 +62,17 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("config.string"),
     :attribute_name => "config.string"
 
+  # Module settings
+  option ["--modules"], "MODULES",
+    I18n.t("logstash.runner.flag.modules"),
+    :multivalued => true,
+    :attribute_name => "modules_list"
+
+  option ["-M", "--modules.variable"], "MODULES_VARIABLE",
+    I18n.t("logstash.runner.flag.modules_variable"),
+    :multivalued => true,
+    :attribute_name => "modules_variable_list"
+
   # Pipeline settings
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
@@ -190,7 +214,7 @@ class LogStash::Runner < Clamp::StrictCommand
     # We invoke post_process to apply extra logic to them.
     # The post_process callbacks have been added in environment.rb
     @settings.post_process
-    
+
     require "logstash/util"
     require "logstash/util/java_version"
     require "stud/task"
@@ -211,6 +235,9 @@ class LogStash::Runner < Clamp::StrictCommand
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
 
+    # Add local modules to the registry before everything else
+    register_local_modules(LogStash::Environment::LOGSTASH_HOME)
+
     # We configure the registry and load any plugin that can register hooks
     # with logstash, this need to be done before any operation.
     LogStash::PLUGIN_REGISTRY.setup!
@@ -230,6 +257,10 @@ class LogStash::Runner < Clamp::StrictCommand
       return 1
     end
 
+    module_parser = LogStash::Modules::CLIParser.new(@modules_list, @modules_variable_list)
+    # Now populate Setting for modules.list with our parsed array.
+    @settings.set("modules.cli", module_parser.output)
+
     LogStash::ShutdownWatcher.unsafe_shutdown = setting("pipeline.unsafe_shutdown")
 
     configure_plugin_paths(setting("path.plugins"))
@@ -243,6 +274,15 @@ class LogStash::Runner < Clamp::StrictCommand
 
     @settings.format_settings.each {|line| logger.debug(line) }
 
+    module_configs = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
+    module_config_hash = module_configs.first
+    if !module_config_hash.nil?
+      @settings.set_value("config.string", module_config_hash["config_string"])
+    end
+    if module_configs.size > 1
+      logger.warn "Multiple modules defined in logstash.yml - using the first one: #{module_config_hash["pipeline_id"]}"
+    end
+
     if setting("config.string").nil? && setting("path.config").nil?
       fail(I18n.t("logstash.runner.missing-configuration"))
     end
@@ -434,7 +474,7 @@ class LogStash::Runner < Clamp::StrictCommand
       nil
     end
   end
-  
+
   # is the user asking for CLI help subcommand?
   def cli_help?(args)
     # I know, double negative

data/lib/logstash/settings.rb

@@ -2,11 +2,14 @@
 require "logstash/util/loggable"
 require "fileutils"
 require "logstash/util/byte_value"
+require "logstash/util/environment_variables"
 require "logstash/util/time_value"
 
 module LogStash
   class Settings
 
+    include LogStash::Util::EnvironmentVariables
+    
     def initialize
       @settings = {}
       # Theses settings were loaded from the yaml file
@@ -107,7 +110,10 @@ module LogStash
 
     def from_yaml(yaml_path)
       settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
-      self.merge(flatten_hash(settings), true)
+      self.merge(
+        deep_replace(flatten_hash(settings)),
+        true
+      )
       self
     end
     

data/lib/logstash/timestamp.rb

@@ -13,6 +13,10 @@ module LogStash
       self.time <=> other.time
     end
 
+    def eql?(other)
+      self.== other
+    end
+
     # TODO (colin) implement in Java
     def +(other)
       self.time + other

data/lib/logstash/util/dead_letter_queue_manager.rb

@@ -0,0 +1,61 @@
+require 'logstash/environment'
+
+module LogStash; module Util
+  class PluginDeadLetterQueueWriter
+
+    attr_reader :plugin_id, :plugin_type, :inner_writer
+
+    def initialize(inner_writer, plugin_id, plugin_type)
+      @plugin_id = plugin_id
+      @plugin_type = plugin_type
+      @inner_writer = inner_writer
+    end
+
+    def write(logstash_event, reason)
+      if @inner_writer && @inner_writer.is_open
+        @inner_writer.writeEntry(logstash_event.to_java, @plugin_type, @plugin_id, reason)
+      end
+    end
+
+    def close
+      if @inner_writer && @inner_writer.is_open
+        @inner_writer.close
+      end
+    end
+  end
+
+  class DummyDeadLetterQueueWriter
+    # class uses to represent a writer when dead_letter_queue is disabled
+    def initialize
+    end
+
+    def write(logstash_event, reason)
+      # noop
+    end
+
+    def is_open
+      false
+    end
+
+    def close
+      # noop
+    end
+  end
+
+  class DeadLetterQueueFactory
+    java_import org.logstash.common.DeadLetterQueueFactory
+
+    def self.get(pipeline_id)
+      if LogStash::SETTINGS.get("dead_letter_queue.enable")
+        return DeadLetterQueueWriter.new(
+          DeadLetterQueueFactory.getWriter(pipeline_id, LogStash::SETTINGS.get("path.dead_letter_queue")))
+      else
+        return DeadLetterQueueWriter.new(nil)
+      end
+    end
+
+    def self.close(pipeline_id)
+      DeadLetterQueueFactory.close(pipeline_id)
+    end
+  end
+end end

data/lib/logstash/util/safe_uri.rb

@@ -11,20 +11,21 @@ class LogStash::Util::SafeURI
   
   extend Forwardable
   
-  def_delegators :@uri, :coerce, :query=, :route_from, :port=, :default_port, :select, :normalize!, :absolute?, :registry=, :path, :password, :hostname, :merge, :normalize, :host, :component_ary, :userinfo=, :query, :set_opaque, :+, :merge!, :-, :password=, :parser, :port, :set_host, :set_path, :opaque=, :scheme, :fragment=, :set_query, :set_fragment, :userinfo, :hostname=, :set_port, :path=, :registry, :opaque, :route_to, :set_password, :hierarchical?, :set_user, :set_registry, :set_userinfo, :fragment, :component, :user=, :set_scheme, :absolute, :host=, :relative?, :scheme=, :user
   
   attr_reader :uri
-  
+
   public
   def initialize(arg)    
     @uri = case arg
            when String
              arg = "//#{arg}" if HOSTNAME_PORT_REGEX.match(arg)
-             URI.parse(arg)
-           when URI
+             java.net.URI.new(arg)
+           when java.net.URI
              arg
+           when URI
+             java.net.URI.new(arg.to_s)
            else
-             raise ArgumentError, "Expected a string or URI, got a #{arg.class} creating a URL"
+             raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
            end
   end
 
@@ -37,11 +38,11 @@ class LogStash::Util::SafeURI
   end
 
   def sanitized
-    return uri unless uri.password # nothing to sanitize here!
+    return uri unless password # nothing to sanitize here!
     
-    safe = uri.clone
-    safe.password = PASS_PLACEHOLDER
-    safe
+    user_info = user ? "#{user}:#{PASS_PLACEHOLDER}" : nil
+
+    make_uri(scheme, user_info, host, port, path, query, fragment)
   end
 
   def ==(other)
@@ -49,8 +50,126 @@ class LogStash::Util::SafeURI
   end
 
   def clone
-    cloned_uri = uri.clone
-    self.class.new(cloned_uri)
+    # No need to clone the URI, in java its immutable
+    self.class.new(uri)
+  end
+
+  def update(field, value)
+    new_scheme = scheme
+    new_user = user
+    new_password = password
+    new_host = host
+    new_port = port
+    new_path = path
+    new_query = query
+    new_fragment = fragment
+
+    case field 
+    when :scheme
+      new_scheme = value
+    when :user
+      new_user = value
+    when :password
+      new_password = value
+    when :host
+      new_host = value
+    when :port
+      new_port = value
+    when :path
+      new_path = value
+    when :query
+      new_query = value
+    when :fragment
+      new_fragment = value
+    end
+
+    user_info = new_user
+    if new_user && new_password
+      user_info += ":" + new_password
+    end
+
+    @uri = make_uri(new_scheme, user_info, new_host, new_port, new_path, new_query, new_fragment)
+  end
+
+  def user
+    if @uri.userInfo
+      @uri.userInfo.split(":")[0]
+    end
+  end
+
+  def user=(new_user)
+    update(:user, new_user)
+  end
+
+  def password
+    if @uri.userInfo
+      @uri.userInfo.split(":")[1]
+    end
+  end
+
+  def password=(new_password)
+    update(:password, new_password)
+  end
+
+  def hostname
+    # Alias from the ruby library
+    host
+  end
+
+  def host=(new_host)
+    update(:host, new_host)
+  end
+
+  def port
+    # In java this is an int
+    uri.port < 1 ? nil : uri.port
+  end
+ 
+  def port=(new_port)
+    update(:port, new_port)
+  end
+
+  def path=(new_path)
+    update(:path, new_path)
+  end
+
+  def query=(new_query)
+    update(:query, new_query)
+  end
+
+  def fragment=(new_fragment)
+    update(:fragment, new_fragment)
+  end
+
+  # Same algorithm as Ruby's URI class uses
+  def normalize!
+    if path && path == ''
+      path = '/'
+    end
+    if scheme && scheme != scheme.downcase
+      scheme = self.scheme.downcase
+    end
+    if host && host != host.downcase
+      host = self.host.downcase
+    end
+  end
+
+  def normalize
+    d = self.dup
+    d.normalize!
+    d
+  end
+
+  def_delegators :@uri, :absolute?, :scheme, :host, :path, :query, :fragment, :userinfo
+
+  private
+
+  # Jruby doesn't guess the constructor correctly if there are some nil things in place
+  # hence, this method
+  def make_uri(scheme, user_info, host, port, path, query, fragment)
+    # It is lot legal to have a path not starting with a /
+    prefixed_path = path && path[0] != "/" ? "/#{path}" : path
+    java.net.URI.new(scheme, user_info, host, port || -1, prefixed_path, query, fragment)
   end
 end