logstash-core 5.5.3-java → 5.6.0-java

data/lib/logstash/modules/kibana_dashboards.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require_relative "kibana_base"
+
+module LogStash module Modules class KibanaDashboards < KibanaBase
+  include LogStash::Util::Loggable
+
+  attr_reader :import_path, :content
+
+  # content is a list of kibana file resources
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    # e.g. curl "http://localhost:5601/api/kibana/dashboards/import"
+    # extract and prepare all objects
+    objects = []
+    content.each do |resource|
+      hash = {
+        "id" => resource.content_id,
+        "type" => resource.content_type,
+        "version" => 1,
+        "attributes" => resource.content_as_object
+      }
+      objects << hash
+    end
+    body = {:version => client.version, :objects => objects}
+    response = client.post(import_path, body)
+    if response.failed?
+      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
+    end
+    response
+  end
+end end end

data/lib/logstash/modules/kibana_importer.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+module LogStash module Modules class KibanaImporter
+  include LogStash::Util::Loggable
+
+  def initialize(client)
+    @client = client
+  end
+
+  def put(via_kibana)
+    path = via_kibana.import_path
+    logger.debug("Attempting POST", :url_path => path, :content => via_kibana.content)
+    via_kibana.import(@client)
+  end
+end end end

data/lib/logstash/modules/kibana_settings.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require_relative "kibana_base"
+
+module LogStash module Modules class KibanaSettings < KibanaBase
+  include LogStash::Util::Loggable
+
+  class Setting
+    attr_reader :name, :value
+    def initialize(name, value)
+      @name, @value = name, value
+    end
+  end
+
+  attr_reader :import_path, :content
+
+  # content is an array of Setting required for this module
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    # e.g. curl "http://localhost:5601/api/kibana/settings"
+    # 6.0.0-alpha3 -> {"settings":{"buildNum":{"userValue":15613},"defaultIndex":{"userValue":"arcsight-*"}}}
+    # 5.4 -> {"settings":{"defaultIndex":{"userValue":"cef-*"},"metrics:max_buckets":{"userValue":"600000"}}}
+    # array of Setting objects
+    # The POST api body { "changes": { "defaultIndex": "arcsight-*", "metrics:max_buckets": "400" } }
+    settings = {}
+    content.each do |setting|
+      settings[setting.name] = "#{setting.value}"
+    end
+    body = {"changes" => settings}
+    response = client.post(import_path, body)
+    if response.failed?
+      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
+    end
+    response
+  end
+end end end

data/lib/logstash/modules/logstash_config.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
 require "logstash/namespace"
 require_relative "file_reader"
+require "logstash/settings"
 
 module LogStash module Modules class LogStashConfig
-
   # We name it `modul` here because `module` has meaning in Ruby.
   def initialize(modul, settings)
     @directory = ::File.join(modul.directory, "logstash")
@@ -15,27 +15,72 @@ module LogStash module Modules class LogStashConfig
     ::File.join(@directory, "#{@name}.conf.erb")
   end
 
-  def setting(value, default)
-    @settings.fetch(value, default)
+  def configured_inputs(default = [], aliases = {})
+    name = "var.inputs"
+    values = get_setting(LogStash::Setting::SplittableStringArray.new(name, String, default))
+
+    aliases.each { |k,v| values << v if values.include?(k) }
+    aliases.invert.each { |k,v| values << v if values.include?(k) }
+    values.flatten.uniq
+  end
+
+  def alias_settings_keys!(aliases)
+    aliased_settings = alias_matching_keys(aliases, @settings)
+    @settings = alias_matching_keys(aliases.invert, aliased_settings)
+  end
+
+  def array_to_string(array)
+    "[#{array.collect { |i| "'#{i}'" }.join(", ")}]"
+  end
+
+  def csv_string(array)
+    "'#{array.join(',')}'"
+  end
+
+  def get_setting(setting_class)
+    raw_value = @settings[setting_class.name]
+    # If we dont check for NIL, the Settings class will try to coerce the value
+    # and most of the it will fails when a NIL value is explicitely set.
+    # This will be fixed once we wrap the plugins settings into a Settings class
+    setting_class.set(raw_value) unless raw_value.nil?
+    setting_class.value
+  end
+
+  def setting(name, default)
+    # by default we use the more permissive setting which is a `NullableString`
+    # This is fine because the end format of the logstash configuration is a string representation
+    # of the pipeline. There is a good reason why I think we should use the settings classes, we
+    # can `preprocess` a template and generate a configuration from the defined settings
+    # validate the values and replace them in the template.
+    case default
+      when String
+        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+      when Numeric
+        get_setting(LogStash::Setting::Numeric.new(name, default))
+      else
+        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+      end
   end
 
   def elasticsearch_output_config(type_string = nil)
-    hosts = setting("var.output.elasticsearch.hosts", "localhost:9200").split(',').map do |s|
-      '"' + s.strip + '"'
-    end.join(',')
-    index = "#{@name}-#{setting("var.output.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"
-    password = "#{setting("var.output.elasticsearch.password", "changeme")}"
-    user = "#{setting("var.output.elasticsearch.user", "elastic")}"
-    document_type_line = type_string ? "document_type => #{type_string}" : ""
+    hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])))
+    index = "#{@name}-#{setting("var.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"
+    user = @settings["var.elasticsearch.username"]
+    password = @settings["var.elasticsearch.password"]
+    lines = ["hosts => #{hosts}", "index => \"#{index}\""]
+    lines.push(user ? "user => \"#{user}\"" : nil)
+    lines.push(password ? "password => \"#{password}\"" : nil)
+    lines.push(type_string ? "document_type => #{type_string}" : nil)
+    lines.push("ssl => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
+    if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
+      lines.push("cacert => \"#{cacert}\"") if cacert
+    end
+    # NOTE: the first line should be indented in the conf.erb
     <<-CONF
 elasticsearch {
-hosts => [#{hosts}]
-index => "#{index}"
-password => "#{password}"
-user => "#{user}"
-manage_template => false
-#{document_type_line}
-}
+    #{lines.compact.join("\n    ")}
+    manage_template => false
+  }
 CONF
   end
 
@@ -45,4 +90,31 @@ CONF
     renderer = ERB.new(FileReader.read(template))
     renderer.result(binding)
   end
+
+  private
+  # For a first version we are copying the values of the original hash,
+  # this might become problematic if we users changes the values of the
+  # settings in the template, which could result in an inconsistent view of the original data
+  #
+  # For v1 of the feature I think its an OK compromise, v2 we have a more advanced hash that
+  # support alias.
+  def alias_matching_keys(aliases, target)
+    aliased_target = target.dup
+
+    aliases.each do |matching_key_prefix, new_key_prefix|
+      target.each do |k, v|
+        re = /^#{matching_key_prefix}\./
+
+        if k =~ re
+          alias_key = k.gsub(re, "#{new_key_prefix}.")
+
+          # If the user setup the same values twices with different values lets just halt.
+          raise "Cannot create an alias, the destination key has already a value set: original key: #{k}, alias key: #{alias_key}" if (!aliased_target[alias_key].nil? && aliased_target[alias_key] != v)
+          aliased_target[alias_key] = v unless v.nil?
+        end
+      end
+    end
+
+    aliased_target
+  end
 end end end

data/lib/logstash/modules/resource_base.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/namespace"
+require "logstash/json"
 require_relative "file_reader"
 
 module LogStash module Modules module ResourceBase
@@ -8,7 +9,11 @@ module LogStash module Modules module ResourceBase
   def initialize(base, content_type, content_path, content = nil, content_id = nil)
     @base, @content_type, @content_path = base, content_type, content_path
     @content_id =  content_id || ::File.basename(@content_path, ".*")
+    # content at this time will be a JSON string
     @content = content
+    if !@content.nil?
+      @content_as_object = LogStash::Json.load(@content) rescue {}
+    end
   end
 
   def content
@@ -19,12 +24,8 @@ module LogStash module Modules module ResourceBase
     "#{base}, #{content_type}, #{content_path}, #{content_id}"
   end
 
-  def contains?(text)
-    content.include?(text)
-  end
-
   def content_as_object
-    LogStash::Json.load(content) rescue nil
+    @content_as_object ||= FileReader.read_json(@content_path) rescue nil
   end
 
   def <=>(other)

data/lib/logstash/modules/scaffold.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/namespace"
 require "logstash/logging"
+require "logstash/util/loggable"
 require "erb"
 
 require_relative "elasticsearch_config"
@@ -10,19 +11,26 @@ require_relative "logstash_config"
 module LogStash module Modules class Scaffold
   include LogStash::Util::Loggable
 
-  attr_reader :directory, :module_name, :logstash_configuration, :kibana_configuration, :elasticsearch_configuration
+  attr_reader :directory, :module_name, :kibana_version_parts
+  attr_reader :kibana_configuration, :logstash_configuration, :elasticsearch_configuration
 
   def initialize(name, directory)
     @module_name = name
     @directory = directory  # this is the 'configuration folder in the GEM root.'
+    @kibana_version_parts = "5.6.0".split('.') # this is backup in case kibana client fails to connect
+    logger.info("Initializing module", :module_name => name, :directory => directory)
   end
 
-  def import(import_engine)
+  def add_kibana_version(version_parts)
+    @kibana_version_parts = version_parts
+  end
+
+  def import(import_engine, kibana_import_engine)
     @elasticsearch_configuration.resources.each do |resource|
       import_engine.put(resource)
     end
     @kibana_configuration.resources.each do |resource|
-      import_engine.put(resource)
+      kibana_import_engine.put(resource)
     end
   end
 

data/lib/logstash/modules/settings_merger.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash module Modules class SettingsMerger
+  def self.merge(cli_settings, yml_settings)
+    # both args are arrays of hashes, e.g.
+    # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
+    # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
+    merged = []
+    # union and group_by preserves order
+    # union will also coalesce identical hashes
+    union_of_settings = (cli_settings | yml_settings)
+    grouped_by_name = union_of_settings.group_by{|e| e["name"]}
+    grouped_by_name.each do |name, array|
+      if array.size == 2
+        merged << array.first.merge(array.last)
+      else
+        merged.concat(array)
+      end
+    end
+    merged
+  end
+end end end

data/lib/logstash/modules/util.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require_relative "scaffold"
+
+# This module function should be used when gems or
+# x-pack defines modules in their folder structures.
+module LogStash module Modules module Util
+  def self.register_local_modules(path)
+    modules_path = ::File.join(path, "modules")
+    ::Dir.foreach(modules_path) do |item|
+      # Ignore unix relative path ids
+      next if item == '.' or item == '..'
+      # Ignore non-directories
+      next if !::File.directory?(::File.join(modules_path, ::File::Separator, item))
+      LogStash::PLUGIN_REGISTRY.add(:modules, item, Scaffold.new(item, ::File.join(modules_path, item, "configuration")))
+    end
+  end
+end end end

data/lib/logstash/output_delegator.rb

@@ -19,7 +19,9 @@ module LogStash class OutputDelegator
     @namespaced_metric = metric.namespace(id.to_sym)
     @namespaced_metric.gauge(:name, config_name)
     @metric_events = @namespaced_metric.namespace(:events)
-
+    @in_counter = @metric_events.counter(:in)
+    @out_counter = @metric_events.counter(:out)
+    @time_metric = @metric_events.counter(:duration_in_millis)
     @strategy = strategy_registry.
                   class_for(self.concurrency).
                   new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
@@ -42,11 +44,11 @@ module LogStash class OutputDelegator
   end
 
   def multi_receive(events)
-    @metric_events.increment(:in, events.length)
-    clock = @metric_events.time(:duration_in_millis)
+    @in_counter.increment(events.length)
+    start_time = java.lang.System.current_time_millis
     @strategy.multi_receive(events)
-    clock.stop
-    @metric_events.increment(:out, events.length)
+    @time_metric.increment(java.lang.System.current_time_millis - start_time)
+    @out_counter.increment(events.length)
   end
 
   def do_close

data/lib/logstash/pipeline.rb

@@ -34,6 +34,7 @@ module LogStash; class BasePipeline
 
   def initialize(config_str, settings = SETTINGS)
     @logger = self.logger
+    @mutex = Mutex.new
     @config_str = config_str
     @config_hash = Digest::SHA1.hexdigest(@config_str)
     # Every time #plugin is invoked this is incremented to give each plugin
@@ -58,6 +59,8 @@ module LogStash; class BasePipeline
     parsed_config = grammar.parse(config_str)
     raise(ConfigurationError, grammar.failure_reason) if parsed_config.nil?
 
+    parsed_config.process_escape_sequences = settings.get_value("config.support_escapes")
+
     config_code = parsed_config.compile
 
     # config_code = BasePipeline.compileConfig(config_str)
@@ -122,6 +125,12 @@ module LogStash; class BasePipeline
   def non_reloadable_plugins
     (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
   end
+
+  private
+
+  def default_logging_keys(other_keys = {})
+    { :pipeline_id => pipeline_id }.merge(other_keys)
+  end
 end; end
 
 module LogStash; class Pipeline < BasePipeline
@@ -313,6 +322,9 @@ module LogStash; class Pipeline < BasePipeline
       config_metric.gauge(:batch_delay, batch_delay)
       config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
       config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
+      config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
+      config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
+
 
       @logger.info("Starting pipeline",
                    "id" => self.pipeline_id,
@@ -347,6 +359,10 @@ module LogStash; class Pipeline < BasePipeline
     end
   end
 
+  def dlq_enabled?
+    @settings.get("dead_letter_queue.enable")
+  end
+
   # Main body of what a worker thread does
   # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
@@ -419,7 +435,7 @@ module LogStash; class Pipeline < BasePipeline
     output_events_map.each do |output, events|
       output.multi_receive(events)
     end
-    
+
     @filter_queue_client.add_output_metrics(batch)
   end
 
@@ -599,6 +615,13 @@ module LogStash; class Pipeline < BasePipeline
       .each {|t| t.delete("status") }
   end
 
+  def collect_dlq_stats
+    if dlq_enabled?
+      dlq_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :dlq])
+      dlq_metric.gauge(:queue_size_in_bytes, @dlq_writer.get_current_queue_size)
+    end
+  end
+
   def collect_stats
     pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
     pipeline_metric.gauge(:type, settings.get("queue.type"))
@@ -639,11 +662,20 @@ module LogStash; class Pipeline < BasePipeline
 
   private
 
+  def default_logging_keys(other_keys = {})
+    keys = super
+    keys[:thread] = thread.inspect if thread
+    keys
+  end
+
   def draining_queue?
     @drain_queue ? !@filter_queue_client.empty? : false
   end
 
   def wrapped_write_client(plugin)
-    LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
+    #need to ensure that metrics are initialized one plugin at a time, else a race condition can exist.
+    @mutex.synchronize do
+      LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
+    end
   end
 end; end