logstash-core 6.0.0.beta1-java → 6.0.0.beta2-java

data/lib/logstash/modules/kibana_client.rb

@@ -50,12 +50,14 @@ module LogStash module Modules class KibanaClient
 
     @client = Manticore::Client.new(client_options)
     @host = @settings.fetch("var.kibana.host", "localhost:5601")
-    username = @settings["var.kibana.username"]
-    password = @settings["var.kibana.password"]
-
     @scheme = @settings.fetch("var.kibana.scheme", "http")
     @http_options = {:headers => {'Content-Type' => 'application/json'}}
+    username = @settings["var.kibana.username"]
     if username
+      password = @settings["var.kibana.password"]
+      if password.is_a?(LogStash::Util::Password)
+        password = password.value
+      end
       @http_options[:headers]['Authorization'] = 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp
     end
 

data/lib/logstash/modules/kibana_config.rb

@@ -11,7 +11,6 @@ module LogStash module Modules class KibanaConfig
   include LogStash::Util::Loggable
 
   ALLOWED_DIRECTORIES = ["search", "visualization"]
-  METRICS_MAX_BUCKETS = (24 * 60 * 60).freeze # 24 hours of events/sec buckets.
   attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
 
   # We name it `modul` here because `module` has meaning in Ruby.
@@ -21,10 +20,8 @@ module LogStash module Modules class KibanaConfig
     @settings = settings
     @index_name = "kibana"
     @pattern_name = "#{@name}-*"
-    @metrics_max_buckets = @settings.fetch("dashboards.metrics_max_buckets", METRICS_MAX_BUCKETS).to_i
     @kibana_settings = [
-      KibanaSettings::Setting.new("defaultIndex", @pattern_name),
-      KibanaSettings::Setting.new("metrics:max_buckets", @metrics_max_buckets)
+      KibanaSettings::Setting.new("defaultIndex", @pattern_name)
     ]
   end
 

data/lib/logstash/modules/scaffold.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/namespace"
 require "logstash/logging"
+require "logstash/util/loggable"
 require "erb"
 
 require_relative "elasticsearch_config"
@@ -17,6 +18,7 @@ module LogStash module Modules class Scaffold
     @module_name = name
     @directory = directory  # this is the 'configuration folder in the GEM root.'
     @kibana_version_parts = "6.0.0".split('.') # this is backup in case kibana client fails to connect
+    logger.info("Initializing module", :module_name => name, :directory => directory)
   end
 
   def add_kibana_version(version_parts)

data/lib/logstash/modules/settings_merger.rb

@@ -1,8 +1,13 @@
 # encoding: utf-8
 require "logstash/namespace"
+require "logstash/util"
+require "logstash/util/loggable"
 
-module LogStash module Modules class SettingsMerger
-  def self.merge(cli_settings, yml_settings)
+module LogStash module Modules module SettingsMerger
+  include LogStash::Util::Loggable
+  extend self
+
+  def merge(cli_settings, yml_settings)
     # both args are arrays of hashes, e.g.
     # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
     # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
@@ -11,13 +16,56 @@ module LogStash module Modules class SettingsMerger
     # union will also coalesce identical hashes
     union_of_settings = (cli_settings | yml_settings)
     grouped_by_name = union_of_settings.group_by{|e| e["name"]}
-    grouped_by_name.each do |name, array|
+    grouped_by_name.each do |_, array|
       if array.size == 2
-        merged << array.first.merge(array.last)
+        merged << array.last.merge(array.first)
       else
         merged.concat(array)
       end
     end
     merged
   end
+
+  def merge_cloud_settings(module_settings, logstash_settings)
+    cloud_id = logstash_settings.get("cloud.id")
+    cloud_auth = logstash_settings.get("cloud.auth")
+    if cloud_id.nil?
+      if cloud_auth.nil?
+        return # user did not specify cloud settings
+      else
+        raise ArgumentError.new("Cloud Auth without Cloud Id")
+      end
+    end
+    if logger.debug?
+      settings_copy = LogStash::Util.deep_clone(module_settings)
+    end
+
+    module_settings["var.kibana.scheme"] = "https"
+    module_settings["var.kibana.host"] = cloud_id.kibana_host
+    module_settings["var.elasticsearch.hosts"] = cloud_id.elasticsearch_host
+    unless cloud_auth.nil?
+      module_settings["var.elasticsearch.username"] = cloud_auth.username
+      module_settings["var.elasticsearch.password"] = cloud_auth.password
+      module_settings["var.kibana.username"] = cloud_auth.username
+      module_settings["var.kibana.password"] = cloud_auth.password
+    end
+    if logger.debug?
+      format_module_settings(settings_copy, module_settings).each {|line| logger.debug(line)}
+    end
+  end
+
+  def format_module_settings(settings_before, settings_after)
+    output = []
+    output << "-------- Module Settings ---------"
+    settings_after.each do |setting_name, setting|
+      setting_before = settings_before.fetch(setting_name, "")
+      line = "#{setting_name}: '#{setting}'"
+      if setting_before != setting
+        line.concat(", was: '#{setting_before}'")
+      end
+      output << line
+    end
+    output << "-------- Module Settings ---------"
+    output
+  end
 end end end

data/lib/logstash/output_delegator.rb

@@ -19,7 +19,9 @@ module LogStash class OutputDelegator
     @namespaced_metric = metric.namespace(id.to_sym)
     @namespaced_metric.gauge(:name, config_name)
     @metric_events = @namespaced_metric.namespace(:events)
-
+    @in_counter = @metric_events.counter(:in)
+    @out_counter = @metric_events.counter(:out)
+    @time_metric = @metric_events.counter(:duration_in_millis)
     @strategy = strategy_registry.
                   class_for(self.concurrency).
                   new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
@@ -42,11 +44,11 @@ module LogStash class OutputDelegator
   end
 
   def multi_receive(events)
-    @metric_events.increment(:in, events.length)
-    clock = @metric_events.time(:duration_in_millis)
+    @in_counter.increment(events.length)
+    start_time = java.lang.System.current_time_millis
     @strategy.multi_receive(events)
-    clock.stop
-    @metric_events.increment(:out, events.length)
+    @time_metric.increment(java.lang.System.current_time_millis - start_time)
+    @out_counter.increment(events.length)
   end
 
   def do_close

data/lib/logstash/pipeline.rb

@@ -38,7 +38,7 @@ module LogStash; class BasePipeline
 
   def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @logger = self.logger
-
+    @mutex = Mutex.new
     @ephemeral_id = SecureRandom.uuid
 
     @pipeline_config = pipeline_config
@@ -107,16 +107,27 @@ module LogStash; class BasePipeline
     LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
   end
 
-  def plugin(plugin_type, name, *args)
+  def plugin(plugin_type, name, line, column, *args)
     @plugin_counter += 1
 
     # Collapse the array of arguments into a single merged hash
     args = args.reduce({}, &:merge)
 
-    id = if args["id"].nil? || args["id"].empty?
-      args["id"] = "#{@config_hash}-#{@plugin_counter}"
+    if plugin_type == "codec"
+      id = SecureRandom.uuid # codecs don't really use their IDs for metrics, so we can use anything here
     else
-      args["id"]
+      # Pull the ID from LIR to keep IDs consistent between the two representations
+      id = lir.graph.vertices.filter do |v| 
+        v.source_with_metadata && 
+        v.source_with_metadata.line == line && 
+        v.source_with_metadata.column == column
+      end.findFirst.get.id
+    end
+
+    args["id"] = id # some code pulls the id out of the args
+
+    if !id
+      raise ConfigurationError, "Could not determine ID for #{plugin_type}/#{plugin_name}"
     end
 
     raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
@@ -231,6 +242,7 @@ module LogStash; class Pipeline < BasePipeline
     @running = Concurrent::AtomicBoolean.new(false)
     @flushing = Concurrent::AtomicReference.new(false)
     @force_shutdown = Concurrent::AtomicBoolean.new(false)
+    @outputs_registered = Concurrent::AtomicBoolean.new(false)
   end # def initialize
 
   def ready?
@@ -392,9 +404,9 @@ module LogStash; class Pipeline < BasePipeline
 
   def start_workers
     @worker_threads.clear # In case we're restarting the pipeline
+    @outputs_registered.make_false
     begin
-      register_plugins(@outputs)
-      register_plugins(@filters)
+      maybe_setup_out_plugins
 
       pipeline_workers = safe_pipeline_worker_count
       batch_size = @settings.get("pipeline.batch.size")
@@ -460,16 +472,17 @@ module LogStash; class Pipeline < BasePipeline
       shutdown_requested |= signal.shutdown? # latch on shutdown signal
 
       batch = @filter_queue_client.read_batch # metrics are started in read_batch
-      if (batch.size > 0)
+      if batch.size > 0
         @events_consumed.increment(batch.size)
         filter_batch(batch)
-        flush_filters_to_batch(batch, :final => false) if signal.flush?
+      end
+      flush_filters_to_batch(batch, :final => false) if signal.flush?
+      if batch.size > 0
         output_batch(batch)
         unless @force_shutdown.true? # ack the current batch
           @filter_queue_client.close_batch(batch)
         end
       end
-
       # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
       break if (shutdown_requested && !draining_queue?) || @force_shutdown.true?
     end
@@ -652,11 +665,11 @@ module LogStash; class Pipeline < BasePipeline
   # for backward compatibility in devutils for the rspec helpers, this method is not used
   # in the pipeline anymore.
   def filter(event, &block)
+    maybe_setup_out_plugins
     # filter_func returns all filtered events, including cancelled ones
-    filter_func(event).each { |e| block.call(e) }
+    filter_func(event).each {|e| block.call(e)}
   end
 
-
   # perform filters flush and yield flushed event to the passed block
   # @param options [Hash]
   # @option options [Boolean] :final => true to signal a final shutdown flush
@@ -791,9 +804,16 @@ module LogStash; class Pipeline < BasePipeline
 
   private
 
+  def maybe_setup_out_plugins
+    if @outputs_registered.make_true
+      register_plugins(@outputs)
+      register_plugins(@filters)
+    end
+  end
+
   def default_logging_keys(other_keys = {})
     keys = super
-    keys[:thread] = thread.inspect if thread
+    keys[:thread] ||= thread.inspect if thread
     keys
   end
 
@@ -802,6 +822,9 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   def wrapped_write_client(plugin)
-    LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
+    #need to ensure that metrics are initialized one plugin at a time, else a race condition can exist.
+    @mutex.synchronize do
+      LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
+    end
   end
 end; end

data/lib/logstash/pipeline_settings.rb

@@ -12,8 +12,10 @@ module LogStash
       "config.reload.interval",
       "config.string",
       "dead_letter_queue.enable",
+      "dead_letter_queue.max_bytes",
       "metric.collect",
       "path.config",
+      "path.dead_letter_queue",
       "path.queue",
       "pipeline.batch.delay",
       "pipeline.batch.size",

data/lib/logstash/runner.rb

@@ -74,6 +74,19 @@ class LogStash::Runner < Clamp::StrictCommand
     :multivalued => true,
     :attribute_name => "modules_variable_list"
 
+  option ["--setup"], :flag,
+    I18n.t("logstash.runner.flag.modules_setup"),
+    :default => LogStash::SETTINGS.get_default("modules_setup"),
+    :attribute_name => "modules_setup"
+
+  option ["--cloud.id"], "CLOUD_ID",
+    I18n.t("logstash.runner.flag.cloud_id"),
+    :attribute_name => "cloud.id"
+
+  option ["--cloud.auth"], "CLOUD_AUTH",
+    I18n.t("logstash.runner.flag.cloud_auth"),
+    :attribute_name => "cloud.auth"
+
   # Pipeline settings
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
@@ -468,8 +481,7 @@ class LogStash::Runner < Clamp::StrictCommand
     Stud::trap("INT") do
       if @interrupted_once
         logger.fatal(I18n.t("logstash.agent.forced_sigint"))
-        @agent.force_shutdown!
-        exit
+        exit(1)
       else
         logger.warn(I18n.t("logstash.agent.sigint"))
         Thread.new(logger) {|lg| sleep 5; lg.warn(I18n.t("logstash.agent.slow_shutdown")) }

data/lib/logstash/settings.rb

@@ -255,6 +255,7 @@ module LogStash
           @default = default
         end
       end
+
       def set(value)
         coerced_value = coerce(value)
         validate(coerced_value)
@@ -557,7 +558,32 @@ module LogStash
         end
       end
     end
+
+    class Modules < Coercible
+      def initialize(name, klass, default = nil)
+        super(name, klass, default, false)
+      end
+
+      def set(value)
+        @value = coerce(value)
+        @value_is_set = true
+        @value
+      end
+
+      def coerce(value)
+        if value.is_a?(@klass)
+          return value
+        end
+        @klass.new(value)
+      end
+
+      protected
+      def validate(value)
+        coerce(value)
+      end
+    end
   end
 
+
   SETTINGS = Settings.new
 end

data/lib/logstash/util/cloud_setting_auth.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util/password"
+
+module LogStash module Util class CloudSettingAuth
+  attr_reader :original, :username, :password
+
+  def initialize(value)
+    return if value.nil?
+
+    unless value.is_a?(String)
+      raise ArgumentError.new("Cloud Auth must be String. Received: #{value.class}")
+    end
+    @original = value
+    @username, sep, password = @original.partition(":")
+    if @username.empty? || sep.empty? || password.empty?
+      raise ArgumentError.new("Cloud Auth username and password format should be \"<username>:<password>\".")
+    end
+    @password = LogStash::Util::Password.new(password)
+  end
+
+  def to_s
+    "#{@username}:#{@password}"
+  end
+
+  def inspect
+    to_s
+  end
+end end end

data/lib/logstash/util/cloud_setting_id.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "base64"
+
+module LogStash module Util class CloudSettingId
+  attr_reader :original, :decoded, :label, :elasticsearch_host, :kibana_host
+
+  def initialize(value)
+    return if value.nil?
+
+    unless value.is_a?(String)
+      raise ArgumentError.new("Cloud Id must be String. Received: #{value.class}")
+    end
+    @original = value
+    @label, sep, last = value.partition(":")
+    if last.empty?
+      @decoded = Base64.urlsafe_decode64(@label) rescue ""
+      @label = ""
+    else
+      @decoded = Base64.urlsafe_decode64(last) rescue ""
+    end
+    unless @decoded.count("$") == 2
+      raise ArgumentError.new("Cloud Id does not decode. Received: \"#{@original}\".")
+    end
+    parts = @decoded.split("$")
+    if parts.any?(&:empty?)
+      raise ArgumentError.new("Cloud Id, after decoding, is invalid. Format: '<part1>$<part2>$<part3>'. Received: \"#{@decoded}\".")
+    end
+    cloud_host, es_server, kb_server = parts
+    @elasticsearch_host = sprintf("%s.%s:443", es_server, cloud_host)
+    @kibana_host  = sprintf("%s.%s:443", kb_server, cloud_host)
+  end
+
+  def to_s
+    @original.to_s
+  end
+
+  def inspect
+    to_s
+  end
+end end end

data/lib/logstash/util/modules_setting_array.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util/password"
+
+module LogStash module Util class ModulesSettingArray
+  extend Forwardable
+  DELEGATED_METHODS = [].public_methods.reject{|symbol| symbol.to_s.end_with?('__')}
+
+  def_delegators :@original, *DELEGATED_METHODS
+
+  attr_reader :original
+  def initialize(value)
+    unless value.is_a?(Array)
+      raise ArgumentError.new("Module Settings must be an Array. Received: #{value.class}")
+    end
+    @original = value
+    # wrap passwords
+    @original.each do |hash|
+      hash.keys.select{|key| key.to_s.end_with?('password')}.each do |key|
+        hash[key] = LogStash::Util::Password.new(hash[key])
+      end
+    end
+  end
+
+  def __class__
+    LogStash::Util::ModulesSettingArray
+  end
+end end end