logstash-core 5.5.3-java → 5.6.0-java

data/lib/logstash/instrument/namespaced_metric.rb

@@ -43,6 +43,10 @@ module LogStash module Instrument
     def collector
       @metric.collector
     end
+    
+    def counter(key)
+      collector.get(@namespace_name, key, :counter)
+    end
 
     def namespace(name)
       NamespacedMetric.new(metric, namespace_name + Array(name))

data/lib/logstash/instrument/namespaced_null_metric.rb

@@ -44,6 +44,10 @@ module LogStash module Instrument
       @metric.collector
     end
 
+    def counter(_)
+      ::LogStash::Instrument::NullMetric::NullGauge
+    end
+
     def namespace(name)
       NamespacedNullMetric.new(metric, namespace_name + Array(name))
     end

data/lib/logstash/instrument/null_metric.rb

@@ -39,6 +39,10 @@ module LogStash module Instrument
       end
     end
 
+    def counter(_)
+      NullGauge
+    end
+
     def namespace(name)
       raise MetricNoNamespaceProvided if name.nil? || name.empty?
       NamespacedNullMetric.new(self, name)
@@ -49,6 +53,12 @@ module LogStash module Instrument
     end
 
     private
+
+    class NullGauge
+      def self.increment(_)
+      end
+    end
+
     # Null implementation of the internal timer class
     #
     # @see LogStash::Instrument::TimedExecution`

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require 'logstash/instrument/periodic_poller/base'
+
+module LogStash module Instrument module PeriodicPoller
+  class DeadLetterQueue < Base
+    def initialize(metric, agent, options = {})
+      super(metric, options)
+      @metric = metric
+      @agent = agent
+    end
+
+    def collect
+      _, pipeline = @agent.running_pipelines.first
+      unless pipeline.nil?
+        pipeline.collect_dlq_stats
+      end
+    end
+  end
+end end end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash/instrument/periodic_poller/dlq"
 require "logstash/instrument/periodic_poller/os"
 require "logstash/instrument/periodic_poller/jvm"
 require "logstash/instrument/periodic_poller/pq"
@@ -14,7 +15,8 @@ module LogStash module Instrument
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
+                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
     end
 
     def start

data/lib/logstash/instrument/wrapped_write_client.rb

@@ -10,7 +10,12 @@ module LogStash module Instrument
       @events_metrics = metric.namespace([:stats, :events])
       @pipeline_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :events])
       @plugin_events_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :plugins, plugin_type, plugin.id.to_sym, :events])
-
+      @events_metrics_counter = @events_metrics.counter(:in)
+      @events_metrics_time = @events_metrics.counter(:queue_push_duration_in_millis)
+      @pipeline_metrics_counter = @pipeline_metrics.counter(:in)
+      @pipeline_metrics_time = @pipeline_metrics.counter(:queue_push_duration_in_millis)
+      @plugin_events_metrics_counter = @plugin_events_metrics.counter(:out)
+      @plugin_events_metrics_time = @plugin_events_metrics.counter(:queue_push_duration_in_millis)
       define_initial_metrics_values
     end
 
@@ -19,41 +24,45 @@ module LogStash module Instrument
     end
 
     def push(event)
-      record_metric { @write_client.push(event) }
+      increment_counters(1)
+      start_time = java.lang.System.current_time_millis
+      result = @write_client.push(event)
+      report_execution_time(start_time)
+      result
     end
+
     alias_method(:<<, :push)
 
     def push_batch(batch)
-      record_metric(batch.size) { @write_client.push_batch(batch) }
+      increment_counters(batch.size)
+      start_time = java.lang.System.current_time_millis
+      result = @write_client.push_batch(batch)
+      report_execution_time(start_time)
+      result
     end
 
     private
-    def record_metric(size = 1)
-      @events_metrics.increment(:in, size)
-      @pipeline_metrics.increment(:in, size)
-      @plugin_events_metrics.increment(:out, size)
-
-      clock = @events_metrics.time(:queue_push_duration_in_millis)
 
-      result = yield
-
-      # Reuse the same values for all the endpoints to make sure we don't have skew in times.
-      execution_time = clock.stop
-
-      @pipeline_metrics.report_time(:queue_push_duration_in_millis, execution_time)
-      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+    def increment_counters(size)
+      @events_metrics_counter.increment(size)
+      @pipeline_metrics_counter.increment(size)
+      @plugin_events_metrics_counter.increment(size)
+    end
 
-      result
+    def report_execution_time(start_time)
+      execution_time = java.lang.System.current_time_millis - start_time
+      @events_metrics_time.increment(execution_time)
+      @pipeline_metrics_time.increment(execution_time)
+      @plugin_events_metrics_time.increment(execution_time)
     end
 
     def define_initial_metrics_values
-      @events_metrics.increment(:in, 0)
-      @pipeline_metrics.increment(:in, 0)
-      @plugin_events_metrics.increment(:out, 0)
-
-      @events_metrics.report_time(:queue_push_duration_in_millis, 0)
-      @pipeline_metrics.report_time(:queue_push_duration_in_millis, 0)
-      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @events_metrics_counter.increment(0)
+      @pipeline_metrics_counter.increment(0)
+      @plugin_events_metrics_counter.increment(0)
+      @events_metrics_time.increment(0)
+      @pipeline_metrics_time.increment(0)
+      @plugin_events_metrics_time.increment(0)
     end
   end
 end end

data/lib/logstash/logging/logger.rb

@@ -8,10 +8,12 @@ module LogStash
     java_import org.apache.logging.log4j.core.config.Configurator
     java_import org.apache.logging.log4j.core.config.DefaultConfiguration
     java_import org.apache.logging.log4j.core.config.LoggerConfig
+    java_import org.logstash.log.LogstashLoggerContextFactory
+    java_import org.apache.logging.log4j.core.LoggerContext
+    java_import java.net.URI
 
     class Logger
       @@config_mutex = Mutex.new
-      @@logging_context = nil
 
       def initialize(name)
         @logger = LogManager.getLogger(name)
@@ -71,45 +73,50 @@ module LogStash
         raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
       end
 
-      def self.initialize(config_location)
+      def self.reconfigure(config_location)
         @@config_mutex.synchronize do
-          if @@logging_context.nil?
-            file_path = URI(config_location).path
-            if ::File.exists?(file_path)
-              logs_location = java.lang.System.getProperty("ls.logs")
-              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
-              @@logging_context = Configurator.initialize(nil, config_location)
-            else
-              # fall back to default config
-              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
-              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
-            end
+          config_location_uri = URI.create(config_location)
+          file_path = config_location_uri.path
+          if ::File.exists?(file_path)
+            logs_location = java.lang.System.getProperty("ls.logs")
+            puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+            #reconfigure the default context to use our log4j2.properties file
+            get_logging_context.setConfigLocation(URI.create(config_location))
+            #ensure everyone agrees which context to use for the LogManager
+            context_factory = LogstashLoggerContextFactory.new(get_logging_context)
+            LogManager.setFactory(context_factory)
+          else
+            # fall back to default config
+            puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs errors to the console"
           end
         end
       end
 
+      # until dev_utils/rspec/spec_helper is changed, we need to have both methods
+      singleton_class.send(:alias_method, :initialize, :reconfigure)
+
       def self.get_logging_context
-        return @@logging_context
+        return  LoggerContext.getContext(false)
       end
 
-      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but using initialized @@logging_context
+      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but ensure the proper context is used
       def self.set_level(_level, path)
-        configuration = @@logging_context.getConfiguration()
+        configuration =  get_logging_context.getConfiguration()
         level = Level.valueOf(_level)
         if path.nil? || path.strip.empty?
           root_logger = configuration.getRootLogger()
           if root_logger.getLevel() != level
             root_logger.setLevel(level)
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           end
         else
           package_logger = configuration.getLoggerConfig(path)
           if package_logger.name != path #no package logger found
             configuration.addLogger(path, LoggerConfig.new(path, level, true))
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           elsif package_logger.getLevel() != level
             package_logger.setLevel(level)
-            @@logging_context.updateLoggers()
+            get_logging_context.updateLoggers()
           end
         end
       end

data/lib/logstash/modules/{importer.rb → elasticsearch_importer.rb}

@@ -2,7 +2,7 @@
 require "logstash/namespace"
 require "logstash/logging"
 
-module LogStash module Modules class Importer
+module LogStash module Modules class ElasticsearchImporter
   include LogStash::Util::Loggable
 
   def initialize(client)
@@ -11,7 +11,7 @@ module LogStash module Modules class Importer
 
   def put(resource, overwrite = true)
     path = resource.import_path
-    logger.info("Attempting PUT", :url_path => path, :file_path => resource.content_path)
+    logger.debug("Attempting PUT", :url_path => path, :file_path => resource.content_path)
     if !overwrite && content_exists?(path)
       logger.debug("Found existing Elasticsearch resource.", :resource => path)
       return
@@ -31,7 +31,7 @@ module LogStash module Modules class Importer
 
   def content_exists?(path)
     response = @client.head(path)
-    response.status >= 200 && response.status <= 299
+    response.status >= 200 && response.status < 300
   end
 
 end end end # class LogStash::Modules::Importer

data/lib/logstash/modules/kibana_base.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/json"
+
+module LogStash module Modules class KibanaBase
+  attr_reader :import_path, :content
+
+  def initialize(import_path, content)
+    @import_path, @content = import_path, content
+  end
+
+  def import(client)
+    raise NotImplementedError, "#{self.class.name} needs to implement `#import`"
+  end
+
+  def to_s
+    import_path
+  end
+
+  def content_as_object
+    return content unless content.is_a?(String)
+    LogStash::Json.load(content) rescue nil
+  end
+end end end

data/lib/logstash/modules/kibana_client.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+require "manticore/client"
+
+module LogStash module Modules class KibanaClient
+  include LogStash::Util::Loggable
+
+  class Response
+    # to create a custom response with body as an Object (Hash or Array)
+    attr_reader :status, :body, :headers
+    def initialize(status, body, headers={})
+      @status, @body, @headers = status, body, headers
+      @body = body.is_a?(String) ? LogStash::Json.load(body) : body
+    end
+
+    def succeeded?
+      @status >= 200 && @status < 300
+    end
+
+    def failed?
+      !succeeded?
+    end
+  end
+
+  attr_reader :version
+
+  def initialize(settings)
+    @settings = settings
+
+    client_options = {
+      request_timeout: 5,
+      connect_timeout: 5,
+      socket_timeout: 5,
+      pool_max: 10,
+      pool_max_per_route: 2
+    }
+
+    ssl_options = {}
+
+    if @settings["var.kibana.ssl.enabled"] == "true"
+      ssl_options[:verify] = @settings.fetch("var.kibana.ssl.verification_mode", "strict").to_sym
+      ssl_options[:ca_file] = @settings.fetch("var.kibana.ssl.certificate_authority", nil)
+      ssl_options[:client_cert] = @settings.fetch("var.kibana.ssl.certificate", nil)
+      ssl_options[:client_key] = @settings.fetch("var.kibana.ssl.key", nil)
+    end
+
+    client_options[:ssl] = ssl_options
+
+    @client = Manticore::Client.new(client_options)
+    @host = @settings.fetch("var.kibana.host", "localhost:5601")
+    username = @settings["var.kibana.username"]
+    password = @settings["var.kibana.password"]
+
+    @scheme = @settings.fetch("var.kibana.scheme", "http")
+    @http_options = {:headers => {'Content-Type' => 'application/json'}}
+    if username
+      @http_options[:headers]['Authorization'] = 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp
+    end
+
+    # e.g. {"name":"Elastics-MacBook-Pro.local","version":{"number":"6.0.0-alpha3","build_hash":"41e69","build_number":15613,"build_snapshot":true}..}
+    @version = "0.0.0"
+    response = get("api/status")
+    if response.succeeded?
+      status = response.body
+      if status["version"].is_a?(Hash)
+        @version = status["version"]["number"]
+        if status["version"]["build_snapshot"]
+          @version.concat("-SNAPSHOT")
+        end
+      else
+        @version = status["version"]
+      end
+    end
+    @http_options[:headers]['kbn-version'] = @version
+  end
+
+  def version_parts
+    @version.split(/\.|\-/)
+  end
+
+  def host_settings
+    "[\"#{@host}\"]"
+  end
+
+  def get(relative_path)
+    # e.g. api/kibana/settings
+    safely(:get, relative_path, @http_options)
+  end
+
+  # content will be converted to a json string
+  def post(relative_path, content, headers = nil)
+
+    body = content.is_a?(String) ? content : LogStash::Json.dump(content)
+    options = {:body => body}.merge(headers || @http_options)
+    safely(:post, relative_path, options)
+  end
+
+  def head(relative_path)
+    safely(:head, relative_path, @http_options)
+  end
+
+  def can_connect?
+    head("api/status").succeeded?
+  end
+
+  private
+
+  def safely(method_sym, relative_path, options = {})
+    begin
+      resp = @client.http(method_sym, full_url(relative_path), options).call
+      Response.new(resp.code, resp.body, resp.headers)
+    rescue Manticore::ManticoreException => e
+      logger.error("Error when executing Kibana client request", :error => e)
+      body = {"statusCode" => 0, "error" => e.message}
+      Response.new(0, body, {})
+    end
+  end
+
+  def full_url(relative)
+    "#{@scheme}://#{@host}/#{relative}"
+  end
+end end end

data/lib/logstash/modules/kibana_config.rb

@@ -3,22 +3,26 @@ require "logstash/namespace"
 require "logstash/logging"
 
 require_relative "file_reader"
+require_relative "kibana_settings"
+require_relative "kibana_dashboards"
 require_relative "kibana_resource"
-require_relative "kibana_base_resource"
 
 module LogStash module Modules class KibanaConfig
   include LogStash::Util::Loggable
 
   ALLOWED_DIRECTORIES = ["search", "visualization"]
-
-  attr_reader :index_name
+  attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
 
   # We name it `modul` here because `module` has meaning in Ruby.
   def initialize(modul, settings)
     @directory = ::File.join(modul.directory, "kibana")
     @name = modul.module_name
     @settings = settings
-    @index_name = settings.fetch("dashboards.kibana_index", ".kibana")
+    @index_name = "kibana"
+    @pattern_name = "#{@name}-*"
+    @kibana_settings = [
+      KibanaSettings::Setting.new("defaultIndex", @pattern_name)
+    ]
   end
 
   def dashboards
@@ -30,21 +34,20 @@ module LogStash module Modules class KibanaConfig
   end
 
   def index_pattern
-    pattern_name = "#{@name}-*"
-    default_index_json = '{"defaultIndex": "#{pattern_name}"}'
-    default_index_content_id = @settings.fetch("index_pattern.kibana_version", "5.5.1")
-    [
-      KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"),nil, pattern_name),
-      KibanaResource.new(@index_name, "config", nil, default_index_json, default_index_content_id)
-    ]
+    [KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"),nil, @pattern_name)]
   end
 
   def resources
     list = index_pattern
     dashboards.each do |board|
+      list << board
       extract_panels_into(board, list)
     end
-    list.concat(extract_saved_searches(list))
+    list.concat(extract_saved_searches_into(list))
+    [
+      KibanaSettings.new("api/kibana/settings", @kibana_settings),
+      KibanaDashboards.new("api/kibana/dashboards/import", list)
+    ]
   end
 
   private
@@ -54,9 +57,7 @@ module LogStash module Modules class KibanaConfig
   end
 
   def extract_panels_into(dashboard, list)
-    list << dashboard
-
-    dash = FileReader.read_json(dashboard.content_path)
+    dash = dashboard.content_as_object
 
     if !dash.is_a?(Hash)
       logger.warn("Kibana dashboard JSON is not an Object", :module => @name)
@@ -85,20 +86,20 @@ module LogStash module Modules class KibanaConfig
         logger.warn("panelJSON contained unknown type", :type => panel_type)
       end
     end
+  end
 
-    def extract_saved_searches(list)
-      result = [] # must not add to list while iterating
-      list.each do |resource|
-        next unless resource.contains?("savedSearchId")
-        content = resource.content_as_object
-        next if content.nil?
-        saved_search = content["savedSearchId"]
-        next if saved_search.nil?
-        ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search))
-        next if list.member?(ss_resource) || result.member?(ss_resource)
-        result << ss_resource
-      end
-      result
+  def extract_saved_searches_into(list)
+    result = [] # must not add to list while iterating
+    list.each do |resource|
+      content = resource.content_as_object
+      next if content.nil?
+      next unless content.keys.include?("savedSearchId")
+      saved_search = content["savedSearchId"]
+      next if saved_search.nil?
+      ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search))
+      next if list.member?(ss_resource) || result.member?(ss_resource)
+      result << ss_resource
     end
+    result
   end
 end end end