logstash-core 5.0.0.alpha6.snapshot5-java → 5.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb6a76091c290d8e2c2a2fb804473617e09125fb
-  data.tar.gz: 87918ac0fa2113c2e8e71020358089947dde54dc
+  metadata.gz: 580f202da437b439172ac7d75a2040fe78b5b2f5
+  data.tar.gz: 5a33bf37b1e0d52b16d71e996bae012f02cbe7d1
 SHA512:
-  metadata.gz: 5cba39c4ec6cba7b92981896a9130ce256bbb78a07b9ce8e332f3711756556ea010dab2933975970c1e141a0f942fa2c8f82de5af10126996835f51d7e47ab05
-  data.tar.gz: 0915d7af51c1430cdaa75f8b23d6f89ca41d05620b0cb8a51e0f50cec919df169f6d204004b0d8f54bf461fabd49edb0d2462fac61a592e517bc9c56fed90aaf
+  metadata.gz: 77708d863472390ae13396223803caa3aa95392a6755811e3d8b5e14085d84904712f97e2dc9911bfc675edfca9c67c0d28a57a2fc92449791d2cc5826d4eec0
+  data.tar.gz: ba5fa135b749b89e3f908b39d4381c45dcb004b3a1c8763cdf94339fc710e6bb3efc17a6c9ff65611cea88d2901d093986c0aaea45d59dd7f8c1b1bdc6b4947e

data/lib/jars.rb

@@ -4,4 +4,4 @@ require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
 require_jar('org.apache.logging.log4j', 'log4j-core', '2.6.2')
 require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.7.4')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.4')
-require_jar('org.logstash', 'logstash-core', '5.0.0-alpha6')
+require_jar('org.logstash', 'logstash-core', '5.0.0')

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0.alpha6.snapshot5"
+LOGSTASH_CORE_VERSION = "5.0.0"

data/lib/logstash/agent.rb

@@ -48,11 +48,13 @@ class LogStash::Agent
 
     # Create the collectors and configured it with the library
     configure_metrics_collectors
+
+    @reload_metric = metric.namespace([:stats, :pipelines])
   end
 
   def execute
     @thread = Thread.current # this var is implicilty used by Stud.stop?
-    @logger.info("starting agent")
+    @logger.debug("starting agent")
 
     start_pipelines
     start_webserver
@@ -99,6 +101,11 @@ class LogStash::Agent
         begin
           reload_pipeline!(pipeline_id)
         rescue => e
+          @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
+            n.increment(:failures)
+            n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
+            n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
+          end
           @logger.error(I18n.t("oops"), :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
         end
       end
@@ -162,9 +169,12 @@ class LogStash::Agent
     @periodic_pollers.start
   end
 
-  def reset_metrics_collectors
-    stop_collecting_metrics
-    configure_metrics_collectors
+  def reset_pipeline_metrics(id)
+    # selectively reset metrics we don't wish to keep after reloading
+    # these include metrics about the plugins and number of processed events
+    # we want to keep other metrics like reload counts and error messages
+    @collector.clear("stats/pipelines/#{id}/plugins")
+    @collector.clear("stats/pipelines/#{id}/events")
   end
 
   def collect_metrics?
@@ -184,6 +194,11 @@ class LogStash::Agent
     begin
       LogStash::Pipeline.new(config, settings, metric)
     rescue => e
+      @reload_metric.namespace([settings.get("pipeline.id").to_sym, :reloads]).tap do |n|
+        n.increment(:failures)
+        n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
+        n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
+      end
       if @logger.debug?
         @logger.error("fetched an invalid config", :config => config, :reason => e.message, :backtrace => e.backtrace)
       else
@@ -208,11 +223,6 @@ class LogStash::Agent
       return
     end
 
-    # Reset the current collected stats,
-    # starting a pipeline with a new configuration should be the same as restarting
-    # logstash.
-    reset_metrics_collectors
-
     new_pipeline = create_pipeline(old_pipeline.settings, new_config)
 
     return if new_pipeline.nil?
@@ -233,12 +243,17 @@ class LogStash::Agent
     pipeline = @pipelines[id]
     return unless pipeline.is_a?(LogStash::Pipeline)
     return if pipeline.ready?
-    @logger.info("starting pipeline", :id => id)
+    @logger.debug("starting pipeline", :id => id)
     Thread.new do
       LogStash::Util.set_thread_name("pipeline.#{id}")
       begin
         pipeline.run
       rescue => e
+        @reload_metric.namespace([id.to_sym, :reloads]) do |n|
+          n.increment(:failures)
+          n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
+          n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
+        end
         @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
       end
     end
@@ -254,7 +269,11 @@ class LogStash::Agent
   end
 
   def start_pipelines
-    @pipelines.each { |id, _| start_pipeline(id) }
+    @pipelines.each do |id, _|
+      start_pipeline(id)
+      # no reloads yet, initalize all the reload metrics
+      init_pipeline_reload_metrics(id)
+    end
   end
 
   def shutdown_pipelines
@@ -268,8 +287,13 @@ class LogStash::Agent
 
   def upgrade_pipeline(pipeline_id, new_pipeline)
     stop_pipeline(pipeline_id)
+    reset_pipeline_metrics(pipeline_id)
     @pipelines[pipeline_id] = new_pipeline
     start_pipeline(pipeline_id)
+    @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
+      n.increment(:successes)
+      n.gauge(:last_success_timestamp, LogStash::Timestamp.now)
+    end
   end
 
   def clean_state?
@@ -279,4 +303,14 @@ class LogStash::Agent
   def setting(key)
     @settings.get(key)
   end
+
+  def init_pipeline_reload_metrics(id)
+    @reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
+      n.increment(:successes, 0)
+      n.increment(:failures, 0)
+      n.gauge(:last_error, nil)
+      n.gauge(:last_success_timestamp, nil)
+      n.gauge(:last_failure_timestamp, nil)
+    end
+  end
 end # class LogStash::Agent

data/lib/logstash/api/app_helpers.rb

@@ -1,27 +1,43 @@
 # encoding: utf-8
 require "logstash/json"
+require "logstash/api/errors"
 
 module LogStash::Api::AppHelpers
-
+  # This method handle both of the normal flow *happy path*
+  # and the display or errors, if more custom logic is added here
+  # it will make sense to separate them.
+  #
+  # See `#error` method in the `LogStash::Api::Module::Base`
   def respond_with(data, options={})
     as     = options.fetch(:as, :json)
     filter = options.fetch(:filter, "")
-    pretty = params.has_key?("pretty")
+
+    status data.respond_to?(:status_code) ? data.status_code : 200
 
     if as == :json
-      selected_fields = extract_fields(filter.to_s.strip)
-      data.select! { |k,v| selected_fields.include?(k) } unless selected_fields.empty?
-      unless options.include?(:exclude_default_metadata)
-        data = default_metadata.merge(data)
+      if api_error?(data)
+        data = generate_error_hash(data)
+      else
+        selected_fields = extract_fields(filter.to_s.strip)
+        data.select! { |k,v| selected_fields.include?(k) } unless selected_fields.empty?
+        unless options.include?(:exclude_default_metadata)
+          data = data.to_hash
+          if data.values.size == 0 && selected_fields.size > 0
+            raise LogStash::Api::NotFoundError
+          end
+          data = default_metadata.merge(data)
+        end
       end
+
       content_type "application/json"
-      LogStash::Json.dump(data, {:pretty => pretty})
+      LogStash::Json.dump(data, {:pretty => pretty?})
     else
       content_type "text/plain"
       data.to_s
     end
   end
 
+  protected
   def extract_fields(filter_string)
     (filter_string.empty? ? [] : filter_string.split(",").map { |s| s.strip.to_sym })
   end
@@ -35,4 +51,24 @@ module LogStash::Api::AppHelpers
   def default_metadata
     @factory.build(:default_metadata).all
   end
+
+  def api_error?(error)
+    error.is_a?(LogStash::Api::ApiError)
+  end
+
+  def pretty?
+    params.has_key?("pretty")
+  end
+
+  def generate_error_hash(error)
+    {
+      :path => request.path,
+      :status => error.status_code,
+      :error => error.to_hash
+    }
+  end
+
+  def human?
+    params.has_key?("human") && (params["human"].nil? || as_boolean(params["human"]) == true)
+  end
 end

data/lib/logstash/api/commands/stats.rb

@@ -91,7 +91,8 @@ module LogStash
                 :inputs => plugin_stats(stats, :inputs),
                 :filters => plugin_stats(stats, :filters),
                 :outputs => plugin_stats(stats, :outputs)
-              }
+              },
+              :reloads => stats[:reloads],
             }
           end
         end # module PluginsStats

data/lib/logstash/api/errors.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    class ApiError < StandardError;
+      def initialize(message = nil)
+        super(message || "Api Error")
+      end
+
+      def status_code
+        500
+      end
+
+      def to_hash
+        { :message => to_s }
+      end
+    end
+
+    class NotFoundError < ApiError
+      def initialize
+        super("Not Found")
+      end
+
+      def status_code
+        404
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/base.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 require "logstash/api/app_helpers"
 require "logstash/api/command_factory"
+require "logstash/api/errors"
 
 module LogStash
   module Api
     module Modules
       class Base < ::Sinatra::Base
+
         helpers AppHelpers
 
         # These options never change
@@ -29,15 +31,15 @@ module LogStash
         end
 
         not_found do
-          status 404
-          as   = params.has_key?("human") ? :string : :json
-          text = as == :string ? "" : {}
-          respond_with(text, :as => as)
+          # We cannot raise here because it wont be catched by the `error` handler.
+          # So we manually create a new instance of NotFound and just pass it down.
+          respond_with(NotFoundError.new)
         end
 
-        protected
-        def human?
-          params.has_key?("human") && (params["human"].nil? || as_boolean(params["human"]) == true)
+        # This allow to have custom exception but keep a consistent
+        # format to report them.
+        error ApiError do |error|
+          respond_with(error)
         end
       end
     end

data/lib/logstash/api/modules/logging.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+#
+java_import org.apache.logging.log4j.core.LoggerContext
+
+module LogStash
+  module Api
+    module Modules
+      class Logging < ::LogStash::Api::Modules::Base
+        # retrieve logging specific parameters from the provided settings
+        #
+        # return any unused configurations
+        def handle_logging(settings)
+          Hash[settings.map do |key, level|
+            if key.start_with?("logger.")
+              _, path = key.split("logger.")
+              LogStash::Logging::Logger::configure_logging(level, path)
+              nil
+            else
+              [key, level]
+            end
+          end]
+        end
+
+        put "/" do
+          begin
+            request.body.rewind
+            req_body = LogStash::Json.load(request.body.read)
+            remaining = handle_logging(req_body)
+            unless remaining.empty?
+              raise ArgumentError, I18n.t("logstash.web_api.logging.unrecognized_option", :option => remaining.keys.first)
+            end
+            respond_with({"acknowledged" => true})
+          rescue ArgumentError => e
+            status 400
+            respond_with({"error" => e.message})
+          end
+        end
+
+        get "/" do
+          context = LogStash::Logging::Logger::get_logging_context
+          if context.nil?
+            status 500
+            respond_with({"error" => "Logstash loggers were not initialized properly"})
+          else
+            loggers = context.getLoggers.map { |lgr| [lgr.getName, lgr.getLevel.name] }.sort
+            respond_with({"loggers" => Hash[loggers]})
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/node.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/api/modules/base"
+require "logstash/api/errors"
 
 module LogStash
   module Api
@@ -12,20 +13,23 @@ module LogStash
         get "/hot_threads" do
           ignore_idle_threads = params["ignore_idle_threads"] || true
 
-          options = {
-            :ignore_idle_threads => as_boolean(ignore_idle_threads),
-            :human => human?
-          }
+          options = { :ignore_idle_threads => as_boolean(ignore_idle_threads) }
           options[:threads] = params["threads"].to_i if params.has_key?("threads")
 
-          as = options[:human] ? :string : :json
+          as = human? ? :string : :json
           respond_with(node.hot_threads(options), {:as => as})
         end
 
-        get "/?:filter?" do
-          selected_fields = extract_fields(params["filter"].to_s.strip)
-          respond_with node.all(selected_fields)
-        end
+         get "/?:filter?" do
+           selected_fields = extract_fields(params["filter"].to_s.strip)
+           values = node.all(selected_fields)
+
+           if values.size == 0
+             raise NotFoundError
+           else
+             respond_with(values)
+           end
+         end
       end
     end
   end

data/lib/logstash/api/modules/root.rb

@@ -3,12 +3,10 @@ module LogStash
   module Api
     module Modules
       class Root < ::LogStash::Api::Modules::Base
-        
         get "/" do
           command = factory.build(:system_basic_info)
           respond_with command.run
         end
-        
       end
     end
   end

data/lib/logstash/api/modules/stats.rb

@@ -3,7 +3,6 @@ module LogStash
   module Api
     module Modules
       class Stats < ::LogStash::Api::Modules::Base
-
         def stats_command
           factory.build(:stats)
         end
@@ -36,7 +35,6 @@ module LogStash
           }
           respond_with(payload, {:filter => params["filter"]})
         end
-
       end
     end
   end

data/lib/logstash/api/rack_app.rb

@@ -5,6 +5,7 @@ require "logstash/api/modules/node"
 require "logstash/api/modules/node_stats"
 require "logstash/api/modules/plugins"
 require "logstash/api/modules/root"
+require "logstash/api/modules/logging"
 require "logstash/api/modules/stats"
 
 module LogStash
@@ -31,9 +32,9 @@ module LogStash
           status, headers, body = res
 
           if fatal_error?(status)
-            @logger.warn? && @logger.warn(LOG_MESSAGE, RackApp.log_metadata(status, env))
+            @logger.error? && @logger.error(LOG_MESSAGE, RackApp.log_metadata(status, env))
           else
-            @logger.info? && @logger.info(LOG_MESSAGE, RackApp.log_metadata(status, env))
+            @logger.debug? && @logger.debug(LOG_MESSAGE, RackApp.log_metadata(status, env))
           end
 
           res
@@ -103,7 +104,8 @@ module LogStash
           "/_node" => LogStash::Api::Modules::Node,
           "/_stats" => LogStash::Api::Modules::Stats,
           "/_node/stats" => LogStash::Api::Modules::NodeStats,
-          "/_node/plugins" => LogStash::Api::Modules::Plugins
+          "/_node/plugins" => LogStash::Api::Modules::Plugins,
+          "/_node/logging" => LogStash::Api::Modules::Logging
         }
       end
     end