logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0eb6c2c950ece7dfa8bff8e470a006e6fe2fb18a
-  data.tar.gz: a9f8b462b9648e2b2e0740ee308cde7843e8ff0e
+  metadata.gz: 34e8ebcf1aeba4f255cd68db9e23c2c4513c9a61
+  data.tar.gz: c523eff7c037d296a719a4dc3330b8b4e0eb0160
 SHA512:
-  metadata.gz: 63aad0c89e475690b429cb582fab138fc5d86680cae1279379a044b195465a01a7ae8bef45f84523d4939647462294bc7eb326596b7723c325805903e60f2387
-  data.tar.gz: cc29c516d56a0d435262a7989480eca295f9f787aa19941723ede215a13f231108feb44f734d6c54957e39362772a7db825be79753b952ccafa6affba42f8917
+  metadata.gz: ab44fd374d1044c5904174280f2e8a3dea7801e3874167c4ea28ca896ed08273a33a981ad0442bd525669391348d913a1ac491018187c0786bc6f380b1a507d3
+  data.tar.gz: f8e2b3d8d4a1a0906c7a9113c30ee3629d75334e409a337337907432c12f6422e24f63c7fabfc1ea1fee037bbef5ef3397ed4c0006fc858633985f62db3e296d

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha2.snapshot2"
+LOGSTASH_CORE_VERSION = "5.0.0-alpha3"

data/lib/logstash/agent.rb

@@ -20,7 +20,8 @@ LogStash::Environment.load_locale!
 class LogStash::Agent
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :node_name, :pipelines, :logger
+  attr_reader :metric, :node_name, :pipelines, :settings
+  attr_accessor :logger
 
   # initialize method for LogStash::Agent
   # @param params [Hash] potential parameters are:
@@ -28,21 +29,23 @@ class LogStash::Agent
   #   :auto_reload [Boolean] - enable reloading of pipelines
   #   :reload_interval [Integer] - reload pipelines every X seconds
   #   :logger [Cabin::Channel] - logger instance
-  def initialize(params)
-    @logger = params[:logger]
-    @auto_reload = params[:auto_reload]
+  def initialize(settings = LogStash::SETTINGS)
+    @settings = settings
+    @logger = Cabin::Channel.get(LogStash)
+    @auto_reload = setting("config.reload.automatic")
 
     @pipelines = {}
-    @node_name = params[:node_name] || Socket.gethostname
-    @web_api_http_host = params[:web_api_http_host]
-    @web_api_http_port = params[:web_api_http_port]
+    @node_name = setting("node.name")
+    @http_host = setting("http.host")
+    @http_port = setting("http.port")
 
-    @config_loader = LogStash::Config::Loader.new(@logger, params[:debug_config])
-    @reload_interval = params[:reload_interval] || 3 # seconds
+    @config_loader = LogStash::Config::Loader.new(@logger)
+    @reload_interval = setting("config.reload.interval")
     @upgrade_mutex = Mutex.new
 
-    @collect_metric = params.fetch(:collect_metric, false)
-    setup_metric_collection
+    @collect_metric = setting("metric.collect")
+    @metric = create_metric_collector
+    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(create_metric_collector)
   end
 
   def execute
@@ -74,8 +77,11 @@ class LogStash::Agent
   # @param pipeline_id [String] pipeline string identifier
   # @param settings [Hash] settings that will be passed when creating the pipeline.
   #   keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
-  def register_pipeline(pipeline_id, settings)
-    pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id, :metric => metric))
+  def register_pipeline(pipeline_id, settings = @settings)
+    pipeline_settings = settings.clone
+    pipeline_settings.set("pipeline.id", pipeline_id)
+
+    pipeline = create_pipeline(pipeline_settings)
     return unless pipeline.is_a?(LogStash::Pipeline)
     if @auto_reload && pipeline.non_reloadable_plugins.any?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
@@ -123,7 +129,7 @@ class LogStash::Agent
 
   private
   def start_webserver
-    options = {:http_host => @web_api_http_host, :http_port => @web_api_http_port }
+    options = {:http_host => @http_host, :http_port => @http_port }
     @webserver = LogStash::WebServer.new(@logger, options)
     Thread.new(@webserver) do |webserver|
       LogStash::Util.set_thread_name("Api Webserver")
@@ -149,28 +155,29 @@ class LogStash::Agent
     end
   end
 
-  def setup_metric_collection
+  def create_metric_collector
     if collect_metrics?
       @logger.debug("Agent: Configuring metric collection")
       LogStash::Instrument::Collector.instance.agent = self
-      @metric = LogStash::Instrument::Metric.new
+      LogStash::Instrument::Metric.new
     else
-      @metric = LogStash::Instrument::NullMetric.new
+      LogStash::Instrument::NullMetric.new
     end
-
-    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
   end
 
   def collect_metrics?
     @collect_metric
   end
 
-  def create_pipeline(settings)
-    begin
-      config = fetch_config(settings)
-    rescue => e
-      @logger.error("failed to fetch pipeline configuration", :message => e.message)
-      return
+  def create_pipeline(settings, config=nil)
+
+    if config.nil?
+      begin
+        config = fetch_config(settings)
+      rescue => e
+        @logger.error("failed to fetch pipeline configuration", :message => e.message)
+        return
+      end
     end
 
     begin
@@ -182,23 +189,29 @@ class LogStash::Agent
   end
 
   def fetch_config(settings)
-    @config_loader.format_config(settings[:config_path], settings[:config_string])
+    @config_loader.format_config(settings.get("path.config"), settings.get("config.string"))
   end
 
   # since this method modifies the @pipelines hash it is
   # wrapped in @upgrade_mutex in the parent call `reload_state!`
   def reload_pipeline!(id)
     old_pipeline = @pipelines[id]
-    new_pipeline = create_pipeline(old_pipeline.original_settings)
+    new_config = fetch_config(old_pipeline.settings)
+    if old_pipeline.config_str == new_config
+      @logger.debug("no configuration change for pipeline",
+                    :pipeline => id, :config => new_config)
+      return
+    end
+
+    new_pipeline = create_pipeline(old_pipeline.settings, new_config)
+
     return if new_pipeline.nil?
 
-    if old_pipeline.config_str == new_pipeline.config_str
-      @logger.debug("no configuration change for pipeline",
-                    :pipeline => id, :config => old_pipeline.config_str)
-    elsif new_pipeline.non_reloadable_plugins.any?
+    if new_pipeline.non_reloadable_plugins.any?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
                     :pipeline_id => id,
                     :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
+      return
     else
       @logger.warn("fetched new config for pipeline. upgrading..",
                    :pipeline => id, :config => new_pipeline.config_str)
@@ -262,4 +275,8 @@ class LogStash::Agent
   def reset_collector
     LogStash::Instrument::Collector.instance.clear
   end
+
+  def setting(key)
+    @settings.get(key)
+  end
 end # class LogStash::Agent

data/lib/logstash/api/command_factory.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require "logstash/api/service"
+require "logstash/api/commands/system/basicinfo_command"
+require "logstash/api/commands/system/plugins_command"
+require "logstash/api/commands/stats"
+
+
+module LogStash
+  module Api
+    class CommandFactory
+      attr_reader :factory, :service
+
+      def initialize(service)
+        @service = service
+        @factory = {
+          :system_basic_info => ::LogStash::Api::Commands::System::BasicInfo,
+          :plugins_command => ::LogStash::Api::Commands::System::Plugins,
+          :stats => ::LogStash::Api::Commands::Stats
+        }
+      end
+
+      def build(*klass_path)
+        # Get a nested path with args like (:parent, :child)
+        klass = klass_path.reduce(factory) {|acc,v| acc[v]}
+
+        if klass
+          klass.new(service)
+        else
+          raise ArgumentError, "Class path '#{klass_path}' does not map to command!"
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/base.rb

@@ -0,0 +1,25 @@
+module LogStash
+  module Api
+    module Commands
+      class Base
+        attr_reader :service
+        
+        def initialize(service = LogStash::Api::Service.instance)
+          @service = service
+        end
+
+        def hostname
+          service.agent.node_name
+        end
+
+        def uptime
+          service.agent.uptime
+        end
+
+        def started_at
+          (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/stats.rb

@@ -0,0 +1,105 @@
+require "logstash/api/commands/base"
+require 'logstash/util/thread_dump'
+
+module LogStash
+  module Api
+    module Commands
+      class Stats < Commands::Base
+
+        def jvm
+          {:threads => service.get_shallow(:jvm, :threads)}
+        end
+
+        def process
+          service.get_shallow(:jvm, :process)
+        end
+
+        def events
+          service.get_shallow(:stats, :events)
+        end
+
+        def memory
+          memory = LogStash::Json.load(service.get(:jvm_memory_stats))
+          {
+            :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
+            :heap_used_percent => memory["heap"]["used_percent"],
+            :heap_committed_in_bytes => memory["heap"]["committed_in_bytes"],
+            :heap_max_in_bytes => memory["heap"]["max_in_bytes"],
+            :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
+            :non_heap_used_in_bytes => memory["non_heap"]["used_in_bytes"],
+            :non_heap_committed_in_bytes => memory["non_heap"]["committed_in_bytes"],
+            :pools => memory["pools"].inject({}) do |acc, (type, hash)|
+              hash.delete("committed_in_bytes")
+              acc[type] = hash
+              acc
+            end
+          }
+        end
+
+        def hot_threads(options={})
+          HotThreadsReport.new(self, options)
+        end
+
+        class HotThreadsReport
+          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
+          
+          def initialize(cmd, options)
+            @cmd = cmd
+            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
+            jr_dump = JRMonitor.threads.generate(filter)
+            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+          end
+          
+          def to_s
+            hash = to_hash
+            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
+            report << '=' * 80
+            report << "\n"
+            hash[:threads].each do |thread|
+              thread_report = ""
+              thread_report = "#{I18n.t("logstash.web_api.
+                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+              thread_report << "#{thread[:path]}\n" if thread[:path]
+              thread[:traces].each do |trace|
+                thread_report << "\t#{trace}\n"
+              end
+              report << thread_report
+              report << '-' * 80
+              report << "\n"
+            end
+            report
+          end
+
+          def to_hash
+            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
+            @thread_dump.each do |thread_name, _hash|
+              thread_name, thread_path = _hash["thread.name"].split(": ")
+              thread = { :name => thread_name,
+                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
+                         :state => _hash["thread.state"]
+                       }
+              thread[:path] = thread_path if thread_path
+              traces = []
+              _hash["thread.stacktrace"].each do |trace|
+                traces << trace
+              end
+              thread[:traces] = traces unless traces.empty?
+              hash[:threads] << thread
+            end
+            hash
+          end
+
+          def cpu_time_as_percent(hash)
+            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
+          end
+
+          def cpu_time(hash)
+            hash["cpu.time"] / 1000000.0
+          end
+
+        end       
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/system/basicinfo_command.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require 'logstash/api/commands/base'
+require "logstash/util/duration_formatter"
+
+module LogStash
+  module Api
+    module Commands
+      module System
+        class BasicInfo < Commands::Base
+
+          def run
+            {
+              "hostname" => hostname,
+              "version" => {
+                "number" => LOGSTASH_VERSION
+              }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/system/plugins_command.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/api/commands/base"
+
+module LogStash
+  module Api
+    module Commands
+      module System
+        class Plugins < Commands::Base
+          def run
+            { :total => plugins.count, :plugins => plugins }
+          end
+
+          private
+
+          def plugins
+            @plugins ||= find_plugins_gem_specs.map do |spec|
+              { :name => spec.name, :version => spec.version.to_s }
+            end.sort_by do |spec|
+              spec[:name]
+            end
+          end
+
+          def find_plugins_gem_specs
+            @specs ||= ::Gem::Specification.find_all.select{|spec| logstash_plugin_gem_spec?(spec)}
+          end
+
+          def logstash_plugin_gem_spec?(spec)
+            spec.metadata && spec.metadata["logstash_plugin"] == "true"
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/base.rb

@@ -0,0 +1,43 @@
+require "logstash/api/app_helpers"
+require "logstash/api/command_factory"
+
+module LogStash
+  module Api
+    module Modules
+      class Base < ::Sinatra::Base
+        helpers AppHelpers
+
+        set :environment, :production
+
+        attr_reader :factory
+
+        if settings.environment != :production
+          set :raise_errors, true
+          set :show_exceptions, :after_handler
+        end
+
+        include LogStash::Util::Loggable
+
+        helpers AppHelpers
+
+        def initialize(app=nil)
+          super(app)
+          @factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.instance)
+        end
+
+        not_found do
+          status 404
+          as   = params.has_key?("human") ? :string : :json
+          text = as == :string ? "" : {}
+          respond_with(text, :as => as)
+        end
+
+        error do
+          e = env['sinatra.error']
+          logger.error(e.message, :url => request.url, :ip => request.ip, :params => request.params, :class => e.class.name, :backtrace => e.backtrace)
+        end
+
+      end
+    end
+  end
+end