logstash-core 5.0.0.alpha4.snapshot1-java → 5.0.0.alpha4.snapshot2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34ea2c9c79b6a5828714ad66dc7ce558d7952280
-  data.tar.gz: 76dcab330cb125653bfe94ef514f82f80a08a974
+  metadata.gz: bd0ebd412460fd45c6abdef1ae4a21b75f24b6c5
+  data.tar.gz: 4addbb53a17c98501b27f8f8fa33f8475a44bca4
 SHA512:
-  metadata.gz: 7274fedd8a22a9518fec19299c53295243832c17a376c85f59c2ea6dfe85c139d2161b0f2d5e6854e348824c38b3427fa60ea4d0e7281a6232bfee628848e45f
-  data.tar.gz: f1432ba6f7d34a38c3fbf57a7ddc6f02fefc6916f1349775a3254f75d720735101ad24da81971f81edbf42cd091c783959b2e46e2e2502162e0c2f10d39742ec
+  metadata.gz: b602623cc49f59c90e0af86c1d43e90e2b611aef12e44d5590a8536c3efd0cf06deab5644f49477fdcc2a4d7615340bef5e4fcac613ddfff9e6e792594511958
+  data.tar.gz: 7dcc49def008f07dae78a07e78c6ba8fa7e2942e30341637aa873068bf71cbc3ccb82557299b915e79934f523eb469d98a4dfaaeb39abcffe715efe66057bfe3

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot1"
+LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot2"

data/lib/logstash/agent.rb

@@ -45,15 +45,15 @@ class LogStash::Agent
     @upgrade_mutex = Mutex.new
 
     @collect_metric = setting("metric.collect")
-    @metric = create_metric_collector
-    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(create_metric_collector)
+
+    # Create the collectors and configured it with the library
+    configure_metrics_collectors
   end
 
   def execute
     @thread = Thread.current # this var is implicilty used by Stud.stop?
     @logger.info("starting agent")
 
-    start_background_services
     start_pipelines
     start_webserver
 
@@ -112,8 +112,13 @@ class LogStash::Agent
     ((Time.now.to_f - STARTED_AT.to_f) * 1000.0).to_i
   end
 
+  def stop_collecting_metrics
+    @collector.stop
+    @periodic_pollers.stop
+  end
+
   def shutdown
-    stop_background_services
+    stop_collecting_metrics
     stop_webserver
     shutdown_pipelines
   end
@@ -131,7 +136,7 @@ class LogStash::Agent
   private
   def start_webserver
     options = {:http_host => @http_host, :http_port => @http_port, :http_environment => @http_environment }
-    @webserver = LogStash::WebServer.new(@logger, options)
+    @webserver = LogStash::WebServer.new(@logger, self, options)
     Thread.new(@webserver) do |webserver|
       LogStash::Util.set_thread_name("Api Webserver")
       webserver.run
@@ -142,28 +147,24 @@ class LogStash::Agent
     @webserver.stop if @webserver
   end
 
-  def start_background_services
-    if collect_metrics?
-      @logger.debug("Agent: Starting metric periodic pollers")
-      @periodic_pollers.start
-    end
-  end
+  def configure_metrics_collectors
+    @collector = LogStash::Instrument::Collector.new
 
-  def stop_background_services
-    if collect_metrics?
-      @logger.debug("Agent: Stopping metric periodic pollers")
-      @periodic_pollers.stop
-    end
+    @metric = if collect_metrics?
+                @logger.debug("Agent: Configuring metric collection")
+                LogStash::Instrument::Metric.new(@collector)
+              else
+                LogStash::Instrument::NullMetric.new
+              end
+
+
+    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(@metric)
+    @periodic_pollers.start
   end
 
-  def create_metric_collector
-    if collect_metrics?
-      @logger.debug("Agent: Configuring metric collection")
-      LogStash::Instrument::Collector.instance.agent = self
-      LogStash::Instrument::Metric.new
-    else
-      LogStash::Instrument::NullMetric.new
-    end
+  def reset_metrics_collectors
+    stop_collecting_metrics
+    configure_metrics_collectors
   end
 
   def collect_metrics?
@@ -171,7 +172,6 @@ class LogStash::Agent
   end
 
   def create_pipeline(settings, config=nil)
-
     if config.nil?
       begin
         config = fetch_config(settings)
@@ -182,7 +182,7 @@ class LogStash::Agent
     end
 
     begin
-      LogStash::Pipeline.new(config, settings)
+      LogStash::Pipeline.new(config, settings, metric)
     rescue => e
       @logger.error("fetched an invalid config", :config => config, :reason => e.message)
       return
@@ -204,6 +204,11 @@ class LogStash::Agent
       return
     end
 
+    # Reset the current collected stats,
+    # starting a pipeline with a new configuration should be the same as restarting
+    # logstash.
+    reset_metrics_collectors
+
     new_pipeline = create_pipeline(old_pipeline.settings, new_config)
 
     return if new_pipeline.nil?
@@ -225,12 +230,6 @@ class LogStash::Agent
     return unless pipeline.is_a?(LogStash::Pipeline)
     return if pipeline.ready?
     @logger.info("starting pipeline", :id => id)
-
-    # Reset the current collected stats,
-    # starting a pipeline with a new configuration should be the same as restarting
-    # logstash.
-    reset_collector
-
     Thread.new do
       LogStash::Util.set_thread_name("pipeline.#{id}")
       begin
@@ -273,10 +272,6 @@ class LogStash::Agent
     @pipelines.empty?
   end
 
-  def reset_collector
-    LogStash::Instrument::Collector.instance.clear
-  end
-
   def setting(key)
     @settings.get(key)
   end

data/lib/logstash/api/command_factory.rb

@@ -3,6 +3,7 @@ require "logstash/api/service"
 require "logstash/api/commands/system/basicinfo_command"
 require "logstash/api/commands/system/plugins_command"
 require "logstash/api/commands/stats"
+require "logstash/api/commands/node"
 
 
 module LogStash
@@ -15,7 +16,8 @@ module LogStash
         @factory = {
           :system_basic_info => ::LogStash::Api::Commands::System::BasicInfo,
           :plugins_command => ::LogStash::Api::Commands::System::Plugins,
-          :stats => ::LogStash::Api::Commands::Stats
+          :stats => ::LogStash::Api::Commands::Stats,
+          :node => ::LogStash::Api::Commands::Node
         }
       end
 

data/lib/logstash/api/commands/base.rb

@@ -19,6 +19,10 @@ module LogStash
         def started_at
           (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
         end
+
+        def extract_metrics(path, *keys)
+          service.extract_metrics(path, *keys)
+        end
       end
     end
   end

data/lib/logstash/api/commands/node.rb

@@ -0,0 +1,116 @@
+require "logstash/api/commands/base"
+
+module LogStash
+  module Api
+    module Commands
+      class Node < Commands::Base
+        def all
+          {
+            :pipeline => pipeline,
+            :os => os,
+            :jvm => jvm
+          }           
+        end
+        
+        def pipeline
+          extract_metrics(
+            [:stats, :pipelines, :main, :config],
+            :workers, :batch_size, :batch_delay
+          )
+        end
+
+        def os
+          {
+            :name => java.lang.System.getProperty("os.name"),
+            :arch => java.lang.System.getProperty("os.arch"),
+            :version => java.lang.System.getProperty("os.version"),
+            :available_processors => java.lang.Runtime.getRuntime().availableProcessors()
+          }
+        end
+
+        def jvm
+          memory_bean = ManagementFactory.getMemoryMXBean()
+          {
+            :pid =>  ManagementFactory.getRuntimeMXBean().getName().split("@").first.to_i,
+            :version => java.lang.System.getProperty("java.version"),
+            :vm_name => java.lang.System.getProperty("java.vm.name"),
+            :vm_version => java.lang.System.getProperty("java.version"),
+            :vm_vendor => java.lang.System.getProperty("java.vendor"),
+            :vm_name => java.lang.System.getProperty("java.vm.name"),            
+            :start_time_in_millis => started_at,
+            :mem => {
+              :heap_init_in_bytes => (memory_bean.getHeapMemoryUsage().getInit() < 0 ? 0 : memory_bean.getHeapMemoryUsage().getInit()),
+              :heap_max_in_bytes => (memory_bean.getHeapMemoryUsage().getMax() < 0 ? 0 : memory_bean.getHeapMemoryUsage().getMax()),
+              :non_heap_init_in_bytes => (memory_bean.getNonHeapMemoryUsage().getInit() < 0 ? 0 : memory_bean.getNonHeapMemoryUsage().getInit()),
+              :non_heap_max_in_bytes => (memory_bean.getNonHeapMemoryUsage().getMax() < 0 ? 0 : memory_bean.getNonHeapMemoryUsage().getMax())
+            }
+          }
+        end
+
+        def hot_threads(options={})
+          HotThreadsReport.new(self, options)
+        end
+
+        class HotThreadsReport
+          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
+          
+          def initialize(cmd, options)
+            @cmd = cmd
+            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
+            jr_dump = JRMonitor.threads.generate(filter)
+            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+          end
+          
+          def to_s
+            hash = to_hash
+            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
+            report << '=' * 80
+            report << "\n"
+            hash[:threads].each do |thread|
+              thread_report = ""
+              thread_report = "#{I18n.t("logstash.web_api.
+                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+              thread_report << "#{thread[:path]}\n" if thread[:path]
+              thread[:traces].each do |trace|
+                thread_report << "\t#{trace}\n"
+              end
+              report << thread_report
+              report << '-' * 80
+              report << "\n"
+            end
+            report
+          end
+
+          def to_hash
+            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
+            @thread_dump.each do |thread_name, _hash|
+              thread_name, thread_path = _hash["thread.name"].split(": ")
+              thread = { :name => thread_name,
+                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
+                         :state => _hash["thread.state"]
+                       }
+              thread[:path] = thread_path if thread_path
+              traces = []
+              _hash["thread.stacktrace"].each do |trace|
+                traces << trace
+              end
+              thread[:traces] = traces unless traces.empty?
+              hash[:threads] << thread
+            end
+            hash
+          end
+
+          def cpu_time_as_percent(hash)
+            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
+          end
+
+          def cpu_time(hash)
+            hash["cpu.time"] / 1000000.0
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/stats.rb

@@ -5,99 +5,50 @@ module LogStash
   module Api
     module Commands
       class Stats < Commands::Base
-
         def jvm
-          {:threads => service.get_shallow(:jvm, :threads)}
+          {
+            :threads => extract_metrics(
+              [:jvm, :threads],
+              :count,
+              :peak_count
+            )
+          }
         end
 
         def process
-          service.get_shallow(:jvm, :process)
+          extract_metrics(
+            [:jvm, :process],
+            :open_file_descriptors,
+            :peak_open_file_descriptors,
+            :max_file_descriptors,
+            [:mem, [:total_virtual_in_bytes]],
+            [:cpu, [:total_in_millis, :percent]]
+          )
         end
 
         def events
-          service.get_shallow(:stats, :events)
+          extract_metrics(
+            [:stats, :events],
+            :in, :filtered, :out
+          )
         end
 
         def memory
-          memory = LogStash::Json.load(service.get(:jvm_memory_stats))
+          memory = service.get_shallow(:jvm, :memory)
           {
-            :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
-            :heap_used_percent => memory["heap"]["used_percent"],
-            :heap_committed_in_bytes => memory["heap"]["committed_in_bytes"],
-            :heap_max_in_bytes => memory["heap"]["max_in_bytes"],
-            :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
-            :non_heap_used_in_bytes => memory["non_heap"]["used_in_bytes"],
-            :non_heap_committed_in_bytes => memory["non_heap"]["committed_in_bytes"],
-            :pools => memory["pools"].inject({}) do |acc, (type, hash)|
+            :heap_used_in_bytes => memory[:heap][:used_in_bytes],
+            :heap_used_percent => memory[:heap][:used_percent],
+            :heap_committed_in_bytes => memory[:heap][:committed_in_bytes],
+            :heap_max_in_bytes => memory[:heap][:max_in_bytes],
+            :heap_used_in_bytes => memory[:heap][:used_in_bytes],
+            :non_heap_used_in_bytes => memory[:non_heap][:used_in_bytes],
+            :non_heap_committed_in_bytes => memory[:non_heap][:committed_in_bytes],
+            :pools => memory[:pools].inject({}) do |acc, (type, hash)|
               hash.delete("committed_in_bytes")
               acc[type] = hash
               acc
             end
           }
-        end
-
-        def hot_threads(options={})
-          HotThreadsReport.new(self, options)
-        end
-
-        class HotThreadsReport
-          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
-          
-          def initialize(cmd, options)
-            @cmd = cmd
-            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-            jr_dump = JRMonitor.threads.generate(filter)
-            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
-          end
-          
-          def to_s
-            hash = to_hash
-            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
-            report << '=' * 80
-            report << "\n"
-            hash[:threads].each do |thread|
-              thread_report = ""
-              thread_report = "#{I18n.t("logstash.web_api.
-                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
-              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
-              thread_report << "#{thread[:path]}\n" if thread[:path]
-              thread[:traces].each do |trace|
-                thread_report << "\t#{trace}\n"
-              end
-              report << thread_report
-              report << '-' * 80
-              report << "\n"
-            end
-            report
-          end
-
-          def to_hash
-            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
-            @thread_dump.each do |thread_name, _hash|
-              thread_name, thread_path = _hash["thread.name"].split(": ")
-              thread = { :name => thread_name,
-                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
-                         :state => _hash["thread.state"]
-                       }
-              thread[:path] = thread_path if thread_path
-              traces = []
-              _hash["thread.stacktrace"].each do |trace|
-                traces << trace
-              end
-              thread[:traces] = traces unless traces.empty?
-              hash[:threads] << thread
-            end
-            hash
-          end
-
-          def cpu_time_as_percent(hash)
-            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
-          end
-
-          def cpu_time(hash)
-            hash["cpu.time"] / 1000000.0
-          end
-
         end       
       end
     end

data/lib/logstash/api/modules/base.rb

@@ -21,9 +21,9 @@ module LogStash
 
         helpers AppHelpers
 
-        def initialize(app=nil)
+        def initialize(app=nil, agent)
           super(app)
-          @factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.instance)
+          @factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(agent))
         end
 
         not_found do

data/lib/logstash/api/modules/node.rb

@@ -5,7 +5,26 @@ module LogStash
   module Api
     module Modules
       class Node < ::LogStash::Api::Modules::Base
-        # return hot threads information
+        def node
+          factory.build(:node)
+        end
+        
+        get "/" do
+          respond_with node.all
+        end
+
+        get "/os" do
+          respond_with :os => node.os
+        end
+
+        get "/jvm" do
+          respond_with :jvm => node.jvm          
+        end
+
+        get "/pipeline" do
+          respond_with :pipeline => node.pipeline
+        end
+        
         get "/hot_threads" do
           ignore_idle_threads = params["ignore_idle_threads"] || true
 
@@ -15,11 +34,9 @@ module LogStash
           }
           options[:threads] = params["threads"].to_i if params.has_key?("threads")
 
-          stats = factory.build(:stats)
-          as    = options[:human] ? :string : :json
-          respond_with(stats.hot_threads(options), {:as => as})
-        end
-
+          as = options[:human] ? :string : :json
+          respond_with({:hot_threads => node.hot_threads(options)}, {:as => as})
+        end       
       end
     end
   end