logstash-core 5.0.0.alpha3.snapshot5-java → 5.0.0.alpha3.snapshot6-java

data/lib/logstash/api/rack_app.rb

@@ -0,0 +1,33 @@
+require "logstash/api/modules/base"
+require "logstash/api/modules/node"
+require "logstash/api/modules/node_stats"
+require "logstash/api/modules/plugins"
+require "logstash/api/modules/root"
+require "logstash/api/modules/stats"
+
+module LogStash
+  module Api
+    module RackApp
+      def self.app
+        namespaces = rack_namespaces 
+        Rack::Builder.new do
+          run LogStash::Api::Modules::Root
+          namespaces.each_pair do |namespace, app|
+            map(namespace) do
+              run app
+            end
+          end
+        end
+      end
+
+      def self.rack_namespaces
+        {
+          "/_node" => LogStash::Api::Modules::Node,
+          "/_stats" => LogStash::Api::Modules::Stats,
+          "/_node/stats" => LogStash::Api::Modules::NodeStats,
+          "/_plugins" => LogStash::Api::Modules::Plugins
+        }
+      end
+    end
+  end
+end

data/lib/logstash/api/service.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+require "logstash/instrument/collector"
+require "logstash/util/loggable"
+
+module LogStash
+  module Api
+    class Service
+
+      include Singleton
+      include LogStash::Util::Loggable
+
+      def initialize
+        @snapshot_rotation_mutex = Mutex.new
+        @snapshot = nil
+        logger.debug("[api-service] start") if logger.debug?
+        LogStash::Instrument::Collector.instance.add_observer(self)
+      end
+
+      def stop
+        logger.debug("[api-service] stop") if logger.debug?
+        LogStash::Instrument::Collector.instance.delete_observer(self)
+      end
+
+      def agent
+        LogStash::Instrument::Collector.instance.agent
+      end
+
+      def started?
+        !@snapshot.nil? && has_counters?
+      end
+
+      def update(snapshot)
+        logger.debug("[api-service] snapshot received", :snapshot_time => snapshot.created_at) if logger.debug?
+
+        @snapshot_rotation_mutex.synchronize do
+          @snapshot = snapshot
+        end
+      end
+
+      def snapshot
+        @snapshot_rotation_mutex.synchronize { @snapshot }
+      end
+
+      def get_shallow(*path)
+        snapshot.metric_store.get_shallow(*path)
+      end
+
+      def get(key)
+        metric_store = @snapshot_rotation_mutex.synchronize { @snapshot.metric_store }
+        if key == :jvm_memory_stats
+          data = metric_store.get_shallow(:jvm, :memory)
+        else
+          data = metric_store.get_with_path("stats/events")
+        end
+        LogStash::Json.dump(data)
+      end
+
+      private
+
+      def has_counters?
+        (["LogStash::Instrument::MetricType::Counter", "LogStash::Instrument::MetricType::Gauge"] - metric_types).empty?
+      end
+
+      def metric_types
+        types = []
+        @snapshot_rotation_mutex.synchronize do
+          types = @snapshot.metric_store.all.map { |t| t.class.to_s }
+        end
+        return types
+      end
+    end
+  end
+end

data/lib/logstash/instrument/metric_store.rb

@@ -86,7 +86,7 @@ module LogStash module Instrument
 
     # Similar to `get_with_path` but use symbols instead of string
     #
-    # @param [Array<Symbol>
+    # @param [Array<Symbol>]
     # @return [Hash]
     def get(*key_paths)
       # Normalize the symbols access
@@ -100,6 +100,16 @@ module LogStash module Instrument
       new_hash
     end
 
+    # Retrieve values like `get`, but don't return them fully nested.
+    # This means that if you call `get_shallow(:foo, :bar)` the result will not
+    # be nested inside of `{:foo {:bar => values}`.
+    #
+    # @param [Array<Symbol>]
+    # @return [Hash]
+    def get_shallow(*key_paths)
+      key_paths.reduce(get(*key_paths)) {|acc, p| acc[p]}
+    end
+
     # Return all the individuals Metric,
     # This call mimic a Enum's each if a block is provided
     #

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -37,6 +37,8 @@ module LogStash module Instrument module PeriodicPoller
       logger.debug("PeriodicPoller: Starting",
                    :polling_interval => @options[:polling_interval],
                    :polling_timeout => @options[:polling_timeout]) if logger.debug?
+      
+      collect # Collect data right away if possible
       @task.execute
     end
 

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -1,6 +1,15 @@
+
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
-require 'monitoring'
+require 'jrmonitor'
+
+java_import 'java.lang.management.ManagementFactory'
+java_import 'java.lang.management.OperatingSystemMXBean'
+java_import 'com.sun.management.UnixOperatingSystemMXBean'
+java_import 'javax.management.MBeanServer'
+java_import 'javax.management.ObjectName'
+java_import 'javax.management.AttributeList'
+java_import 'javax.naming.directory.Attribute'
 
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
@@ -13,14 +22,50 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = JRMonitor.memory.generate
+      raw = JRMonitor.memory.generate      
       collect_heap_metrics(raw)
       collect_non_heap_metrics(raw)
       collect_pools_metrics(raw)
+      collect_threads_metrics
+      collect_process_metrics
     end
 
     private
 
+    def collect_threads_metrics      
+      threads = JRMonitor.threads.generate
+      
+      current = threads.count
+      if @peak_threads.nil? || @peak_threads < current
+        @peak_threads = current
+      end      
+      
+      metric.gauge([:jvm, :threads], :count, threads.count)     
+      metric.gauge([:jvm, :threads], :peak_count, @peak_threads)
+    end
+
+    def collect_process_metrics
+      process_metrics = JRMonitor.process.generate
+      
+      path = [:jvm, :process]
+
+
+      open_fds = process_metrics["open_file_descriptors"]
+      if @peak_open_fds.nil? || open_fds > @peak_open_fds
+        @peak_open_fds = open_fds
+      end
+      metric.gauge(path, :open_file_descriptors, open_fds)
+      metric.gauge(path, :peak_open_file_descriptors, @peak_open_fds)
+      metric.gauge(path, :max_file_descriptors, process_metrics["max_file_descriptors"])
+
+      cpu_path = path + [:cpu]
+      cpu_metrics = process_metrics["cpu"]
+      metric.gauge(cpu_path, :percent, cpu_metrics["process_percent"])
+      metric.gauge(cpu_path, :total_in_millis, cpu_metrics["total_in_millis"])
+
+      metric.gauge(path + [:mem], :total_virtual_in_bytes, process_metrics["mem"]["total_virtual_in_bytes"])
+    end
+
     def collect_heap_metrics(data)
       heap = aggregate_information_for(data["heap"].values)
       heap[:used_percent] = (heap[:used_in_bytes] / heap[:max_in_bytes].to_f)*100.0

data/lib/logstash/util/thread_dump.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+module LogStash
+  module Util
+    class ThreadDump
+      SKIPPED_THREADS             = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
+      THREADS_COUNT_DEFAULT       = 3.freeze
+      IGNORE_IDLE_THREADS_DEFAULT = true.freeze
+
+      attr_reader :top_count, :ignore, :dump
+
+      def initialize(options={})
+        @options   = options
+        @dump = options.fetch(:dump, JRMonitor.threads.generate({}))
+        @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
+        @ignore    = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
+      end
+
+      def each(&block)
+        i=0
+        dump.each_pair do |thread_name, _hash|
+          break if i >= top_count
+          if ignore
+            next if idle_thread?(thread_name, _hash)
+          end
+          block.call(thread_name, _hash)
+          i += 1
+        end
+      end
+
+      def idle_thread?(thread_name, data)
+        idle = false
+        if SKIPPED_THREADS.include?(thread_name)
+          # these are likely JVM dependent
+          idle = true
+        elsif thread_name.match(/Ruby-\d+-JIT-\d+/)
+          # This are internal JRuby JIT threads, 
+          # see java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor for details.
+          idle = true
+        elsif thread_name.match(/pool-\d+-thread-\d+/)
+          # This are threads used by the internal JRuby implementation to dispatch
+          # calls and tasks, see prg.jruby.internal.runtime.methods.DynamicMethod.call
+          idle = true
+        else
+          data["thread.stacktrace"].each do |trace|
+            if trace.start_with?("java.util.concurrent.ThreadPoolExecutor.getTask")
+              idle = true
+              break
+            end
+          end
+        end
+        idle
+      end
+    end
+  end
+end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha3.snapshot5"
+LOGSTASH_VERSION = "5.0.0-alpha3.snapshot6"

data/lib/logstash/webserver.rb

@@ -1,16 +1,15 @@
 # encoding: utf-8
 require "puma"
-require "puma/single"
-require "puma/binder"
-require "puma/configuration"
-require "puma/commonlogger"
+require "puma/server"
+require "sinatra"
+require "rack"
+require "logstash/api/rack_app"
 
 module LogStash 
   class WebServer
-
     extend Forwardable
 
-    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events
+    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port
 
     def_delegator :@runner, :stats
 
@@ -19,8 +18,8 @@ module LogStash
 
     def initialize(logger, options={})
       @logger      = logger
-      http_host    = options[:http_host] || DEFAULT_HOST
-      http_port    = options[:http_port] || DEFAULT_PORT
+      @http_host    = options[:http_host] || DEFAULT_HOST
+      @http_port    = options[:http_port] || DEFAULT_PORT
       @options     = {}
       @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
                                      :binds => ["tcp://#{http_host}:#{http_port}"],
@@ -31,24 +30,17 @@ module LogStash
                                      :queue_requests => false
       })
       @status      = nil
-
-      parse_options
-
-      @runner  = nil
-      @events  = ::Puma::Events.strings
-      @binder  = ::Puma::Binder.new(@events)
-      @binder.import_from_env
-
-      set_environment
     end
 
     def run
       log "=== puma start: #{Time.now} ==="
 
-      @runner = Puma::Single.new(self)
-      @status = :run
-      @runner.run
-      stop(:graceful => true)
+      stop # Just in case
+
+      @server = ::Puma::Server.new(LogStash::Api::RackApp.app)
+      @server.add_tcp_listener(http_host, http_port)
+
+      @server.run.join
     end
 
     def log(str)
@@ -58,7 +50,7 @@ module LogStash
     def error(str)
       logger.error(str)
     end
-
+    
     # Empty method, this method is required because of the puma usage we make through
     # the Single interface, https://github.com/puma/puma/blob/master/lib/puma/single.rb#L82
     # for more details. This can always be implemented when we want to keep track of this
@@ -66,33 +58,7 @@ module LogStash
     def write_state; end
 
     def stop(options={})
-      graceful = options.fetch(:graceful, true)
-
-      if graceful
-        @runner.stop_blocked
-      else
-        @runner.stop
-      end rescue nil
-
-      @status = :stop
-      log "=== puma shutdown: #{Time.now} ==="
-    end
-
-    private
-
-    def env
-      @options[:debug] ? "development" : "production"
-    end
-
-    def set_environment
-      @options[:environment] = env
-      ENV['RACK_ENV']        = env
-    end
-
-    def parse_options
-      @config  = ::Puma::Configuration.new(cli_options)
-      @config.load
-      @options = @config.options
+      @server.stop(true) if @server
     end
   end
 end

data/locales/en.yml

@@ -1,6 +1,6 @@
 # YAML notes
 #   |- means 'scalar block' useful for formatted text
-#   > means 'scalar block' but it chomps all newlines. Useful 
+#   > means 'scalar block' but it chomps all newlines. Useful
 #     for unformatted text.
 en:
   oops: |-
@@ -76,7 +76,7 @@ en:
       hot_threads:
         title: |-
           ::: {%{hostname}}
-            Hot threads at %{time}, busiestThreads=%{top_count}:
+          Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
             %{percent_of_cpu_time} % of cpu usage by %{thread_state} thread named '%{thread_name}'
     runner:
@@ -141,18 +141,18 @@ en:
           after %{after}
         invalid_plugin_register: >-
           Cannot register %{plugin} %{type} plugin.
-          The error reported is: 
+          The error reported is:
             %{error}
         plugin_path_missing: >-
           You specified a plugin path that does not exist: %{path}
         no_plugins_found: |-
           Could not find any plugins in "%{path}"
-          I tried to find files matching the following, but found none: 
+          I tried to find files matching the following, but found none:
             %{plugin_glob}
         log_file_failed: |-
           Failed to open %{path} for writing: %{error}
 
-          This is often a permissions issue, or the wrong 
+          This is often a permissions issue, or the wrong
           path was specified?
       flag:
         # Note: Wrap these at 55 chars so they display nicely when clamp emits

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha3.snapshot5"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha3.snapshot6"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -27,10 +27,10 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
   gem.add_runtime_dependency "concurrent-ruby", "1.0.0"
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
-  gem.add_runtime_dependency 'puma', '~> 2.16', '>= 2.16.0'
+  gem.add_runtime_dependency 'puma', '~> 3.4.0'
   gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
-  gem.add_runtime_dependency "jruby-monitoring", '~> 0.3.1'
+  gem.add_runtime_dependency "jrmonitor", '~> 0.4.2'
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan

data/spec/api/lib/api/node_spec.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
 require "sinatra"
-require "app/modules/node"
+require "logstash/api/modules/node"
 require "logstash/json"
 
-describe LogStash::Api::Node do
+describe LogStash::Api::Modules::Node do
 
   include Rack::Test::Methods