logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/api/init.ru

@@ -0,0 +1,31 @@
+ROOT = File.expand_path(File.dirname(__FILE__))
+$LOAD_PATH.unshift File.join(ROOT, 'lib')
+Dir.glob('lib/**').each{ |d| $LOAD_PATH.unshift(File.join(ROOT, d)) }
+
+require 'sinatra'
+require 'app/root'
+require 'app/modules/stats'
+require 'app/modules/node'
+require 'app/modules/node_stats'
+require 'app/modules/plugins'
+
+env = ENV["RACK_ENV"].to_sym
+set :environment, env
+
+set :service, LogStash::Api::Service.instance
+
+configure do
+  enable :logging
+end
+run LogStash::Api::Root
+
+namespaces = { "/_node" => LogStash::Api::Node,
+               "/_node/stats" => LogStash::Api::NodeStats,
+               "/_stats" => LogStash::Api::Stats,
+               "/_plugins" => LogStash::Api::Plugins }
+
+namespaces.each_pair do |namespace, app|
+  map(namespace) do
+    run app
+  end
+end

data/lib/logstash/api/lib/app.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "cabin"
+require "logstash/json"
+require "helpers/app_helpers"
+require "app/service"
+require "app/command_factory"
+require "logstash/util/loggable"
+
+module LogStash::Api
+  class BaseApp < ::Sinatra::Application
+
+    attr_reader :factory
+
+    if settings.environment != :production
+      set :raise_errors, true
+      set :show_exceptions, :after_handler
+    end
+
+    include LogStash::Util::Loggable
+
+    helpers AppHelpers
+
+    def initialize(app=nil)
+      super(app)
+      @factory = CommandFactory.new(settings.service)
+    end
+
+    not_found do
+      status 404
+      as   = params.has_key?("human") ? :string : :json
+      text = as == :string ? "" : {}
+      respond_with(text, :as => as)
+    end
+
+    error do
+      logger.error(env['sinatra.error'].message, :url => request.url, :ip => request.ip, :params => request.params)
+    end
+
+  end
+end

data/lib/logstash/api/lib/app/command.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "app/service"
+
+module LogStash::Api
+  class Command
+
+    attr_reader :service
+
+    def initialize(service = LogStash::Api::Service.instance)
+      @service = service
+    end
+
+    def run
+      raise "Not implemented"
+    end
+
+    def hostname
+      service.agent.node_name
+    end
+
+    def uptime
+      service.agent.uptime
+    end
+
+    def started_at
+      (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
+    end
+  end
+end

data/lib/logstash/api/lib/app/command_factory.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "app/service"
+require "app/commands/system/basicinfo_command"
+require "app/commands/stats/events_command"
+require "app/commands/stats/hotthreads_command"
+require "app/commands/stats/memory_command"
+require "app/commands/system/plugins_command"
+
+module LogStash::Api
+  class CommandFactory
+
+    attr_reader :factory, :service
+
+    def initialize(service)
+      @service = service
+      @factory = {}.merge(
+        :system_basic_info => SystemBasicInfoCommand,
+        :events_command => StatsEventsCommand,
+        :hot_threads_command => HotThreadsCommand,
+        :memory_command => JvmMemoryCommand,
+        :plugins_command => PluginsCommand
+      )
+    end
+
+    def build(klass)
+      factory[klass].new(service)
+    end
+  end
+end

data/lib/logstash/api/lib/app/commands/stats/events_command.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "app/command"
+
+class LogStash::Api::StatsEventsCommand < LogStash::Api::Command
+
+  def run
+    #return whatever is comming out of the snapshot event, this obvoiusly
+    #need to be tailored to the right metrics for this command.
+    stats =  LogStash::Json.load(service.get(:events_stats))
+    stats["stats"]["events"]
+  end
+
+end

data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "app/command"
+require 'monitoring'
+require "socket"
+
+class LogStash::Api::HotThreadsCommand < LogStash::Api::Command
+
+  STACK_TRACES_SIZE_DEFAULT = 10.freeze
+
+  def run(options={})
+    filter = { :stacktrace_size => options.fetch(:stacktrace_size, STACK_TRACES_SIZE_DEFAULT) }
+    hash   = JRMonitor.threads.generate(filter)
+    ThreadDump.new(hash, self, options)
+  end
+
+  private
+
+  class ThreadDump
+
+    SKIPPED_THREADS             = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
+    THREADS_COUNT_DEFAULT       = 3.freeze
+    IGNORE_IDLE_THREADS_DEFAULT = true.freeze
+
+    attr_reader :top_count, :ignore, :dump
+
+    def initialize(dump, cmd, options={})
+      @dump      = dump
+      @options   = options
+      @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
+      @ignore    = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
+      @cmd       = cmd
+    end
+
+    def to_s
+      hash = to_hash
+      report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => top_count )} \n"
+      hash[:threads].each do |thread|
+        thread_report = ""
+        thread_report = "\t #{I18n.t("logstash.web_api.hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+        thread_report = "\t #{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+        thread_report << "\t\t #{thread[:path]}\n" if thread[:path]
+        thread[:traces].split("\n").each do |trace|
+          thread_report << "#{trace}\n"
+        end
+        report << thread_report
+      end
+      report
+    end
+
+    def to_hash
+      hash = { :hostname => hostname, :time => Time.now.iso8601, :busiest_threads => top_count, :threads => [] }
+      each do |thread_name, _hash|
+        thread_name, thread_path = _hash["thread.name"].split(": ")
+        thread = { :name => thread_name,
+                   :percent_of_cpu_time => cpu_time_as_percent(_hash),
+                   :state => _hash["thread.state"]
+        }
+        thread[:path] = thread_path if thread_path
+        traces = ""
+        _hash["thread.stacktrace"].each do |trace|
+          traces << "\t\t#{trace}\n"
+        end
+        thread[:traces] = traces unless traces.empty?
+        hash[:threads] << thread
+      end
+      hash
+    end
+
+    private
+
+    def each(&block)
+      i=0
+      dump.each_pair do |thread_name, _hash|
+        break if i >= top_count
+        if ignore
+          next if idle_thread?(thread_name, _hash)
+        end
+        block.call(thread_name, _hash)
+        i += 1
+      end
+    end
+
+    def idle_thread?(thread_name, data)
+      idle = false
+      if SKIPPED_THREADS.include?(thread_name)
+        # these are likely JVM dependent
+        idle = true
+      elsif thread_name.match(/Ruby-\d+-JIT-\d+/)
+        # This are internal JRuby JIT threads, 
+        # see java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor for details.
+        idle = true
+      elsif thread_name.match(/pool-\d+-thread-\d+/)
+        # This are threads used by the internal JRuby implementation to dispatch
+        # calls and tasks, see prg.jruby.internal.runtime.methods.DynamicMethod.call
+        idle = true
+      else
+        data["thread.stacktrace"].each do |trace|
+          if trace.start_with?("java.util.concurrent.ThreadPoolExecutor.getTask")
+            idle = true
+            break
+          end
+        end
+      end
+      idle
+    end
+
+    def hostname
+      @cmd.hostname
+    end
+
+    def cpu_time_as_percent(hash)
+      (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
+    end
+
+    def cpu_time(hash)
+      hash["cpu.time"] / 1000000.0
+    end
+  end
+
+end

data/lib/logstash/api/lib/app/commands/stats/memory_command.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "app/command"
+require 'monitoring'
+
+class LogStash::Api::JvmMemoryCommand < LogStash::Api::Command
+
+  def run
+    memory = LogStash::Json.load(service.get(:jvm_memory_stats))
+    {
+      :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
+      :heap_used_percent => memory["heap"]["used_percent"],
+      :heap_committed_in_bytes => memory["heap"]["committed_in_bytes"],
+      :heap_max_in_bytes => memory["heap"]["max_in_bytes"],
+      :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
+      :non_heap_used_in_bytes => memory["non_heap"]["used_in_bytes"],
+      :non_heap_committed_in_bytes => memory["non_heap"]["committed_in_bytes"],
+      :pools => memory["pools"].inject({}) do |acc, (type, hash)|
+          hash.delete("committed_in_bytes")
+          acc[type] = hash
+          acc
+    end
+    }
+  end
+
+end

data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "app/command"
+require "logstash/util/duration_formatter"
+
+class LogStash::Api::SystemBasicInfoCommand < LogStash::Api::Command
+
+  def run
+    {
+      "hostname" => hostname,
+      "version" => {
+        "number" => LOGSTASH_VERSION
+      }
+    }
+  end
+end

data/lib/logstash/api/lib/app/commands/system/plugins_command.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require "app/command"
+
+class LogStash::Api::PluginsCommand < LogStash::Api::Command
+
+  def run
+    { :total => plugins.count, :plugins => plugins }
+  end
+
+  private
+
+  def plugins
+    @plugins ||= find_plugins_gem_specs.map do |spec|
+      { :name => spec.name, :version => spec.version.to_s }
+    end.sort_by do |spec|
+      spec[:name]
+    end
+  end
+
+  def find_plugins_gem_specs
+    @specs ||= Gem::Specification.find_all.select{|spec| logstash_plugin_gem_spec?(spec)}
+  end
+
+  def logstash_plugin_gem_spec?(spec)
+    spec.metadata && spec.metadata["logstash_plugin"] == "true"
+  end
+
+end

data/lib/logstash/api/lib/app/modules/node.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "app"
+
+module LogStash::Api
+  class Node < BaseApp
+
+    helpers AppHelpers
+
+    # return hot threads information
+    get "/hot_threads" do
+      ignore_idle_threads = params["ignore_idle_threads"] || true
+
+      options = {
+        :ignore_idle_threads => as_boolean(ignore_idle_threads),
+        :human => params.has_key?("human")
+      }
+      options[:threads] = params["threads"].to_i if params.has_key?("threads")
+
+      command = factory.build(:hot_threads_command)
+      as    = options[:human] ? :string : :json
+      respond_with(command.run(options), {:as => as})
+    end
+
+  end
+end

data/lib/logstash/api/lib/app/modules/node_stats.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "app"
+
+module LogStash::Api
+  class NodeStats < BaseApp
+
+    helpers AppHelpers
+
+
+    # Global _stats resource where all information is 
+    # retrieved and show
+    get "/" do
+      events_command = factory.build(:events_command)
+      payload = {
+        :events => events_command.run,
+        :jvm => jvm_payload
+      }
+
+      respond_with payload
+    end
+
+    # Show all events stats information
+    # (for ingested, emitted, dropped)
+    # - #events since startup
+    # - #data (bytes) since startup
+    # - events/s
+    # - bytes/s
+    # - dropped events/s
+    # - events in the pipeline
+    get "/events" do
+      command = factory.build(:events_command)
+      respond_with({ :events => command.run })
+    end
+
+    # return hot threads information
+    get "/jvm" do
+      respond_with jvm_payload
+    end
+
+    private
+
+    def jvm_payload
+      command = factory.build(:memory_command)
+      {
+        :timestamp => command.started_at,
+        :uptime_in_millis => command.uptime,
+        :mem => command.run
+      }
+    end
+  end
+end

data/lib/logstash/api/lib/app/modules/plugins.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "app"
+
+module LogStash::Api
+  class Plugins < BaseApp
+
+    helpers AppHelpers
+
+    get "/" do
+      command = factory.build(:plugins_command)
+      respond_with(command.run())
+    end
+
+  end
+end

data/lib/logstash/api/lib/app/modules/stats.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "app"
+
+module LogStash::Api
+  class Stats < BaseApp
+
+    helpers AppHelpers
+
+    # return hot threads information
+    get "/jvm" do
+      command = factory.build(:memory_command)
+      jvm_payload = {
+        :timestamp => command.started_at,
+        :uptime_in_millis => command.uptime,
+        :mem => command.run
+      }
+      respond_with({:jvm => jvm_payload})
+    end
+
+  end
+end