logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash-core/version.rb +1 -1
  3. data/lib/logstash/agent.rb +48 -31
  4. data/lib/logstash/api/{lib/helpers/app_helpers.rb → app_helpers.rb} +0 -0
  5. data/lib/logstash/api/command_factory.rb +34 -0
  6. data/lib/logstash/api/commands/base.rb +25 -0
  7. data/lib/logstash/api/commands/stats.rb +105 -0
  8. data/lib/logstash/api/commands/system/basicinfo_command.rb +23 -0
  9. data/lib/logstash/api/commands/system/plugins_command.rb +35 -0
  10. data/lib/logstash/api/modules/base.rb +43 -0
  11. data/lib/logstash/api/modules/node.rb +24 -0
  12. data/lib/logstash/api/modules/node_stats.rb +59 -0
  13. data/lib/logstash/api/modules/plugins.rb +15 -0
  14. data/lib/logstash/api/modules/root.rb +15 -0
  15. data/lib/logstash/api/modules/stats.rb +63 -0
  16. data/lib/logstash/api/rack_app.rb +33 -0
  17. data/lib/logstash/api/service.rb +73 -0
  18. data/lib/logstash/config/config_ast.rb +23 -18
  19. data/lib/logstash/config/loader.rb +4 -4
  20. data/lib/logstash/config/mixin.rb +10 -21
  21. data/lib/logstash/environment.rb +29 -0
  22. data/lib/logstash/filters/base.rb +2 -2
  23. data/lib/logstash/inputs/base.rb +2 -2
  24. data/lib/logstash/instrument/collector.rb +1 -1
  25. data/lib/logstash/instrument/metric_store.rb +11 -1
  26. data/lib/logstash/instrument/periodic_poller/base.rb +2 -0
  27. data/lib/logstash/instrument/periodic_poller/jvm.rb +47 -2
  28. data/lib/logstash/logging/json.rb +21 -0
  29. data/lib/logstash/output_delegator.rb +2 -2
  30. data/lib/logstash/patches/clamp.rb +69 -0
  31. data/lib/logstash/pipeline.rb +36 -69
  32. data/lib/logstash/plugin.rb +1 -1
  33. data/lib/logstash/runner.rb +171 -146
  34. data/lib/logstash/settings.rb +267 -0
  35. data/lib/logstash/util/decorators.rb +6 -6
  36. data/lib/logstash/util/java_version.rb +1 -10
  37. data/lib/logstash/util/thread_dump.rb +55 -0
  38. data/lib/logstash/util/worker_threads_default_printer.rb +2 -2
  39. data/lib/logstash/version.rb +1 -1
  40. data/lib/logstash/webserver.rb +15 -49
  41. data/locales/en.yml +22 -25
  42. data/logstash-core.gemspec +3 -3
  43. data/spec/api/lib/api/node_spec.rb +2 -2
  44. data/spec/api/lib/api/node_stats_spec.rb +32 -57
  45. data/spec/api/lib/api/plugins_spec.rb +3 -3
  46. data/spec/api/lib/api/root_spec.rb +2 -2
  47. data/spec/api/lib/api/support/resource_dsl_methods.rb +47 -0
  48. data/spec/api/lib/commands/stats.rb +47 -0
  49. data/spec/api/spec_helper.rb +21 -21
  50. data/spec/conditionals_spec.rb +113 -113
  51. data/spec/logstash/agent_spec.rb +85 -68
  52. data/spec/logstash/config/config_ast_spec.rb +4 -2
  53. data/spec/logstash/config/mixin_spec.rb +33 -7
  54. data/spec/logstash/filters/base_spec.rb +16 -16
  55. data/spec/logstash/inputs/base_spec.rb +8 -8
  56. data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +45 -0
  57. data/spec/logstash/output_delegator_spec.rb +2 -0
  58. data/spec/logstash/pipeline_reporter_spec.rb +5 -1
  59. data/spec/logstash/pipeline_spec.rb +75 -30
  60. data/spec/logstash/plugin_spec.rb +2 -2
  61. data/spec/logstash/runner_spec.rb +112 -25
  62. data/spec/logstash/setting_spec.rb +130 -0
  63. data/spec/logstash/settings_spec.rb +62 -0
  64. metadata +37 -43
  65. data/lib/logstash/api/init.ru +0 -31
  66. data/lib/logstash/api/lib/app.rb +0 -40
  67. data/lib/logstash/api/lib/app/command.rb +0 -29
  68. data/lib/logstash/api/lib/app/command_factory.rb +0 -29
  69. data/lib/logstash/api/lib/app/commands/stats/events_command.rb +0 -13
  70. data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb +0 -120
  71. data/lib/logstash/api/lib/app/commands/stats/memory_command.rb +0 -25
  72. data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb +0 -15
  73. data/lib/logstash/api/lib/app/commands/system/plugins_command.rb +0 -28
  74. data/lib/logstash/api/lib/app/modules/node.rb +0 -25
  75. data/lib/logstash/api/lib/app/modules/node_stats.rb +0 -51
  76. data/lib/logstash/api/lib/app/modules/plugins.rb +0 -15
  77. data/lib/logstash/api/lib/app/modules/stats.rb +0 -21
  78. data/lib/logstash/api/lib/app/root.rb +0 -13
  79. data/lib/logstash/api/lib/app/service.rb +0 -61
  80. data/lib/logstash/api/lib/app/stats.rb +0 -56
  81. data/lib/logstash/util/defaults_printer.rb +0 -31
  82. data/spec/api/lib/api/stats_spec.rb +0 -19
  83. data/spec/api/lib/commands/events_spec.rb +0 -17
  84. data/spec/api/lib/commands/jvm_spec.rb +0 -45
  85. data/spec/logstash/util/defaults_printer_spec.rb +0 -50
  86. data/spec/logstash/util/worker_threads_default_printer_spec.rb +0 -45
@@ -1,31 +0,0 @@
1
- ROOT = File.expand_path(File.dirname(__FILE__))
2
- $LOAD_PATH.unshift File.join(ROOT, 'lib')
3
- Dir.glob('lib/**').each{ |d| $LOAD_PATH.unshift(File.join(ROOT, d)) }
4
-
5
- require 'sinatra'
6
- require 'app/root'
7
- require 'app/modules/stats'
8
- require 'app/modules/node'
9
- require 'app/modules/node_stats'
10
- require 'app/modules/plugins'
11
-
12
- env = ENV["RACK_ENV"].to_sym
13
- set :environment, env
14
-
15
- set :service, LogStash::Api::Service.instance
16
-
17
- configure do
18
- enable :logging
19
- end
20
- run LogStash::Api::Root
21
-
22
- namespaces = { "/_node" => LogStash::Api::Node,
23
- "/_node/stats" => LogStash::Api::NodeStats,
24
- "/_stats" => LogStash::Api::Stats,
25
- "/_plugins" => LogStash::Api::Plugins }
26
-
27
- namespaces.each_pair do |namespace, app|
28
- map(namespace) do
29
- run app
30
- end
31
- end
@@ -1,40 +0,0 @@
1
- # encoding: utf-8
2
- require "cabin"
3
- require "logstash/json"
4
- require "helpers/app_helpers"
5
- require "app/service"
6
- require "app/command_factory"
7
- require "logstash/util/loggable"
8
-
9
- module LogStash::Api
10
- class BaseApp < ::Sinatra::Application
11
-
12
- attr_reader :factory
13
-
14
- if settings.environment != :production
15
- set :raise_errors, true
16
- set :show_exceptions, :after_handler
17
- end
18
-
19
- include LogStash::Util::Loggable
20
-
21
- helpers AppHelpers
22
-
23
- def initialize(app=nil)
24
- super(app)
25
- @factory = CommandFactory.new(settings.service)
26
- end
27
-
28
- not_found do
29
- status 404
30
- as = params.has_key?("human") ? :string : :json
31
- text = as == :string ? "" : {}
32
- respond_with(text, :as => as)
33
- end
34
-
35
- error do
36
- logger.error(env['sinatra.error'].message, :url => request.url, :ip => request.ip, :params => request.params)
37
- end
38
-
39
- end
40
- end
@@ -1,29 +0,0 @@
1
- # encoding: utf-8
2
- require "app/service"
3
-
4
- module LogStash::Api
5
- class Command
6
-
7
- attr_reader :service
8
-
9
- def initialize(service = LogStash::Api::Service.instance)
10
- @service = service
11
- end
12
-
13
- def run
14
- raise "Not implemented"
15
- end
16
-
17
- def hostname
18
- service.agent.node_name
19
- end
20
-
21
- def uptime
22
- service.agent.uptime
23
- end
24
-
25
- def started_at
26
- (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
27
- end
28
- end
29
- end
@@ -1,29 +0,0 @@
1
- # encoding: utf-8
2
- require "app/service"
3
- require "app/commands/system/basicinfo_command"
4
- require "app/commands/stats/events_command"
5
- require "app/commands/stats/hotthreads_command"
6
- require "app/commands/stats/memory_command"
7
- require "app/commands/system/plugins_command"
8
-
9
- module LogStash::Api
10
- class CommandFactory
11
-
12
- attr_reader :factory, :service
13
-
14
- def initialize(service)
15
- @service = service
16
- @factory = {}.merge(
17
- :system_basic_info => SystemBasicInfoCommand,
18
- :events_command => StatsEventsCommand,
19
- :hot_threads_command => HotThreadsCommand,
20
- :memory_command => JvmMemoryCommand,
21
- :plugins_command => PluginsCommand
22
- )
23
- end
24
-
25
- def build(klass)
26
- factory[klass].new(service)
27
- end
28
- end
29
- end
@@ -1,13 +0,0 @@
1
- # encoding: utf-8
2
- require "app/command"
3
-
4
- class LogStash::Api::StatsEventsCommand < LogStash::Api::Command
5
-
6
- def run
7
- #return whatever is comming out of the snapshot event, this obvoiusly
8
- #need to be tailored to the right metrics for this command.
9
- stats = LogStash::Json.load(service.get(:events_stats))
10
- stats["stats"]["events"]
11
- end
12
-
13
- end
@@ -1,120 +0,0 @@
1
- # encoding: utf-8
2
- require "app/command"
3
- require 'monitoring'
4
- require "socket"
5
-
6
- class LogStash::Api::HotThreadsCommand < LogStash::Api::Command
7
-
8
- STACK_TRACES_SIZE_DEFAULT = 10.freeze
9
-
10
- def run(options={})
11
- filter = { :stacktrace_size => options.fetch(:stacktrace_size, STACK_TRACES_SIZE_DEFAULT) }
12
- hash = JRMonitor.threads.generate(filter)
13
- ThreadDump.new(hash, self, options)
14
- end
15
-
16
- private
17
-
18
- class ThreadDump
19
-
20
- SKIPPED_THREADS = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
21
- THREADS_COUNT_DEFAULT = 3.freeze
22
- IGNORE_IDLE_THREADS_DEFAULT = true.freeze
23
-
24
- attr_reader :top_count, :ignore, :dump
25
-
26
- def initialize(dump, cmd, options={})
27
- @dump = dump
28
- @options = options
29
- @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
30
- @ignore = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
31
- @cmd = cmd
32
- end
33
-
34
- def to_s
35
- hash = to_hash
36
- report = "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => top_count )} \n"
37
- hash[:threads].each do |thread|
38
- thread_report = ""
39
- thread_report = "\t #{I18n.t("logstash.web_api.hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
40
- thread_report = "\t #{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
41
- thread_report << "\t\t #{thread[:path]}\n" if thread[:path]
42
- thread[:traces].split("\n").each do |trace|
43
- thread_report << "#{trace}\n"
44
- end
45
- report << thread_report
46
- end
47
- report
48
- end
49
-
50
- def to_hash
51
- hash = { :hostname => hostname, :time => Time.now.iso8601, :busiest_threads => top_count, :threads => [] }
52
- each do |thread_name, _hash|
53
- thread_name, thread_path = _hash["thread.name"].split(": ")
54
- thread = { :name => thread_name,
55
- :percent_of_cpu_time => cpu_time_as_percent(_hash),
56
- :state => _hash["thread.state"]
57
- }
58
- thread[:path] = thread_path if thread_path
59
- traces = ""
60
- _hash["thread.stacktrace"].each do |trace|
61
- traces << "\t\t#{trace}\n"
62
- end
63
- thread[:traces] = traces unless traces.empty?
64
- hash[:threads] << thread
65
- end
66
- hash
67
- end
68
-
69
- private
70
-
71
- def each(&block)
72
- i=0
73
- dump.each_pair do |thread_name, _hash|
74
- break if i >= top_count
75
- if ignore
76
- next if idle_thread?(thread_name, _hash)
77
- end
78
- block.call(thread_name, _hash)
79
- i += 1
80
- end
81
- end
82
-
83
- def idle_thread?(thread_name, data)
84
- idle = false
85
- if SKIPPED_THREADS.include?(thread_name)
86
- # these are likely JVM dependent
87
- idle = true
88
- elsif thread_name.match(/Ruby-\d+-JIT-\d+/)
89
- # This are internal JRuby JIT threads,
90
- # see java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor for details.
91
- idle = true
92
- elsif thread_name.match(/pool-\d+-thread-\d+/)
93
- # This are threads used by the internal JRuby implementation to dispatch
94
- # calls and tasks, see prg.jruby.internal.runtime.methods.DynamicMethod.call
95
- idle = true
96
- else
97
- data["thread.stacktrace"].each do |trace|
98
- if trace.start_with?("java.util.concurrent.ThreadPoolExecutor.getTask")
99
- idle = true
100
- break
101
- end
102
- end
103
- end
104
- idle
105
- end
106
-
107
- def hostname
108
- @cmd.hostname
109
- end
110
-
111
- def cpu_time_as_percent(hash)
112
- (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
113
- end
114
-
115
- def cpu_time(hash)
116
- hash["cpu.time"] / 1000000.0
117
- end
118
- end
119
-
120
- end
@@ -1,25 +0,0 @@
1
- # encoding: utf-8
2
- require "app/command"
3
- require 'monitoring'
4
-
5
- class LogStash::Api::JvmMemoryCommand < LogStash::Api::Command
6
-
7
- def run
8
- memory = LogStash::Json.load(service.get(:jvm_memory_stats))
9
- {
10
- :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
11
- :heap_used_percent => memory["heap"]["used_percent"],
12
- :heap_committed_in_bytes => memory["heap"]["committed_in_bytes"],
13
- :heap_max_in_bytes => memory["heap"]["max_in_bytes"],
14
- :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
15
- :non_heap_used_in_bytes => memory["non_heap"]["used_in_bytes"],
16
- :non_heap_committed_in_bytes => memory["non_heap"]["committed_in_bytes"],
17
- :pools => memory["pools"].inject({}) do |acc, (type, hash)|
18
- hash.delete("committed_in_bytes")
19
- acc[type] = hash
20
- acc
21
- end
22
- }
23
- end
24
-
25
- end
@@ -1,15 +0,0 @@
1
- # encoding: utf-8
2
- require "app/command"
3
- require "logstash/util/duration_formatter"
4
-
5
- class LogStash::Api::SystemBasicInfoCommand < LogStash::Api::Command
6
-
7
- def run
8
- {
9
- "hostname" => hostname,
10
- "version" => {
11
- "number" => LOGSTASH_VERSION
12
- }
13
- }
14
- end
15
- end
@@ -1,28 +0,0 @@
1
- # encoding: utf-8
2
- require "app/command"
3
-
4
- class LogStash::Api::PluginsCommand < LogStash::Api::Command
5
-
6
- def run
7
- { :total => plugins.count, :plugins => plugins }
8
- end
9
-
10
- private
11
-
12
- def plugins
13
- @plugins ||= find_plugins_gem_specs.map do |spec|
14
- { :name => spec.name, :version => spec.version.to_s }
15
- end.sort_by do |spec|
16
- spec[:name]
17
- end
18
- end
19
-
20
- def find_plugins_gem_specs
21
- @specs ||= Gem::Specification.find_all.select{|spec| logstash_plugin_gem_spec?(spec)}
22
- end
23
-
24
- def logstash_plugin_gem_spec?(spec)
25
- spec.metadata && spec.metadata["logstash_plugin"] == "true"
26
- end
27
-
28
- end
@@ -1,25 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
-
4
- module LogStash::Api
5
- class Node < BaseApp
6
-
7
- helpers AppHelpers
8
-
9
- # return hot threads information
10
- get "/hot_threads" do
11
- ignore_idle_threads = params["ignore_idle_threads"] || true
12
-
13
- options = {
14
- :ignore_idle_threads => as_boolean(ignore_idle_threads),
15
- :human => params.has_key?("human")
16
- }
17
- options[:threads] = params["threads"].to_i if params.has_key?("threads")
18
-
19
- command = factory.build(:hot_threads_command)
20
- as = options[:human] ? :string : :json
21
- respond_with(command.run(options), {:as => as})
22
- end
23
-
24
- end
25
- end
@@ -1,51 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
-
4
- module LogStash::Api
5
- class NodeStats < BaseApp
6
-
7
- helpers AppHelpers
8
-
9
-
10
- # Global _stats resource where all information is
11
- # retrieved and show
12
- get "/" do
13
- events_command = factory.build(:events_command)
14
- payload = {
15
- :events => events_command.run,
16
- :jvm => jvm_payload
17
- }
18
-
19
- respond_with payload
20
- end
21
-
22
- # Show all events stats information
23
- # (for ingested, emitted, dropped)
24
- # - #events since startup
25
- # - #data (bytes) since startup
26
- # - events/s
27
- # - bytes/s
28
- # - dropped events/s
29
- # - events in the pipeline
30
- get "/events" do
31
- command = factory.build(:events_command)
32
- respond_with({ :events => command.run })
33
- end
34
-
35
- # return hot threads information
36
- get "/jvm" do
37
- respond_with jvm_payload
38
- end
39
-
40
- private
41
-
42
- def jvm_payload
43
- command = factory.build(:memory_command)
44
- {
45
- :timestamp => command.started_at,
46
- :uptime_in_millis => command.uptime,
47
- :mem => command.run
48
- }
49
- end
50
- end
51
- end
@@ -1,15 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
-
4
- module LogStash::Api
5
- class Plugins < BaseApp
6
-
7
- helpers AppHelpers
8
-
9
- get "/" do
10
- command = factory.build(:plugins_command)
11
- respond_with(command.run())
12
- end
13
-
14
- end
15
- end
@@ -1,21 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
-
4
- module LogStash::Api
5
- class Stats < BaseApp
6
-
7
- helpers AppHelpers
8
-
9
- # return hot threads information
10
- get "/jvm" do
11
- command = factory.build(:memory_command)
12
- jvm_payload = {
13
- :timestamp => command.started_at,
14
- :uptime_in_millis => command.uptime,
15
- :mem => command.run
16
- }
17
- respond_with({:jvm => jvm_payload})
18
- end
19
-
20
- end
21
- end