logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash-core/version.rb +1 -1
  3. data/lib/logstash/agent.rb +48 -31
  4. data/lib/logstash/api/{lib/helpers/app_helpers.rb → app_helpers.rb} +0 -0
  5. data/lib/logstash/api/command_factory.rb +34 -0
  6. data/lib/logstash/api/commands/base.rb +25 -0
  7. data/lib/logstash/api/commands/stats.rb +105 -0
  8. data/lib/logstash/api/commands/system/basicinfo_command.rb +23 -0
  9. data/lib/logstash/api/commands/system/plugins_command.rb +35 -0
  10. data/lib/logstash/api/modules/base.rb +43 -0
  11. data/lib/logstash/api/modules/node.rb +24 -0
  12. data/lib/logstash/api/modules/node_stats.rb +59 -0
  13. data/lib/logstash/api/modules/plugins.rb +15 -0
  14. data/lib/logstash/api/modules/root.rb +15 -0
  15. data/lib/logstash/api/modules/stats.rb +63 -0
  16. data/lib/logstash/api/rack_app.rb +33 -0
  17. data/lib/logstash/api/service.rb +73 -0
  18. data/lib/logstash/config/config_ast.rb +23 -18
  19. data/lib/logstash/config/loader.rb +4 -4
  20. data/lib/logstash/config/mixin.rb +10 -21
  21. data/lib/logstash/environment.rb +29 -0
  22. data/lib/logstash/filters/base.rb +2 -2
  23. data/lib/logstash/inputs/base.rb +2 -2
  24. data/lib/logstash/instrument/collector.rb +1 -1
  25. data/lib/logstash/instrument/metric_store.rb +11 -1
  26. data/lib/logstash/instrument/periodic_poller/base.rb +2 -0
  27. data/lib/logstash/instrument/periodic_poller/jvm.rb +47 -2
  28. data/lib/logstash/logging/json.rb +21 -0
  29. data/lib/logstash/output_delegator.rb +2 -2
  30. data/lib/logstash/patches/clamp.rb +69 -0
  31. data/lib/logstash/pipeline.rb +36 -69
  32. data/lib/logstash/plugin.rb +1 -1
  33. data/lib/logstash/runner.rb +171 -146
  34. data/lib/logstash/settings.rb +267 -0
  35. data/lib/logstash/util/decorators.rb +6 -6
  36. data/lib/logstash/util/java_version.rb +1 -10
  37. data/lib/logstash/util/thread_dump.rb +55 -0
  38. data/lib/logstash/util/worker_threads_default_printer.rb +2 -2
  39. data/lib/logstash/version.rb +1 -1
  40. data/lib/logstash/webserver.rb +15 -49
  41. data/locales/en.yml +22 -25
  42. data/logstash-core.gemspec +3 -3
  43. data/spec/api/lib/api/node_spec.rb +2 -2
  44. data/spec/api/lib/api/node_stats_spec.rb +32 -57
  45. data/spec/api/lib/api/plugins_spec.rb +3 -3
  46. data/spec/api/lib/api/root_spec.rb +2 -2
  47. data/spec/api/lib/api/support/resource_dsl_methods.rb +47 -0
  48. data/spec/api/lib/commands/stats.rb +47 -0
  49. data/spec/api/spec_helper.rb +21 -21
  50. data/spec/conditionals_spec.rb +113 -113
  51. data/spec/logstash/agent_spec.rb +85 -68
  52. data/spec/logstash/config/config_ast_spec.rb +4 -2
  53. data/spec/logstash/config/mixin_spec.rb +33 -7
  54. data/spec/logstash/filters/base_spec.rb +16 -16
  55. data/spec/logstash/inputs/base_spec.rb +8 -8
  56. data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +45 -0
  57. data/spec/logstash/output_delegator_spec.rb +2 -0
  58. data/spec/logstash/pipeline_reporter_spec.rb +5 -1
  59. data/spec/logstash/pipeline_spec.rb +75 -30
  60. data/spec/logstash/plugin_spec.rb +2 -2
  61. data/spec/logstash/runner_spec.rb +112 -25
  62. data/spec/logstash/setting_spec.rb +130 -0
  63. data/spec/logstash/settings_spec.rb +62 -0
  64. metadata +37 -43
  65. data/lib/logstash/api/init.ru +0 -31
  66. data/lib/logstash/api/lib/app.rb +0 -40
  67. data/lib/logstash/api/lib/app/command.rb +0 -29
  68. data/lib/logstash/api/lib/app/command_factory.rb +0 -29
  69. data/lib/logstash/api/lib/app/commands/stats/events_command.rb +0 -13
  70. data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb +0 -120
  71. data/lib/logstash/api/lib/app/commands/stats/memory_command.rb +0 -25
  72. data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb +0 -15
  73. data/lib/logstash/api/lib/app/commands/system/plugins_command.rb +0 -28
  74. data/lib/logstash/api/lib/app/modules/node.rb +0 -25
  75. data/lib/logstash/api/lib/app/modules/node_stats.rb +0 -51
  76. data/lib/logstash/api/lib/app/modules/plugins.rb +0 -15
  77. data/lib/logstash/api/lib/app/modules/stats.rb +0 -21
  78. data/lib/logstash/api/lib/app/root.rb +0 -13
  79. data/lib/logstash/api/lib/app/service.rb +0 -61
  80. data/lib/logstash/api/lib/app/stats.rb +0 -56
  81. data/lib/logstash/util/defaults_printer.rb +0 -31
  82. data/spec/api/lib/api/stats_spec.rb +0 -19
  83. data/spec/api/lib/commands/events_spec.rb +0 -17
  84. data/spec/api/lib/commands/jvm_spec.rb +0 -45
  85. data/spec/logstash/util/defaults_printer_spec.rb +0 -50
  86. data/spec/logstash/util/worker_threads_default_printer_spec.rb +0 -45
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0eb6c2c950ece7dfa8bff8e470a006e6fe2fb18a
4
- data.tar.gz: a9f8b462b9648e2b2e0740ee308cde7843e8ff0e
3
+ metadata.gz: 34e8ebcf1aeba4f255cd68db9e23c2c4513c9a61
4
+ data.tar.gz: c523eff7c037d296a719a4dc3330b8b4e0eb0160
5
5
  SHA512:
6
- metadata.gz: 63aad0c89e475690b429cb582fab138fc5d86680cae1279379a044b195465a01a7ae8bef45f84523d4939647462294bc7eb326596b7723c325805903e60f2387
7
- data.tar.gz: cc29c516d56a0d435262a7989480eca295f9f787aa19941723ede215a13f231108feb44f734d6c54957e39362772a7db825be79753b952ccafa6affba42f8917
6
+ metadata.gz: ab44fd374d1044c5904174280f2e8a3dea7801e3874167c4ea28ca896ed08273a33a981ad0442bd525669391348d913a1ac491018187c0786bc6f380b1a507d3
7
+ data.tar.gz: f8e2b3d8d4a1a0906c7a9113c30ee3629d75334e409a337337907432c12f6422e24f63c7fabfc1ea1fee037bbef5ef3397ed4c0006fc858633985f62db3e296d
@@ -5,4 +5,4 @@
5
5
  # Note to authors: this should not include dashes because 'gem' barfs if
6
6
  # you include a dash in the version string.
7
7
 
8
- LOGSTASH_CORE_VERSION = "5.0.0-alpha2.snapshot2"
8
+ LOGSTASH_CORE_VERSION = "5.0.0-alpha3"
@@ -20,7 +20,8 @@ LogStash::Environment.load_locale!
20
20
  class LogStash::Agent
21
21
  STARTED_AT = Time.now.freeze
22
22
 
23
- attr_reader :metric, :node_name, :pipelines, :logger
23
+ attr_reader :metric, :node_name, :pipelines, :settings
24
+ attr_accessor :logger
24
25
 
25
26
  # initialize method for LogStash::Agent
26
27
  # @param params [Hash] potential parameters are:
@@ -28,21 +29,23 @@ class LogStash::Agent
28
29
  # :auto_reload [Boolean] - enable reloading of pipelines
29
30
  # :reload_interval [Integer] - reload pipelines every X seconds
30
31
  # :logger [Cabin::Channel] - logger instance
31
- def initialize(params)
32
- @logger = params[:logger]
33
- @auto_reload = params[:auto_reload]
32
+ def initialize(settings = LogStash::SETTINGS)
33
+ @settings = settings
34
+ @logger = Cabin::Channel.get(LogStash)
35
+ @auto_reload = setting("config.reload.automatic")
34
36
 
35
37
  @pipelines = {}
36
- @node_name = params[:node_name] || Socket.gethostname
37
- @web_api_http_host = params[:web_api_http_host]
38
- @web_api_http_port = params[:web_api_http_port]
38
+ @node_name = setting("node.name")
39
+ @http_host = setting("http.host")
40
+ @http_port = setting("http.port")
39
41
 
40
- @config_loader = LogStash::Config::Loader.new(@logger, params[:debug_config])
41
- @reload_interval = params[:reload_interval] || 3 # seconds
42
+ @config_loader = LogStash::Config::Loader.new(@logger)
43
+ @reload_interval = setting("config.reload.interval")
42
44
  @upgrade_mutex = Mutex.new
43
45
 
44
- @collect_metric = params.fetch(:collect_metric, false)
45
- setup_metric_collection
46
+ @collect_metric = setting("metric.collect")
47
+ @metric = create_metric_collector
48
+ @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(create_metric_collector)
46
49
  end
47
50
 
48
51
  def execute
@@ -74,8 +77,11 @@ class LogStash::Agent
74
77
  # @param pipeline_id [String] pipeline string identifier
75
78
  # @param settings [Hash] settings that will be passed when creating the pipeline.
76
79
  # keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
77
- def register_pipeline(pipeline_id, settings)
78
- pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id, :metric => metric))
80
+ def register_pipeline(pipeline_id, settings = @settings)
81
+ pipeline_settings = settings.clone
82
+ pipeline_settings.set("pipeline.id", pipeline_id)
83
+
84
+ pipeline = create_pipeline(pipeline_settings)
79
85
  return unless pipeline.is_a?(LogStash::Pipeline)
80
86
  if @auto_reload && pipeline.non_reloadable_plugins.any?
81
87
  @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
@@ -123,7 +129,7 @@ class LogStash::Agent
123
129
 
124
130
  private
125
131
  def start_webserver
126
- options = {:http_host => @web_api_http_host, :http_port => @web_api_http_port }
132
+ options = {:http_host => @http_host, :http_port => @http_port }
127
133
  @webserver = LogStash::WebServer.new(@logger, options)
128
134
  Thread.new(@webserver) do |webserver|
129
135
  LogStash::Util.set_thread_name("Api Webserver")
@@ -149,28 +155,29 @@ class LogStash::Agent
149
155
  end
150
156
  end
151
157
 
152
- def setup_metric_collection
158
+ def create_metric_collector
153
159
  if collect_metrics?
154
160
  @logger.debug("Agent: Configuring metric collection")
155
161
  LogStash::Instrument::Collector.instance.agent = self
156
- @metric = LogStash::Instrument::Metric.new
162
+ LogStash::Instrument::Metric.new
157
163
  else
158
- @metric = LogStash::Instrument::NullMetric.new
164
+ LogStash::Instrument::NullMetric.new
159
165
  end
160
-
161
- @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
162
166
  end
163
167
 
164
168
  def collect_metrics?
165
169
  @collect_metric
166
170
  end
167
171
 
168
- def create_pipeline(settings)
169
- begin
170
- config = fetch_config(settings)
171
- rescue => e
172
- @logger.error("failed to fetch pipeline configuration", :message => e.message)
173
- return
172
+ def create_pipeline(settings, config=nil)
173
+
174
+ if config.nil?
175
+ begin
176
+ config = fetch_config(settings)
177
+ rescue => e
178
+ @logger.error("failed to fetch pipeline configuration", :message => e.message)
179
+ return
180
+ end
174
181
  end
175
182
 
176
183
  begin
@@ -182,23 +189,29 @@ class LogStash::Agent
182
189
  end
183
190
 
184
191
  def fetch_config(settings)
185
- @config_loader.format_config(settings[:config_path], settings[:config_string])
192
+ @config_loader.format_config(settings.get("path.config"), settings.get("config.string"))
186
193
  end
187
194
 
188
195
  # since this method modifies the @pipelines hash it is
189
196
  # wrapped in @upgrade_mutex in the parent call `reload_state!`
190
197
  def reload_pipeline!(id)
191
198
  old_pipeline = @pipelines[id]
192
- new_pipeline = create_pipeline(old_pipeline.original_settings)
199
+ new_config = fetch_config(old_pipeline.settings)
200
+ if old_pipeline.config_str == new_config
201
+ @logger.debug("no configuration change for pipeline",
202
+ :pipeline => id, :config => new_config)
203
+ return
204
+ end
205
+
206
+ new_pipeline = create_pipeline(old_pipeline.settings, new_config)
207
+
193
208
  return if new_pipeline.nil?
194
209
 
195
- if old_pipeline.config_str == new_pipeline.config_str
196
- @logger.debug("no configuration change for pipeline",
197
- :pipeline => id, :config => old_pipeline.config_str)
198
- elsif new_pipeline.non_reloadable_plugins.any?
210
+ if new_pipeline.non_reloadable_plugins.any?
199
211
  @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
200
212
  :pipeline_id => id,
201
213
  :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
214
+ return
202
215
  else
203
216
  @logger.warn("fetched new config for pipeline. upgrading..",
204
217
  :pipeline => id, :config => new_pipeline.config_str)
@@ -262,4 +275,8 @@ class LogStash::Agent
262
275
  def reset_collector
263
276
  LogStash::Instrument::Collector.instance.clear
264
277
  end
278
+
279
+ def setting(key)
280
+ @settings.get(key)
281
+ end
265
282
  end # class LogStash::Agent
@@ -0,0 +1,34 @@
1
+ # encoding: utf-8
2
+ require "logstash/api/service"
3
+ require "logstash/api/commands/system/basicinfo_command"
4
+ require "logstash/api/commands/system/plugins_command"
5
+ require "logstash/api/commands/stats"
6
+
7
+
8
+ module LogStash
9
+ module Api
10
+ class CommandFactory
11
+ attr_reader :factory, :service
12
+
13
+ def initialize(service)
14
+ @service = service
15
+ @factory = {
16
+ :system_basic_info => ::LogStash::Api::Commands::System::BasicInfo,
17
+ :plugins_command => ::LogStash::Api::Commands::System::Plugins,
18
+ :stats => ::LogStash::Api::Commands::Stats
19
+ }
20
+ end
21
+
22
+ def build(*klass_path)
23
+ # Get a nested path with args like (:parent, :child)
24
+ klass = klass_path.reduce(factory) {|acc,v| acc[v]}
25
+
26
+ if klass
27
+ klass.new(service)
28
+ else
29
+ raise ArgumentError, "Class path '#{klass_path}' does not map to command!"
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,25 @@
1
+ module LogStash
2
+ module Api
3
+ module Commands
4
+ class Base
5
+ attr_reader :service
6
+
7
+ def initialize(service = LogStash::Api::Service.instance)
8
+ @service = service
9
+ end
10
+
11
+ def hostname
12
+ service.agent.node_name
13
+ end
14
+
15
+ def uptime
16
+ service.agent.uptime
17
+ end
18
+
19
+ def started_at
20
+ (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,105 @@
1
+ require "logstash/api/commands/base"
2
+ require 'logstash/util/thread_dump'
3
+
4
+ module LogStash
5
+ module Api
6
+ module Commands
7
+ class Stats < Commands::Base
8
+
9
+ def jvm
10
+ {:threads => service.get_shallow(:jvm, :threads)}
11
+ end
12
+
13
+ def process
14
+ service.get_shallow(:jvm, :process)
15
+ end
16
+
17
+ def events
18
+ service.get_shallow(:stats, :events)
19
+ end
20
+
21
+ def memory
22
+ memory = LogStash::Json.load(service.get(:jvm_memory_stats))
23
+ {
24
+ :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
25
+ :heap_used_percent => memory["heap"]["used_percent"],
26
+ :heap_committed_in_bytes => memory["heap"]["committed_in_bytes"],
27
+ :heap_max_in_bytes => memory["heap"]["max_in_bytes"],
28
+ :heap_used_in_bytes => memory["heap"]["used_in_bytes"],
29
+ :non_heap_used_in_bytes => memory["non_heap"]["used_in_bytes"],
30
+ :non_heap_committed_in_bytes => memory["non_heap"]["committed_in_bytes"],
31
+ :pools => memory["pools"].inject({}) do |acc, (type, hash)|
32
+ hash.delete("committed_in_bytes")
33
+ acc[type] = hash
34
+ acc
35
+ end
36
+ }
37
+ end
38
+
39
+ def hot_threads(options={})
40
+ HotThreadsReport.new(self, options)
41
+ end
42
+
43
+ class HotThreadsReport
44
+ HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
45
+
46
+ def initialize(cmd, options)
47
+ @cmd = cmd
48
+ filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
49
+ jr_dump = JRMonitor.threads.generate(filter)
50
+ @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
51
+ end
52
+
53
+ def to_s
54
+ hash = to_hash
55
+ report = "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
56
+ report << '=' * 80
57
+ report << "\n"
58
+ hash[:threads].each do |thread|
59
+ thread_report = ""
60
+ thread_report = "#{I18n.t("logstash.web_api.
61
+ hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
62
+ thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
63
+ thread_report << "#{thread[:path]}\n" if thread[:path]
64
+ thread[:traces].each do |trace|
65
+ thread_report << "\t#{trace}\n"
66
+ end
67
+ report << thread_report
68
+ report << '-' * 80
69
+ report << "\n"
70
+ end
71
+ report
72
+ end
73
+
74
+ def to_hash
75
+ hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
76
+ @thread_dump.each do |thread_name, _hash|
77
+ thread_name, thread_path = _hash["thread.name"].split(": ")
78
+ thread = { :name => thread_name,
79
+ :percent_of_cpu_time => cpu_time_as_percent(_hash),
80
+ :state => _hash["thread.state"]
81
+ }
82
+ thread[:path] = thread_path if thread_path
83
+ traces = []
84
+ _hash["thread.stacktrace"].each do |trace|
85
+ traces << trace
86
+ end
87
+ thread[:traces] = traces unless traces.empty?
88
+ hash[:threads] << thread
89
+ end
90
+ hash
91
+ end
92
+
93
+ def cpu_time_as_percent(hash)
94
+ (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
95
+ end
96
+
97
+ def cpu_time(hash)
98
+ hash["cpu.time"] / 1000000.0
99
+ end
100
+
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,23 @@
1
+ # encoding: utf-8
2
+ require 'logstash/api/commands/base'
3
+ require "logstash/util/duration_formatter"
4
+
5
+ module LogStash
6
+ module Api
7
+ module Commands
8
+ module System
9
+ class BasicInfo < Commands::Base
10
+
11
+ def run
12
+ {
13
+ "hostname" => hostname,
14
+ "version" => {
15
+ "number" => LOGSTASH_VERSION
16
+ }
17
+ }
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,35 @@
1
+ # encoding: utf-8
2
+ require "logstash/api/commands/base"
3
+
4
+ module LogStash
5
+ module Api
6
+ module Commands
7
+ module System
8
+ class Plugins < Commands::Base
9
+ def run
10
+ { :total => plugins.count, :plugins => plugins }
11
+ end
12
+
13
+ private
14
+
15
+ def plugins
16
+ @plugins ||= find_plugins_gem_specs.map do |spec|
17
+ { :name => spec.name, :version => spec.version.to_s }
18
+ end.sort_by do |spec|
19
+ spec[:name]
20
+ end
21
+ end
22
+
23
+ def find_plugins_gem_specs
24
+ @specs ||= ::Gem::Specification.find_all.select{|spec| logstash_plugin_gem_spec?(spec)}
25
+ end
26
+
27
+ def logstash_plugin_gem_spec?(spec)
28
+ spec.metadata && spec.metadata["logstash_plugin"] == "true"
29
+ end
30
+
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,43 @@
1
+ require "logstash/api/app_helpers"
2
+ require "logstash/api/command_factory"
3
+
4
+ module LogStash
5
+ module Api
6
+ module Modules
7
+ class Base < ::Sinatra::Base
8
+ helpers AppHelpers
9
+
10
+ set :environment, :production
11
+
12
+ attr_reader :factory
13
+
14
+ if settings.environment != :production
15
+ set :raise_errors, true
16
+ set :show_exceptions, :after_handler
17
+ end
18
+
19
+ include LogStash::Util::Loggable
20
+
21
+ helpers AppHelpers
22
+
23
+ def initialize(app=nil)
24
+ super(app)
25
+ @factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.instance)
26
+ end
27
+
28
+ not_found do
29
+ status 404
30
+ as = params.has_key?("human") ? :string : :json
31
+ text = as == :string ? "" : {}
32
+ respond_with(text, :as => as)
33
+ end
34
+
35
+ error do
36
+ e = env['sinatra.error']
37
+ logger.error(e.message, :url => request.url, :ip => request.ip, :params => request.params, :class => e.class.name, :backtrace => e.backtrace)
38
+ end
39
+
40
+ end
41
+ end
42
+ end
43
+ end