logstash-core 2.4.1-java → 5.0.0.alpha1-java

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash-core/version.rb +1 -1
  3. data/lib/logstash/agent.rb +124 -411
  4. data/lib/logstash/api/init.ru +31 -0
  5. data/lib/logstash/api/lib/app.rb +40 -0
  6. data/lib/logstash/api/lib/app/command.rb +29 -0
  7. data/lib/logstash/api/lib/app/command_factory.rb +29 -0
  8. data/lib/logstash/api/lib/app/commands/stats/events_command.rb +13 -0
  9. data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb +120 -0
  10. data/lib/logstash/api/lib/app/commands/stats/memory_command.rb +25 -0
  11. data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb +15 -0
  12. data/lib/logstash/api/lib/app/commands/system/plugins_command.rb +28 -0
  13. data/lib/logstash/api/lib/app/modules/node.rb +25 -0
  14. data/lib/logstash/api/lib/app/modules/node_stats.rb +51 -0
  15. data/lib/logstash/api/lib/app/modules/plugins.rb +15 -0
  16. data/lib/logstash/api/lib/app/modules/stats.rb +21 -0
  17. data/lib/logstash/api/lib/app/root.rb +13 -0
  18. data/lib/logstash/api/lib/app/service.rb +61 -0
  19. data/lib/logstash/api/lib/app/stats.rb +56 -0
  20. data/lib/logstash/api/lib/helpers/app_helpers.rb +23 -0
  21. data/lib/logstash/codecs/base.rb +1 -29
  22. data/lib/logstash/config/config_ast.rb +18 -31
  23. data/lib/logstash/config/loader.rb +3 -5
  24. data/lib/logstash/config/mixin.rb +25 -64
  25. data/lib/logstash/filter_delegator.rb +65 -0
  26. data/lib/logstash/inputs/base.rb +1 -1
  27. data/lib/logstash/inputs/metrics.rb +47 -0
  28. data/lib/logstash/instrument/collector.rb +109 -0
  29. data/lib/logstash/instrument/metric.rb +102 -0
  30. data/lib/logstash/instrument/metric_store.rb +228 -0
  31. data/lib/logstash/instrument/metric_type.rb +24 -0
  32. data/lib/logstash/instrument/metric_type/base.rb +35 -0
  33. data/lib/logstash/instrument/metric_type/counter.rb +29 -0
  34. data/lib/logstash/instrument/metric_type/gauge.rb +22 -0
  35. data/lib/logstash/instrument/metric_type/mean.rb +33 -0
  36. data/lib/logstash/instrument/namespaced_metric.rb +54 -0
  37. data/lib/logstash/instrument/null_metric.rb +4 -3
  38. data/lib/logstash/instrument/periodic_poller/base.rb +57 -0
  39. data/lib/logstash/instrument/periodic_poller/jvm.rb +92 -0
  40. data/lib/logstash/instrument/periodic_poller/os.rb +13 -0
  41. data/lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb +19 -0
  42. data/lib/logstash/instrument/periodic_pollers.rb +26 -0
  43. data/lib/logstash/instrument/snapshot.rb +16 -0
  44. data/lib/logstash/json.rb +2 -3
  45. data/lib/logstash/namespace.rb +1 -0
  46. data/lib/logstash/output_delegator.rb +16 -3
  47. data/lib/logstash/outputs/base.rb +1 -32
  48. data/lib/logstash/pipeline.rb +67 -8
  49. data/lib/logstash/plugin.rb +57 -19
  50. data/lib/logstash/runner.rb +348 -84
  51. data/lib/logstash/util.rb +9 -0
  52. data/lib/logstash/util/duration_formatter.rb +15 -0
  53. data/lib/logstash/util/java_version.rb +2 -4
  54. data/lib/logstash/util/loggable.rb +29 -0
  55. data/lib/logstash/version.rb +1 -1
  56. data/lib/logstash/webserver.rb +98 -0
  57. data/locales/en.yml +42 -24
  58. data/logstash-core.gemspec +9 -6
  59. data/spec/api/lib/api/node_spec.rb +64 -0
  60. data/spec/api/lib/api/node_stats_spec.rb +68 -0
  61. data/spec/api/lib/api/plugins_spec.rb +57 -0
  62. data/spec/api/lib/api/root_spec.rb +20 -0
  63. data/spec/api/lib/api/stats_spec.rb +19 -0
  64. data/spec/api/lib/commands/events_spec.rb +17 -0
  65. data/spec/api/lib/commands/jvm_spec.rb +45 -0
  66. data/spec/api/spec_helper.rb +128 -0
  67. data/spec/logstash/agent_spec.rb +62 -169
  68. data/spec/logstash/config/config_ast_spec.rb +2 -47
  69. data/spec/logstash/config/mixin_spec.rb +0 -157
  70. data/spec/logstash/filter_delegator_spec.rb +143 -0
  71. data/spec/logstash/inputs/metrics_spec.rb +52 -0
  72. data/spec/logstash/instrument/collector_spec.rb +49 -0
  73. data/spec/logstash/instrument/metric_spec.rb +110 -0
  74. data/spec/logstash/instrument/metric_store_spec.rb +163 -0
  75. data/spec/logstash/instrument/metric_type/counter_spec.rb +40 -0
  76. data/spec/logstash/instrument/metric_type/gauge_spec.rb +40 -0
  77. data/spec/logstash/instrument/namespaced_metric_spec.rb +25 -0
  78. data/spec/logstash/instrument/null_metric_spec.rb +9 -51
  79. data/spec/logstash/json_spec.rb +14 -0
  80. data/spec/logstash/output_delegator_spec.rb +6 -3
  81. data/spec/logstash/outputs/base_spec.rb +0 -107
  82. data/spec/logstash/pipeline_spec.rb +204 -33
  83. data/spec/logstash/plugin_spec.rb +80 -15
  84. data/spec/logstash/runner_spec.rb +134 -38
  85. data/spec/logstash/shutdown_watcher_spec.rb +0 -1
  86. data/spec/logstash/util/duration_formatter_spec.rb +11 -0
  87. data/spec/logstash/util/java_version_spec.rb +10 -2
  88. data/spec/logstash/util_spec.rb +28 -0
  89. data/spec/support/matchers.rb +30 -0
  90. metadata +154 -20
  91. data/lib/logstash/logging/json.rb +0 -21
  92. data/lib/logstash/special_agent.rb +0 -8
  93. data/lib/logstash/util/safe_uri.rb +0 -50
  94. data/spec/logstash/codecs/base_spec.rb +0 -74
  95. data/spec/static/i18n_spec.rb +0 -25
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b6a7f0d38201ab4563c72f1ca25ac108ccbf391d
4
- data.tar.gz: d0c3c5b7b7da1324c6a8a6c4cd6c57d15593d304
3
+ metadata.gz: a30f7e05e09f1acd5ad8f97731d571cbb5cd2484
4
+ data.tar.gz: bae7a3ca45ab83289124f0599b5be0677ad51de5
5
5
  SHA512:
6
- metadata.gz: e6bc3903f583b5ec8f2598882a2ff481486854cf61039c92fabdedd570d8bcbe79448fac1bba57b9ee7572f8ba061d6ee59bb39a565b6fe4e93b86ac43e164e1
7
- data.tar.gz: 05fc89766e51bc2696fd009fbeaa4fa7541be9df1cf3786ae4abf1f938473a7b69cd2ba7e87fc59d6cd2728086de7f6efafbe71c69872849b5469801ad239df1
6
+ metadata.gz: 330c5e30231a116fdf8f414f30a9b6b82e48e54514740dc3edb0aecdb0a4c2b20cf879993f035c27de64b943aa1315a22acf8c0c37a462033bb03d9baa4d3839
7
+ data.tar.gz: 92155469ea8a858b5d6927522fc9663bf74a447aa5223406c1713fde5ebff13b30903930fe0f2c6320f31ecdbddcbfcd5c79e5827a63a474fff96b26f45ca743
@@ -5,4 +5,4 @@
5
5
  # Note to authors: this should not include dashes because 'gem' barfs if
6
6
  # you include a dash in the version string.
7
7
 
8
- LOGSTASH_CORE_VERSION = "2.4.1"
8
+ LOGSTASH_CORE_VERSION = "5.0.0-alpha1"
@@ -1,225 +1,64 @@
1
1
  # encoding: utf-8
2
- require "clamp" # gem 'clamp'
3
2
  require "logstash/environment"
4
3
  require "logstash/errors"
5
4
  require "logstash/config/cpu_core_strategy"
5
+ require "logstash/instrument/collector"
6
+ require "logstash/instrument/metric"
7
+ require "logstash/instrument/periodic_pollers"
8
+ require "logstash/instrument/collector"
9
+ require "logstash/instrument/metric"
10
+ require "logstash/pipeline"
11
+ require "logstash/webserver"
6
12
  require "stud/trap"
7
13
  require "logstash/config/loader"
8
14
  require "uri"
9
- require "net/http"
10
- require "logstash/pipeline"
11
-
12
- class LogStash::Agent < Clamp::Command
13
-
14
- attr_reader :pipelines, :config_loader
15
-
16
- DEFAULT_INPUT = "input { stdin { type => stdin } }"
17
- DEFAULT_OUTPUT = "output { stdout { codec => rubydebug } }"
18
-
19
- option ["-f", "--config"], "CONFIG_PATH",
20
- I18n.t("logstash.agent.flag.config"),
21
- :attribute_name => :config_path
22
-
23
- option "-e", "CONFIG_STRING",
24
- I18n.t("logstash.agent.flag.config-string",
25
- :default_input => DEFAULT_INPUT, :default_output => DEFAULT_OUTPUT),
26
- :default => "", :attribute_name => :config_string
27
-
28
- option ["-w", "--pipeline-workers"], "COUNT",
29
- I18n.t("logstash.agent.flag.pipeline-workers"),
30
- :attribute_name => :pipeline_workers,
31
- :default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
32
-
33
-
34
- option ["-b", "--pipeline-batch-size"], "SIZE",
35
- I18n.t("logstash.agent.flag.pipeline-batch-size"),
36
- :attribute_name => :pipeline_batch_size,
37
- :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_size]
38
-
39
- option ["-u", "--pipeline-batch-delay"], "DELAY_IN_MS",
40
- I18n.t("logstash.agent.flag.pipeline-batch-delay"),
41
- :attribute_name => :pipeline_batch_delay,
42
- :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_delay]
15
+ require "socket"
16
+ require "securerandom"
43
17
 
44
- option ["--filterworkers"], "COUNT",
45
- I18n.t("logstash.agent.flag.filterworkers"),
46
- :attribute_name => :filter_workers
18
+ LogStash::Environment.load_locale!
47
19
 
48
- option ["-l", "--log"], "FILE",
49
- I18n.t("logstash.agent.flag.log"),
50
- :attribute_name => :log_file
20
+ class LogStash::Agent
21
+ STARTED_AT = Time.now.freeze
51
22
 
52
- # Old support for the '-v' flag'
53
- option "-v", :flag,
54
- I18n.t("logstash.agent.flag.verbosity"),
55
- :attribute_name => :verbosity, :multivalued => true
23
+ attr_reader :metric, :node_name, :pipelines, :logger
56
24
 
57
- option "--quiet", :flag, I18n.t("logstash.agent.flag.quiet")
58
- option "--verbose", :flag, I18n.t("logstash.agent.flag.verbose")
59
- option "--debug", :flag, I18n.t("logstash.agent.flag.debug")
25
+ # initialize method for LogStash::Agent
26
+ # @param params [Hash] potential parameters are:
27
+ # :node_name [String] - identifier for the agent
28
+ # :auto_reload [Boolean] - enable reloading of pipelines
29
+ # :reload_interval [Integer] - reload pipelines every X seconds
30
+ # :logger [Cabin::Channel] - logger instance
31
+ def initialize(params)
32
+ @logger = params[:logger]
33
+ @auto_reload = params[:auto_reload]
60
34
 
61
- option ["--debug-config"], :flag,
62
- I18n.t("logstash.agent.flag.debug_config"),
63
- :attribute_name => :debug_config, :default => false
64
-
65
- option ["-V", "--version"], :flag,
66
- I18n.t("logstash.agent.flag.version")
67
-
68
- option ["-p", "--pluginpath"] , "PATH",
69
- I18n.t("logstash.agent.flag.pluginpath"),
70
- :multivalued => true,
71
- :attribute_name => :plugin_paths
72
-
73
- option ["-t", "--configtest"], :flag,
74
- I18n.t("logstash.agent.flag.configtest"),
75
- :attribute_name => :config_test
76
-
77
- option "--[no-]allow-unsafe-shutdown", :flag,
78
- I18n.t("logstash.agent.flag.unsafe_shutdown"),
79
- :attribute_name => :unsafe_shutdown,
80
- :default => false
81
-
82
- option ["-r", "--[no-]auto-reload"], :flag,
83
- I18n.t("logstash.agent.flag.auto_reload"),
84
- :attribute_name => :auto_reload, :default => false
85
-
86
- option ["--reload-interval"], "RELOAD_INTERVAL",
87
- I18n.t("logstash.agent.flag.reload_interval"),
88
- :attribute_name => :reload_interval, :default => 3, &:to_i
89
-
90
- option ["--allow-env"], :flag,
91
- I18n.t("logstash.agent.flag.allow-env"),
92
- :attribute_name => :allow_env, :default => false
93
-
94
- option ["--[no-]log-in-json"], :flag,
95
- I18n.t("logstash.agent.flag.log-in-json"),
96
- :default => false
97
-
98
- def initialize(*params)
99
- super(*params)
100
- @logger = Cabin::Channel.get(LogStash)
101
35
  @pipelines = {}
102
- @pipeline_settings ||= { :pipeline_id => "main" }
103
- @upgrade_mutex = Mutex.new
104
- @config_loader = LogStash::Config::Loader.new(@logger)
105
- end
36
+ @node_name = params[:node_name] || Socket.gethostname
37
+ @web_api_http_host = params[:web_api_http_host]
38
+ @web_api_http_port = params[:web_api_http_port]
106
39
 
107
- def pipeline_workers=(pipeline_workers_value)
108
- @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
109
- end
110
-
111
- def pipeline_batch_size=(pipeline_batch_size_value)
112
- @pipeline_settings[:pipeline_batch_size] = validate_positive_integer(pipeline_batch_size_value)
113
- end
114
-
115
- def pipeline_batch_delay=(pipeline_batch_delay_value)
116
- @pipeline_settings[:pipeline_batch_delay] = validate_positive_integer(pipeline_batch_delay_value)
117
- end
118
-
119
- def debug_config=(debug_config)
120
- @config_loader.debug_config = debug_config
121
- @debug_config = true
122
- end
123
-
124
- def validate_positive_integer(str_arg)
125
- int_arg = str_arg.to_i
126
- if str_arg !~ /^\d+$/ || int_arg < 1
127
- raise ArgumentError, "Expected a positive integer, got '#{str_arg}'"
128
- end
40
+ @config_loader = LogStash::Config::Loader.new(@logger, params[:debug_config])
41
+ @reload_interval = params[:reload_interval] || 3 # seconds
42
+ @upgrade_mutex = Mutex.new
129
43
 
130
- int_arg
44
+ @collect_metric = params.fetch(:collect_metric, false)
45
+ setup_metric_collection
131
46
  end
132
47
 
133
- # Emit a warning message.
134
- def warn(message)
135
- # For now, all warnings are fatal.
136
- signal_usage_error(message)
137
- end # def warn
138
-
139
- def fail(message)
140
- signal_usage_error(message)
141
- end # def fail
142
-
143
- # Run the agent. This method is invoked after clamp parses the
144
- # flags given to this program.
145
48
  def execute
146
- require "logstash/pipeline"
147
- require "cabin" # gem 'cabin'
148
- require "logstash/plugin"
149
- require "logstash/logging/json"
150
-
151
- LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
152
- LogStash::ShutdownWatcher.logger = @logger
153
-
154
- if version?
155
- show_version
156
- return 0
157
- end
158
-
159
- # temporarily send logs to stdout as well if a --log is specified
160
- # and stdout appears to be a tty
161
- show_startup_errors = log_file && STDOUT.tty?
162
-
163
- if show_startup_errors
164
- stdout_logs = @logger.subscribe(STDOUT)
165
- end
166
- configure
167
-
168
-
169
- if filter_workers
170
- @logger.warn("--filter-workers is deprecated! Please use --pipeline-workers or -w. This setting will be removed in the next major version!")
171
- self.pipeline_workers = filter_workers
172
- end
173
-
174
- # You must specify a config_string or config_path
175
- if config_string.nil? && config_path.nil?
176
- fail(I18n.t("logstash.agent.missing-configuration"))
177
- end
178
-
179
- if auto_reload? && config_path.nil?
180
- # there's nothing to reload
181
- fail(I18n.t("logstash.agent.reload-without-config-path"))
182
- end
183
-
184
- if config_test?
185
- config_str = @config_loader.format_config(config_path, config_string)
186
- begin
187
- # currently the best strategy to validate the configuration
188
- # is creating a pipeline instance and checking for exceptions
189
- LogStash::Pipeline.new(config_str)
190
- @logger.terminal "Configuration OK"
191
- return 0
192
- rescue => e
193
- @logger.fatal I18n.t("logstash.agent.invalid-configuration", :error => e.message)
194
- return 1
195
- end
196
- end
197
-
198
- register_pipeline("main", @pipeline_settings.merge({
199
- :config_string => config_string,
200
- :config_path => config_path,
201
- :debug_config => debug_config?,
202
- :allow_env => allow_env?
203
- }))
204
-
205
49
  @thread = Thread.current # this var is implicilty used by Stud.stop?
206
-
207
- sigint_id = trap_sigint()
208
- sigterm_id = trap_sigterm()
209
- sighup_id = trap_sighup()
210
-
211
- @logger.unsubscribe(stdout_logs) if show_startup_errors
212
-
213
50
  @logger.info("starting agent")
214
51
 
52
+ start_background_services
215
53
  start_pipelines
54
+ start_webserver
216
55
 
217
56
  return 1 if clean_state?
218
57
 
219
- Stud.stoppable_sleep(reload_interval) # sleep before looping
58
+ Stud.stoppable_sleep(@reload_interval) # sleep before looping
220
59
 
221
- if auto_reload?
222
- Stud.interval(reload_interval) { reload_state! }
60
+ if @auto_reload
61
+ Stud.interval(@reload_interval) { reload_state! }
223
62
  else
224
63
  while !Stud.stop?
225
64
  if clean_state? || running_pipelines?
@@ -229,149 +68,14 @@ class LogStash::Agent < Clamp::Command
229
68
  end
230
69
  end
231
70
  end
232
-
233
- shutdown
234
-
235
- return 0
236
- rescue LogStash::ConfigurationError => e
237
- @logger.unsubscribe(stdout_logs) if show_startup_errors
238
- @logger.error I18n.t("logstash.agent.error", :error => e)
239
- if !config_test?
240
- @logger.info I18n.t("logstash.agent.configtest-flag-information")
241
- end
242
- return 1
243
- rescue => e
244
- if show_startup_errors
245
- @logger.terminal(e.message)
246
- @logger.unsubscribe(stdout_logs)
247
- end
248
- @logger.warn(I18n.t("oops"), :error => e.message, :class => e.class.name, :backtrace => e.backtrace)
249
- return 1
250
- ensure
251
- @log_fd.close if @log_fd
252
- Stud::untrap("INT", sigint_id) unless sigint_id.nil?
253
- Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
254
- Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
255
- end # def execute
256
-
257
-
258
- # Do any start-time configuration.
259
- #
260
- # Log file stuff, plugin path checking, etc.
261
- def configure
262
- configure_logging(log_file)
263
- configure_plugin_paths(plugin_paths)
264
- end # def configure
265
-
266
- # Point logging at a specific path.
267
- def configure_logging(path)
268
- # Set with the -v (or -vv...) flag
269
- if quiet?
270
- @logger.level = :error
271
- elsif verbose?
272
- @logger.level = :info
273
- elsif debug?
274
- @logger.level = :debug
275
- else
276
- # Old support for the -v and -vv stuff.
277
- if verbosity? && verbosity?.any?
278
- # this is an array with length of how many times the flag is given
279
- if verbosity?.length == 1
280
- @logger.warn("The -v flag is deprecated and will be removed in a future release. You should use --verbose instead.")
281
- @logger.level = :info
282
- else
283
- @logger.warn("The -vv flag is deprecated and will be removed in a future release. You should use --debug instead.")
284
- @logger.level = :debug
285
- end
286
- else
287
- @logger.level = :warn
288
- end
289
- end
290
-
291
- if log_file
292
- # TODO(sissel): Implement file output/rotation in Cabin.
293
- # TODO(sissel): Catch exceptions, report sane errors.
294
- begin
295
- @log_fd.close if @log_fd
296
- @log_fd = File.new(path, "a")
297
- rescue => e
298
- fail(I18n.t("logstash.agent.configuration.log_file_failed",
299
- :path => path, :error => e))
300
- end
301
-
302
- puts "Sending logstash logs to #{path}."
303
- @logger.unsubscribe(@logger_subscription) if @logger_subscription
304
- if log_in_json?
305
- @logger_subscription = @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
306
- @logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
307
- else
308
- @logger_subscription = @logger.subscribe(@log_fd)
309
- @logger.subscribe(STDOUT, :level => :fatal)
310
- end
311
- else
312
- if log_in_json?
313
- @logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
314
- else
315
- @logger.subscribe(STDOUT)
316
- end
317
- end
318
-
319
-
320
- # TODO(sissel): redirect stdout/stderr to the log as well
321
- # http://jira.codehaus.org/browse/JRUBY-7003
322
- end # def configure_logging
323
-
324
- # add the given paths for ungemified/bare plugins lookups
325
- # @param paths [String, Array<String>] plugins path string or list of path strings to add
326
- def configure_plugin_paths(paths)
327
- Array(paths).each do |path|
328
- fail(I18n.t("logstash.agent.configuration.plugin_path_missing", :path => path)) unless File.directory?(path)
329
- LogStash::Environment.add_plugin_path(path)
330
- end
331
- end
332
-
333
- ## Signal Trapping ##
334
- def trap_sigint
335
- Stud::trap("INT") do
336
- if @interrupted_once
337
- @logger.fatal(I18n.t("logstash.agent.forced_sigint"))
338
- exit
339
- else
340
- @logger.warn(I18n.t("logstash.agent.sigint"))
341
- Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
342
- @interrupted_once = true
343
- Stud.stop!(@thread)
344
- end
345
- end
346
- end
347
-
348
- def trap_sigterm
349
- Stud::trap("TERM") do
350
- @logger.warn(I18n.t("logstash.agent.sigterm"))
351
- Stud.stop!(@thread)
352
- end
353
- end
354
-
355
- def trap_sighup
356
- Stud::trap("HUP") do
357
- @logger.warn(I18n.t("logstash.agent.sighup"))
358
- reload_state!
359
- end
360
71
  end
361
72
 
362
- ## Pipeline CRUD ##
363
- def shutdown(pipeline)
364
- pipeline.shutdown do
365
- ::LogStash::ShutdownWatcher.start(pipeline)
366
- end
367
- end
368
- #
369
73
  # register_pipeline - adds a pipeline to the agent's state
370
74
  # @param pipeline_id [String] pipeline string identifier
371
75
  # @param settings [Hash] settings that will be passed when creating the pipeline.
372
76
  # keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
373
77
  def register_pipeline(pipeline_id, settings)
374
- pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id))
78
+ pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id, :metric => metric))
375
79
  return unless pipeline.is_a?(LogStash::Pipeline)
376
80
  if @auto_reload && pipeline.non_reloadable_plugins.any?
377
81
  @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
@@ -388,30 +92,27 @@ class LogStash::Agent < Clamp::Command
388
92
  begin
389
93
  reload_pipeline!(pipeline_id)
390
94
  rescue => e
391
- @logger.error(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
95
+ @logger.error(I18n.t("oops"), :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
392
96
  end
393
97
  end
394
98
  end
395
99
  end
396
100
 
397
- def start_pipelines
398
- @pipelines.each { |id, _| start_pipeline(id) }
101
+ # Calculate the Logstash uptime in milliseconds
102
+ #
103
+ # @return [Fixnum] Uptime in milliseconds
104
+ def uptime
105
+ ((Time.now.to_f - STARTED_AT.to_f) * 1000.0).to_i
399
106
  end
400
107
 
401
108
  def shutdown
109
+ stop_background_services
110
+ stop_webserver
402
111
  shutdown_pipelines
403
112
  end
404
113
 
405
- def shutdown_pipelines
406
- @pipelines.each { |id, _| stop_pipeline(id) }
407
- end
408
-
409
- def stop_pipeline(id)
410
- pipeline = @pipelines[id]
411
- return unless pipeline
412
- @logger.log("stopping pipeline", :id => id)
413
- pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
414
- @pipelines[id].thread.join
114
+ def node_uuid
115
+ @node_uuid ||= SecureRandom.uuid
415
116
  end
416
117
 
417
118
  def running_pipelines?
@@ -420,26 +121,56 @@ class LogStash::Agent < Clamp::Command
420
121
  end
421
122
  end
422
123
 
423
- def running_pipeline?(pipeline_id)
424
- thread = @pipelines[pipeline_id].thread
425
- thread.is_a?(Thread) && thread.alive?
124
+ private
125
+ def start_webserver
126
+ options = {:http_host => @web_api_http_host, :http_port => @web_api_http_port }
127
+ @webserver = LogStash::WebServer.new(@logger, options)
128
+ Thread.new(@webserver) do |webserver|
129
+ LogStash::Util.set_thread_name("Api Webserver")
130
+ webserver.run
131
+ end
426
132
  end
427
133
 
428
- def upgrade_pipeline(pipeline_id, new_pipeline)
429
- stop_pipeline(pipeline_id)
430
- @pipelines[pipeline_id] = new_pipeline
431
- start_pipeline(pipeline_id)
134
+ def stop_webserver
135
+ @webserver.stop if @webserver
432
136
  end
433
137
 
434
- def create_pipeline(settings, config=nil)
138
+ def start_background_services
139
+ if collect_metrics?
140
+ @logger.debug("Agent: Starting metric periodic pollers")
141
+ @periodic_pollers.start
142
+ end
143
+ end
435
144
 
436
- if config.nil?
437
- begin
438
- config = fetch_config(settings)
439
- rescue => e
440
- @logger.error("failed to fetch pipeline configuration", :message => e.message)
441
- return
442
- end
145
+ def stop_background_services
146
+ if collect_metrics?
147
+ @logger.debug("Agent: Stopping metric periodic pollers")
148
+ @periodic_pollers.stop
149
+ end
150
+ end
151
+
152
+ def setup_metric_collection
153
+ if collect_metrics?
154
+ @logger.debug("Agent: Configuring metric collection")
155
+ LogStash::Instrument::Collector.instance.agent = self
156
+ @metric = LogStash::Instrument::Metric.new
157
+ else
158
+ @metric = LogStash::Instrument::NullMetric.new
159
+ end
160
+
161
+ @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
162
+ end
163
+
164
+ def collect_metrics?
165
+ @collect_metric
166
+ end
167
+
168
+ def create_pipeline(settings)
169
+ begin
170
+ config = fetch_config(settings)
171
+ rescue => e
172
+ @logger.error("failed to fetch pipeline configuration", :message => e.message)
173
+ return
443
174
  end
444
175
 
445
176
  begin
@@ -450,31 +181,26 @@ class LogStash::Agent < Clamp::Command
450
181
  end
451
182
  end
452
183
 
453
- def clean_state?
454
- @pipelines.empty?
184
+ def fetch_config(settings)
185
+ @config_loader.format_config(settings[:config_path], settings[:config_string])
455
186
  end
456
187
 
457
188
  # since this method modifies the @pipelines hash it is
458
189
  # wrapped in @upgrade_mutex in the parent call `reload_state!`
459
190
  def reload_pipeline!(id)
460
191
  old_pipeline = @pipelines[id]
461
- new_config = fetch_config(old_pipeline.original_settings)
462
- if old_pipeline.config_str == new_config
463
- @logger.debug("no configuration change for pipeline",
464
- :pipeline => id, :config => new_config)
465
- return
466
- end
467
-
468
- new_pipeline = create_pipeline(old_pipeline.original_settings, new_config)
192
+ new_pipeline = create_pipeline(old_pipeline.original_settings)
469
193
  return if new_pipeline.nil?
470
194
 
471
- if new_pipeline.non_reloadable_plugins.any?
195
+ if old_pipeline.config_str == new_pipeline.config_str
196
+ @logger.debug("no configuration change for pipeline",
197
+ :pipeline => id, :config => old_pipeline.config_str)
198
+ elsif new_pipeline.non_reloadable_plugins.any?
472
199
  @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
473
200
  :pipeline_id => id,
474
201
  :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
475
- return
476
202
  else
477
- @logger.log("fetched new config for pipeline. upgrading..",
203
+ @logger.warn("fetched new config for pipeline. upgrading..",
478
204
  :pipeline => id, :config => new_pipeline.config_str)
479
205
  upgrade_pipeline(id, new_pipeline)
480
206
  end
@@ -490,53 +216,40 @@ class LogStash::Agent < Clamp::Command
490
216
  begin
491
217
  pipeline.run
492
218
  rescue => e
493
- @logger.error("Pipeline aborted due to error", :exception => e.class.name, :error => e.message, :backtrace => e.backtrace)
219
+ @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
494
220
  end
495
221
  end
496
222
  sleep 0.01 until pipeline.ready?
497
223
  end
498
224
 
499
- ## Pipeline Aux methods ##
500
- def fetch_config(settings)
501
- @config_loader.format_config(settings[:config_path], settings[:config_string])
225
+ def stop_pipeline(id)
226
+ pipeline = @pipelines[id]
227
+ return unless pipeline
228
+ @logger.warn("stopping pipeline", :id => id)
229
+ pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
230
+ @pipelines[id].thread.join
502
231
  end
503
232
 
504
- private
505
- def node_uuid
506
- @node_uuid ||= SecureRandom.uuid
233
+ def start_pipelines
234
+ @pipelines.each { |id, _| start_pipeline(id) }
507
235
  end
508
236
 
509
- ### Version actions ###
510
- def show_version
511
- show_version_logstash
237
+ def shutdown_pipelines
238
+ @pipelines.each { |id, _| stop_pipeline(id) }
239
+ end
512
240
 
513
- if [:info, :debug].include?(verbosity?) || debug? || verbose?
514
- show_version_ruby
515
- show_version_java if LogStash::Environment.jruby?
516
- show_gems if [:debug].include?(verbosity?) || debug?
517
- end
518
- end # def show_version
519
-
520
- def show_version_logstash
521
- require "logstash/version"
522
- puts "logstash #{LOGSTASH_VERSION}"
523
- end # def show_version_logstash
524
-
525
- def show_version_ruby
526
- puts RUBY_DESCRIPTION
527
- end # def show_version_ruby
528
-
529
- def show_version_java
530
- properties = java.lang.System.getProperties
531
- puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
532
- puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
533
- end # def show_version_java
534
-
535
- def show_gems
536
- require "rubygems"
537
- Gem::Specification.each do |spec|
538
- puts "gem #{spec.name} #{spec.version}"
539
- end
540
- end # def show_gems
241
+ def running_pipeline?(pipeline_id)
242
+ thread = @pipelines[pipeline_id].thread
243
+ thread.is_a?(Thread) && thread.alive?
244
+ end
541
245
 
246
+ def upgrade_pipeline(pipeline_id, new_pipeline)
247
+ stop_pipeline(pipeline_id)
248
+ @pipelines[pipeline_id] = new_pipeline
249
+ start_pipeline(pipeline_id)
250
+ end
251
+
252
+ def clean_state?
253
+ @pipelines.empty?
254
+ end
542
255
  end # class LogStash::Agent