logstash-core 2.4.1-java → 5.0.0.alpha1-java
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of logstash-core might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/agent.rb +124 -411
- data/lib/logstash/api/init.ru +31 -0
- data/lib/logstash/api/lib/app.rb +40 -0
- data/lib/logstash/api/lib/app/command.rb +29 -0
- data/lib/logstash/api/lib/app/command_factory.rb +29 -0
- data/lib/logstash/api/lib/app/commands/stats/events_command.rb +13 -0
- data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb +120 -0
- data/lib/logstash/api/lib/app/commands/stats/memory_command.rb +25 -0
- data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb +15 -0
- data/lib/logstash/api/lib/app/commands/system/plugins_command.rb +28 -0
- data/lib/logstash/api/lib/app/modules/node.rb +25 -0
- data/lib/logstash/api/lib/app/modules/node_stats.rb +51 -0
- data/lib/logstash/api/lib/app/modules/plugins.rb +15 -0
- data/lib/logstash/api/lib/app/modules/stats.rb +21 -0
- data/lib/logstash/api/lib/app/root.rb +13 -0
- data/lib/logstash/api/lib/app/service.rb +61 -0
- data/lib/logstash/api/lib/app/stats.rb +56 -0
- data/lib/logstash/api/lib/helpers/app_helpers.rb +23 -0
- data/lib/logstash/codecs/base.rb +1 -29
- data/lib/logstash/config/config_ast.rb +18 -31
- data/lib/logstash/config/loader.rb +3 -5
- data/lib/logstash/config/mixin.rb +25 -64
- data/lib/logstash/filter_delegator.rb +65 -0
- data/lib/logstash/inputs/base.rb +1 -1
- data/lib/logstash/inputs/metrics.rb +47 -0
- data/lib/logstash/instrument/collector.rb +109 -0
- data/lib/logstash/instrument/metric.rb +102 -0
- data/lib/logstash/instrument/metric_store.rb +228 -0
- data/lib/logstash/instrument/metric_type.rb +24 -0
- data/lib/logstash/instrument/metric_type/base.rb +35 -0
- data/lib/logstash/instrument/metric_type/counter.rb +29 -0
- data/lib/logstash/instrument/metric_type/gauge.rb +22 -0
- data/lib/logstash/instrument/metric_type/mean.rb +33 -0
- data/lib/logstash/instrument/namespaced_metric.rb +54 -0
- data/lib/logstash/instrument/null_metric.rb +4 -3
- data/lib/logstash/instrument/periodic_poller/base.rb +57 -0
- data/lib/logstash/instrument/periodic_poller/jvm.rb +92 -0
- data/lib/logstash/instrument/periodic_poller/os.rb +13 -0
- data/lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb +19 -0
- data/lib/logstash/instrument/periodic_pollers.rb +26 -0
- data/lib/logstash/instrument/snapshot.rb +16 -0
- data/lib/logstash/json.rb +2 -3
- data/lib/logstash/namespace.rb +1 -0
- data/lib/logstash/output_delegator.rb +16 -3
- data/lib/logstash/outputs/base.rb +1 -32
- data/lib/logstash/pipeline.rb +67 -8
- data/lib/logstash/plugin.rb +57 -19
- data/lib/logstash/runner.rb +348 -84
- data/lib/logstash/util.rb +9 -0
- data/lib/logstash/util/duration_formatter.rb +15 -0
- data/lib/logstash/util/java_version.rb +2 -4
- data/lib/logstash/util/loggable.rb +29 -0
- data/lib/logstash/version.rb +1 -1
- data/lib/logstash/webserver.rb +98 -0
- data/locales/en.yml +42 -24
- data/logstash-core.gemspec +9 -6
- data/spec/api/lib/api/node_spec.rb +64 -0
- data/spec/api/lib/api/node_stats_spec.rb +68 -0
- data/spec/api/lib/api/plugins_spec.rb +57 -0
- data/spec/api/lib/api/root_spec.rb +20 -0
- data/spec/api/lib/api/stats_spec.rb +19 -0
- data/spec/api/lib/commands/events_spec.rb +17 -0
- data/spec/api/lib/commands/jvm_spec.rb +45 -0
- data/spec/api/spec_helper.rb +128 -0
- data/spec/logstash/agent_spec.rb +62 -169
- data/spec/logstash/config/config_ast_spec.rb +2 -47
- data/spec/logstash/config/mixin_spec.rb +0 -157
- data/spec/logstash/filter_delegator_spec.rb +143 -0
- data/spec/logstash/inputs/metrics_spec.rb +52 -0
- data/spec/logstash/instrument/collector_spec.rb +49 -0
- data/spec/logstash/instrument/metric_spec.rb +110 -0
- data/spec/logstash/instrument/metric_store_spec.rb +163 -0
- data/spec/logstash/instrument/metric_type/counter_spec.rb +40 -0
- data/spec/logstash/instrument/metric_type/gauge_spec.rb +40 -0
- data/spec/logstash/instrument/namespaced_metric_spec.rb +25 -0
- data/spec/logstash/instrument/null_metric_spec.rb +9 -51
- data/spec/logstash/json_spec.rb +14 -0
- data/spec/logstash/output_delegator_spec.rb +6 -3
- data/spec/logstash/outputs/base_spec.rb +0 -107
- data/spec/logstash/pipeline_spec.rb +204 -33
- data/spec/logstash/plugin_spec.rb +80 -15
- data/spec/logstash/runner_spec.rb +134 -38
- data/spec/logstash/shutdown_watcher_spec.rb +0 -1
- data/spec/logstash/util/duration_formatter_spec.rb +11 -0
- data/spec/logstash/util/java_version_spec.rb +10 -2
- data/spec/logstash/util_spec.rb +28 -0
- data/spec/support/matchers.rb +30 -0
- metadata +154 -20
- data/lib/logstash/logging/json.rb +0 -21
- data/lib/logstash/special_agent.rb +0 -8
- data/lib/logstash/util/safe_uri.rb +0 -50
- data/spec/logstash/codecs/base_spec.rb +0 -74
- data/spec/static/i18n_spec.rb +0 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a30f7e05e09f1acd5ad8f97731d571cbb5cd2484
|
4
|
+
data.tar.gz: bae7a3ca45ab83289124f0599b5be0677ad51de5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 330c5e30231a116fdf8f414f30a9b6b82e48e54514740dc3edb0aecdb0a4c2b20cf879993f035c27de64b943aa1315a22acf8c0c37a462033bb03d9baa4d3839
|
7
|
+
data.tar.gz: 92155469ea8a858b5d6927522fc9663bf74a447aa5223406c1713fde5ebff13b30903930fe0f2c6320f31ecdbddcbfcd5c79e5827a63a474fff96b26f45ca743
|
data/lib/logstash/agent.rb
CHANGED
@@ -1,225 +1,64 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "clamp" # gem 'clamp'
|
3
2
|
require "logstash/environment"
|
4
3
|
require "logstash/errors"
|
5
4
|
require "logstash/config/cpu_core_strategy"
|
5
|
+
require "logstash/instrument/collector"
|
6
|
+
require "logstash/instrument/metric"
|
7
|
+
require "logstash/instrument/periodic_pollers"
|
8
|
+
require "logstash/instrument/collector"
|
9
|
+
require "logstash/instrument/metric"
|
10
|
+
require "logstash/pipeline"
|
11
|
+
require "logstash/webserver"
|
6
12
|
require "stud/trap"
|
7
13
|
require "logstash/config/loader"
|
8
14
|
require "uri"
|
9
|
-
require "
|
10
|
-
require "
|
11
|
-
|
12
|
-
class LogStash::Agent < Clamp::Command
|
13
|
-
|
14
|
-
attr_reader :pipelines, :config_loader
|
15
|
-
|
16
|
-
DEFAULT_INPUT = "input { stdin { type => stdin } }"
|
17
|
-
DEFAULT_OUTPUT = "output { stdout { codec => rubydebug } }"
|
18
|
-
|
19
|
-
option ["-f", "--config"], "CONFIG_PATH",
|
20
|
-
I18n.t("logstash.agent.flag.config"),
|
21
|
-
:attribute_name => :config_path
|
22
|
-
|
23
|
-
option "-e", "CONFIG_STRING",
|
24
|
-
I18n.t("logstash.agent.flag.config-string",
|
25
|
-
:default_input => DEFAULT_INPUT, :default_output => DEFAULT_OUTPUT),
|
26
|
-
:default => "", :attribute_name => :config_string
|
27
|
-
|
28
|
-
option ["-w", "--pipeline-workers"], "COUNT",
|
29
|
-
I18n.t("logstash.agent.flag.pipeline-workers"),
|
30
|
-
:attribute_name => :pipeline_workers,
|
31
|
-
:default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
|
32
|
-
|
33
|
-
|
34
|
-
option ["-b", "--pipeline-batch-size"], "SIZE",
|
35
|
-
I18n.t("logstash.agent.flag.pipeline-batch-size"),
|
36
|
-
:attribute_name => :pipeline_batch_size,
|
37
|
-
:default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_size]
|
38
|
-
|
39
|
-
option ["-u", "--pipeline-batch-delay"], "DELAY_IN_MS",
|
40
|
-
I18n.t("logstash.agent.flag.pipeline-batch-delay"),
|
41
|
-
:attribute_name => :pipeline_batch_delay,
|
42
|
-
:default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_delay]
|
15
|
+
require "socket"
|
16
|
+
require "securerandom"
|
43
17
|
|
44
|
-
|
45
|
-
I18n.t("logstash.agent.flag.filterworkers"),
|
46
|
-
:attribute_name => :filter_workers
|
18
|
+
LogStash::Environment.load_locale!
|
47
19
|
|
48
|
-
|
49
|
-
|
50
|
-
:attribute_name => :log_file
|
20
|
+
class LogStash::Agent
|
21
|
+
STARTED_AT = Time.now.freeze
|
51
22
|
|
52
|
-
|
53
|
-
option "-v", :flag,
|
54
|
-
I18n.t("logstash.agent.flag.verbosity"),
|
55
|
-
:attribute_name => :verbosity, :multivalued => true
|
23
|
+
attr_reader :metric, :node_name, :pipelines, :logger
|
56
24
|
|
57
|
-
|
58
|
-
|
59
|
-
|
25
|
+
# initialize method for LogStash::Agent
|
26
|
+
# @param params [Hash] potential parameters are:
|
27
|
+
# :node_name [String] - identifier for the agent
|
28
|
+
# :auto_reload [Boolean] - enable reloading of pipelines
|
29
|
+
# :reload_interval [Integer] - reload pipelines every X seconds
|
30
|
+
# :logger [Cabin::Channel] - logger instance
|
31
|
+
def initialize(params)
|
32
|
+
@logger = params[:logger]
|
33
|
+
@auto_reload = params[:auto_reload]
|
60
34
|
|
61
|
-
option ["--debug-config"], :flag,
|
62
|
-
I18n.t("logstash.agent.flag.debug_config"),
|
63
|
-
:attribute_name => :debug_config, :default => false
|
64
|
-
|
65
|
-
option ["-V", "--version"], :flag,
|
66
|
-
I18n.t("logstash.agent.flag.version")
|
67
|
-
|
68
|
-
option ["-p", "--pluginpath"] , "PATH",
|
69
|
-
I18n.t("logstash.agent.flag.pluginpath"),
|
70
|
-
:multivalued => true,
|
71
|
-
:attribute_name => :plugin_paths
|
72
|
-
|
73
|
-
option ["-t", "--configtest"], :flag,
|
74
|
-
I18n.t("logstash.agent.flag.configtest"),
|
75
|
-
:attribute_name => :config_test
|
76
|
-
|
77
|
-
option "--[no-]allow-unsafe-shutdown", :flag,
|
78
|
-
I18n.t("logstash.agent.flag.unsafe_shutdown"),
|
79
|
-
:attribute_name => :unsafe_shutdown,
|
80
|
-
:default => false
|
81
|
-
|
82
|
-
option ["-r", "--[no-]auto-reload"], :flag,
|
83
|
-
I18n.t("logstash.agent.flag.auto_reload"),
|
84
|
-
:attribute_name => :auto_reload, :default => false
|
85
|
-
|
86
|
-
option ["--reload-interval"], "RELOAD_INTERVAL",
|
87
|
-
I18n.t("logstash.agent.flag.reload_interval"),
|
88
|
-
:attribute_name => :reload_interval, :default => 3, &:to_i
|
89
|
-
|
90
|
-
option ["--allow-env"], :flag,
|
91
|
-
I18n.t("logstash.agent.flag.allow-env"),
|
92
|
-
:attribute_name => :allow_env, :default => false
|
93
|
-
|
94
|
-
option ["--[no-]log-in-json"], :flag,
|
95
|
-
I18n.t("logstash.agent.flag.log-in-json"),
|
96
|
-
:default => false
|
97
|
-
|
98
|
-
def initialize(*params)
|
99
|
-
super(*params)
|
100
|
-
@logger = Cabin::Channel.get(LogStash)
|
101
35
|
@pipelines = {}
|
102
|
-
@
|
103
|
-
@
|
104
|
-
@
|
105
|
-
end
|
36
|
+
@node_name = params[:node_name] || Socket.gethostname
|
37
|
+
@web_api_http_host = params[:web_api_http_host]
|
38
|
+
@web_api_http_port = params[:web_api_http_port]
|
106
39
|
|
107
|
-
|
108
|
-
@
|
109
|
-
|
110
|
-
|
111
|
-
def pipeline_batch_size=(pipeline_batch_size_value)
|
112
|
-
@pipeline_settings[:pipeline_batch_size] = validate_positive_integer(pipeline_batch_size_value)
|
113
|
-
end
|
114
|
-
|
115
|
-
def pipeline_batch_delay=(pipeline_batch_delay_value)
|
116
|
-
@pipeline_settings[:pipeline_batch_delay] = validate_positive_integer(pipeline_batch_delay_value)
|
117
|
-
end
|
118
|
-
|
119
|
-
def debug_config=(debug_config)
|
120
|
-
@config_loader.debug_config = debug_config
|
121
|
-
@debug_config = true
|
122
|
-
end
|
123
|
-
|
124
|
-
def validate_positive_integer(str_arg)
|
125
|
-
int_arg = str_arg.to_i
|
126
|
-
if str_arg !~ /^\d+$/ || int_arg < 1
|
127
|
-
raise ArgumentError, "Expected a positive integer, got '#{str_arg}'"
|
128
|
-
end
|
40
|
+
@config_loader = LogStash::Config::Loader.new(@logger, params[:debug_config])
|
41
|
+
@reload_interval = params[:reload_interval] || 3 # seconds
|
42
|
+
@upgrade_mutex = Mutex.new
|
129
43
|
|
130
|
-
|
44
|
+
@collect_metric = params.fetch(:collect_metric, false)
|
45
|
+
setup_metric_collection
|
131
46
|
end
|
132
47
|
|
133
|
-
# Emit a warning message.
|
134
|
-
def warn(message)
|
135
|
-
# For now, all warnings are fatal.
|
136
|
-
signal_usage_error(message)
|
137
|
-
end # def warn
|
138
|
-
|
139
|
-
def fail(message)
|
140
|
-
signal_usage_error(message)
|
141
|
-
end # def fail
|
142
|
-
|
143
|
-
# Run the agent. This method is invoked after clamp parses the
|
144
|
-
# flags given to this program.
|
145
48
|
def execute
|
146
|
-
require "logstash/pipeline"
|
147
|
-
require "cabin" # gem 'cabin'
|
148
|
-
require "logstash/plugin"
|
149
|
-
require "logstash/logging/json"
|
150
|
-
|
151
|
-
LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
|
152
|
-
LogStash::ShutdownWatcher.logger = @logger
|
153
|
-
|
154
|
-
if version?
|
155
|
-
show_version
|
156
|
-
return 0
|
157
|
-
end
|
158
|
-
|
159
|
-
# temporarily send logs to stdout as well if a --log is specified
|
160
|
-
# and stdout appears to be a tty
|
161
|
-
show_startup_errors = log_file && STDOUT.tty?
|
162
|
-
|
163
|
-
if show_startup_errors
|
164
|
-
stdout_logs = @logger.subscribe(STDOUT)
|
165
|
-
end
|
166
|
-
configure
|
167
|
-
|
168
|
-
|
169
|
-
if filter_workers
|
170
|
-
@logger.warn("--filter-workers is deprecated! Please use --pipeline-workers or -w. This setting will be removed in the next major version!")
|
171
|
-
self.pipeline_workers = filter_workers
|
172
|
-
end
|
173
|
-
|
174
|
-
# You must specify a config_string or config_path
|
175
|
-
if config_string.nil? && config_path.nil?
|
176
|
-
fail(I18n.t("logstash.agent.missing-configuration"))
|
177
|
-
end
|
178
|
-
|
179
|
-
if auto_reload? && config_path.nil?
|
180
|
-
# there's nothing to reload
|
181
|
-
fail(I18n.t("logstash.agent.reload-without-config-path"))
|
182
|
-
end
|
183
|
-
|
184
|
-
if config_test?
|
185
|
-
config_str = @config_loader.format_config(config_path, config_string)
|
186
|
-
begin
|
187
|
-
# currently the best strategy to validate the configuration
|
188
|
-
# is creating a pipeline instance and checking for exceptions
|
189
|
-
LogStash::Pipeline.new(config_str)
|
190
|
-
@logger.terminal "Configuration OK"
|
191
|
-
return 0
|
192
|
-
rescue => e
|
193
|
-
@logger.fatal I18n.t("logstash.agent.invalid-configuration", :error => e.message)
|
194
|
-
return 1
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
register_pipeline("main", @pipeline_settings.merge({
|
199
|
-
:config_string => config_string,
|
200
|
-
:config_path => config_path,
|
201
|
-
:debug_config => debug_config?,
|
202
|
-
:allow_env => allow_env?
|
203
|
-
}))
|
204
|
-
|
205
49
|
@thread = Thread.current # this var is implicilty used by Stud.stop?
|
206
|
-
|
207
|
-
sigint_id = trap_sigint()
|
208
|
-
sigterm_id = trap_sigterm()
|
209
|
-
sighup_id = trap_sighup()
|
210
|
-
|
211
|
-
@logger.unsubscribe(stdout_logs) if show_startup_errors
|
212
|
-
|
213
50
|
@logger.info("starting agent")
|
214
51
|
|
52
|
+
start_background_services
|
215
53
|
start_pipelines
|
54
|
+
start_webserver
|
216
55
|
|
217
56
|
return 1 if clean_state?
|
218
57
|
|
219
|
-
Stud.stoppable_sleep(reload_interval) # sleep before looping
|
58
|
+
Stud.stoppable_sleep(@reload_interval) # sleep before looping
|
220
59
|
|
221
|
-
if auto_reload
|
222
|
-
Stud.interval(reload_interval) { reload_state! }
|
60
|
+
if @auto_reload
|
61
|
+
Stud.interval(@reload_interval) { reload_state! }
|
223
62
|
else
|
224
63
|
while !Stud.stop?
|
225
64
|
if clean_state? || running_pipelines?
|
@@ -229,149 +68,14 @@ class LogStash::Agent < Clamp::Command
|
|
229
68
|
end
|
230
69
|
end
|
231
70
|
end
|
232
|
-
|
233
|
-
shutdown
|
234
|
-
|
235
|
-
return 0
|
236
|
-
rescue LogStash::ConfigurationError => e
|
237
|
-
@logger.unsubscribe(stdout_logs) if show_startup_errors
|
238
|
-
@logger.error I18n.t("logstash.agent.error", :error => e)
|
239
|
-
if !config_test?
|
240
|
-
@logger.info I18n.t("logstash.agent.configtest-flag-information")
|
241
|
-
end
|
242
|
-
return 1
|
243
|
-
rescue => e
|
244
|
-
if show_startup_errors
|
245
|
-
@logger.terminal(e.message)
|
246
|
-
@logger.unsubscribe(stdout_logs)
|
247
|
-
end
|
248
|
-
@logger.warn(I18n.t("oops"), :error => e.message, :class => e.class.name, :backtrace => e.backtrace)
|
249
|
-
return 1
|
250
|
-
ensure
|
251
|
-
@log_fd.close if @log_fd
|
252
|
-
Stud::untrap("INT", sigint_id) unless sigint_id.nil?
|
253
|
-
Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
|
254
|
-
Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
|
255
|
-
end # def execute
|
256
|
-
|
257
|
-
|
258
|
-
# Do any start-time configuration.
|
259
|
-
#
|
260
|
-
# Log file stuff, plugin path checking, etc.
|
261
|
-
def configure
|
262
|
-
configure_logging(log_file)
|
263
|
-
configure_plugin_paths(plugin_paths)
|
264
|
-
end # def configure
|
265
|
-
|
266
|
-
# Point logging at a specific path.
|
267
|
-
def configure_logging(path)
|
268
|
-
# Set with the -v (or -vv...) flag
|
269
|
-
if quiet?
|
270
|
-
@logger.level = :error
|
271
|
-
elsif verbose?
|
272
|
-
@logger.level = :info
|
273
|
-
elsif debug?
|
274
|
-
@logger.level = :debug
|
275
|
-
else
|
276
|
-
# Old support for the -v and -vv stuff.
|
277
|
-
if verbosity? && verbosity?.any?
|
278
|
-
# this is an array with length of how many times the flag is given
|
279
|
-
if verbosity?.length == 1
|
280
|
-
@logger.warn("The -v flag is deprecated and will be removed in a future release. You should use --verbose instead.")
|
281
|
-
@logger.level = :info
|
282
|
-
else
|
283
|
-
@logger.warn("The -vv flag is deprecated and will be removed in a future release. You should use --debug instead.")
|
284
|
-
@logger.level = :debug
|
285
|
-
end
|
286
|
-
else
|
287
|
-
@logger.level = :warn
|
288
|
-
end
|
289
|
-
end
|
290
|
-
|
291
|
-
if log_file
|
292
|
-
# TODO(sissel): Implement file output/rotation in Cabin.
|
293
|
-
# TODO(sissel): Catch exceptions, report sane errors.
|
294
|
-
begin
|
295
|
-
@log_fd.close if @log_fd
|
296
|
-
@log_fd = File.new(path, "a")
|
297
|
-
rescue => e
|
298
|
-
fail(I18n.t("logstash.agent.configuration.log_file_failed",
|
299
|
-
:path => path, :error => e))
|
300
|
-
end
|
301
|
-
|
302
|
-
puts "Sending logstash logs to #{path}."
|
303
|
-
@logger.unsubscribe(@logger_subscription) if @logger_subscription
|
304
|
-
if log_in_json?
|
305
|
-
@logger_subscription = @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
|
306
|
-
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
|
307
|
-
else
|
308
|
-
@logger_subscription = @logger.subscribe(@log_fd)
|
309
|
-
@logger.subscribe(STDOUT, :level => :fatal)
|
310
|
-
end
|
311
|
-
else
|
312
|
-
if log_in_json?
|
313
|
-
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
|
314
|
-
else
|
315
|
-
@logger.subscribe(STDOUT)
|
316
|
-
end
|
317
|
-
end
|
318
|
-
|
319
|
-
|
320
|
-
# TODO(sissel): redirect stdout/stderr to the log as well
|
321
|
-
# http://jira.codehaus.org/browse/JRUBY-7003
|
322
|
-
end # def configure_logging
|
323
|
-
|
324
|
-
# add the given paths for ungemified/bare plugins lookups
|
325
|
-
# @param paths [String, Array<String>] plugins path string or list of path strings to add
|
326
|
-
def configure_plugin_paths(paths)
|
327
|
-
Array(paths).each do |path|
|
328
|
-
fail(I18n.t("logstash.agent.configuration.plugin_path_missing", :path => path)) unless File.directory?(path)
|
329
|
-
LogStash::Environment.add_plugin_path(path)
|
330
|
-
end
|
331
|
-
end
|
332
|
-
|
333
|
-
## Signal Trapping ##
|
334
|
-
def trap_sigint
|
335
|
-
Stud::trap("INT") do
|
336
|
-
if @interrupted_once
|
337
|
-
@logger.fatal(I18n.t("logstash.agent.forced_sigint"))
|
338
|
-
exit
|
339
|
-
else
|
340
|
-
@logger.warn(I18n.t("logstash.agent.sigint"))
|
341
|
-
Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
|
342
|
-
@interrupted_once = true
|
343
|
-
Stud.stop!(@thread)
|
344
|
-
end
|
345
|
-
end
|
346
|
-
end
|
347
|
-
|
348
|
-
def trap_sigterm
|
349
|
-
Stud::trap("TERM") do
|
350
|
-
@logger.warn(I18n.t("logstash.agent.sigterm"))
|
351
|
-
Stud.stop!(@thread)
|
352
|
-
end
|
353
|
-
end
|
354
|
-
|
355
|
-
def trap_sighup
|
356
|
-
Stud::trap("HUP") do
|
357
|
-
@logger.warn(I18n.t("logstash.agent.sighup"))
|
358
|
-
reload_state!
|
359
|
-
end
|
360
71
|
end
|
361
72
|
|
362
|
-
## Pipeline CRUD ##
|
363
|
-
def shutdown(pipeline)
|
364
|
-
pipeline.shutdown do
|
365
|
-
::LogStash::ShutdownWatcher.start(pipeline)
|
366
|
-
end
|
367
|
-
end
|
368
|
-
#
|
369
73
|
# register_pipeline - adds a pipeline to the agent's state
|
370
74
|
# @param pipeline_id [String] pipeline string identifier
|
371
75
|
# @param settings [Hash] settings that will be passed when creating the pipeline.
|
372
76
|
# keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
|
373
77
|
def register_pipeline(pipeline_id, settings)
|
374
|
-
pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id))
|
78
|
+
pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id, :metric => metric))
|
375
79
|
return unless pipeline.is_a?(LogStash::Pipeline)
|
376
80
|
if @auto_reload && pipeline.non_reloadable_plugins.any?
|
377
81
|
@logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
|
@@ -388,30 +92,27 @@ class LogStash::Agent < Clamp::Command
|
|
388
92
|
begin
|
389
93
|
reload_pipeline!(pipeline_id)
|
390
94
|
rescue => e
|
391
|
-
@logger.error(I18n.t("oops"), :
|
95
|
+
@logger.error(I18n.t("oops"), :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
|
392
96
|
end
|
393
97
|
end
|
394
98
|
end
|
395
99
|
end
|
396
100
|
|
397
|
-
|
398
|
-
|
101
|
+
# Calculate the Logstash uptime in milliseconds
|
102
|
+
#
|
103
|
+
# @return [Fixnum] Uptime in milliseconds
|
104
|
+
def uptime
|
105
|
+
((Time.now.to_f - STARTED_AT.to_f) * 1000.0).to_i
|
399
106
|
end
|
400
107
|
|
401
108
|
def shutdown
|
109
|
+
stop_background_services
|
110
|
+
stop_webserver
|
402
111
|
shutdown_pipelines
|
403
112
|
end
|
404
113
|
|
405
|
-
def
|
406
|
-
@
|
407
|
-
end
|
408
|
-
|
409
|
-
def stop_pipeline(id)
|
410
|
-
pipeline = @pipelines[id]
|
411
|
-
return unless pipeline
|
412
|
-
@logger.log("stopping pipeline", :id => id)
|
413
|
-
pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
|
414
|
-
@pipelines[id].thread.join
|
114
|
+
def node_uuid
|
115
|
+
@node_uuid ||= SecureRandom.uuid
|
415
116
|
end
|
416
117
|
|
417
118
|
def running_pipelines?
|
@@ -420,26 +121,56 @@ class LogStash::Agent < Clamp::Command
|
|
420
121
|
end
|
421
122
|
end
|
422
123
|
|
423
|
-
|
424
|
-
|
425
|
-
|
124
|
+
private
|
125
|
+
def start_webserver
|
126
|
+
options = {:http_host => @web_api_http_host, :http_port => @web_api_http_port }
|
127
|
+
@webserver = LogStash::WebServer.new(@logger, options)
|
128
|
+
Thread.new(@webserver) do |webserver|
|
129
|
+
LogStash::Util.set_thread_name("Api Webserver")
|
130
|
+
webserver.run
|
131
|
+
end
|
426
132
|
end
|
427
133
|
|
428
|
-
def
|
429
|
-
|
430
|
-
@pipelines[pipeline_id] = new_pipeline
|
431
|
-
start_pipeline(pipeline_id)
|
134
|
+
def stop_webserver
|
135
|
+
@webserver.stop if @webserver
|
432
136
|
end
|
433
137
|
|
434
|
-
def
|
138
|
+
def start_background_services
|
139
|
+
if collect_metrics?
|
140
|
+
@logger.debug("Agent: Starting metric periodic pollers")
|
141
|
+
@periodic_pollers.start
|
142
|
+
end
|
143
|
+
end
|
435
144
|
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
145
|
+
def stop_background_services
|
146
|
+
if collect_metrics?
|
147
|
+
@logger.debug("Agent: Stopping metric periodic pollers")
|
148
|
+
@periodic_pollers.stop
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def setup_metric_collection
|
153
|
+
if collect_metrics?
|
154
|
+
@logger.debug("Agent: Configuring metric collection")
|
155
|
+
LogStash::Instrument::Collector.instance.agent = self
|
156
|
+
@metric = LogStash::Instrument::Metric.new
|
157
|
+
else
|
158
|
+
@metric = LogStash::Instrument::NullMetric.new
|
159
|
+
end
|
160
|
+
|
161
|
+
@periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
|
162
|
+
end
|
163
|
+
|
164
|
+
def collect_metrics?
|
165
|
+
@collect_metric
|
166
|
+
end
|
167
|
+
|
168
|
+
def create_pipeline(settings)
|
169
|
+
begin
|
170
|
+
config = fetch_config(settings)
|
171
|
+
rescue => e
|
172
|
+
@logger.error("failed to fetch pipeline configuration", :message => e.message)
|
173
|
+
return
|
443
174
|
end
|
444
175
|
|
445
176
|
begin
|
@@ -450,31 +181,26 @@ class LogStash::Agent < Clamp::Command
|
|
450
181
|
end
|
451
182
|
end
|
452
183
|
|
453
|
-
def
|
454
|
-
@
|
184
|
+
def fetch_config(settings)
|
185
|
+
@config_loader.format_config(settings[:config_path], settings[:config_string])
|
455
186
|
end
|
456
187
|
|
457
188
|
# since this method modifies the @pipelines hash it is
|
458
189
|
# wrapped in @upgrade_mutex in the parent call `reload_state!`
|
459
190
|
def reload_pipeline!(id)
|
460
191
|
old_pipeline = @pipelines[id]
|
461
|
-
|
462
|
-
if old_pipeline.config_str == new_config
|
463
|
-
@logger.debug("no configuration change for pipeline",
|
464
|
-
:pipeline => id, :config => new_config)
|
465
|
-
return
|
466
|
-
end
|
467
|
-
|
468
|
-
new_pipeline = create_pipeline(old_pipeline.original_settings, new_config)
|
192
|
+
new_pipeline = create_pipeline(old_pipeline.original_settings)
|
469
193
|
return if new_pipeline.nil?
|
470
194
|
|
471
|
-
if new_pipeline.
|
195
|
+
if old_pipeline.config_str == new_pipeline.config_str
|
196
|
+
@logger.debug("no configuration change for pipeline",
|
197
|
+
:pipeline => id, :config => old_pipeline.config_str)
|
198
|
+
elsif new_pipeline.non_reloadable_plugins.any?
|
472
199
|
@logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
|
473
200
|
:pipeline_id => id,
|
474
201
|
:plugins => new_pipeline.non_reloadable_plugins.map(&:class))
|
475
|
-
return
|
476
202
|
else
|
477
|
-
@logger.
|
203
|
+
@logger.warn("fetched new config for pipeline. upgrading..",
|
478
204
|
:pipeline => id, :config => new_pipeline.config_str)
|
479
205
|
upgrade_pipeline(id, new_pipeline)
|
480
206
|
end
|
@@ -490,53 +216,40 @@ class LogStash::Agent < Clamp::Command
|
|
490
216
|
begin
|
491
217
|
pipeline.run
|
492
218
|
rescue => e
|
493
|
-
@logger.error("Pipeline aborted due to error", :exception => e
|
219
|
+
@logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
|
494
220
|
end
|
495
221
|
end
|
496
222
|
sleep 0.01 until pipeline.ready?
|
497
223
|
end
|
498
224
|
|
499
|
-
|
500
|
-
|
501
|
-
|
225
|
+
def stop_pipeline(id)
|
226
|
+
pipeline = @pipelines[id]
|
227
|
+
return unless pipeline
|
228
|
+
@logger.warn("stopping pipeline", :id => id)
|
229
|
+
pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
|
230
|
+
@pipelines[id].thread.join
|
502
231
|
end
|
503
232
|
|
504
|
-
|
505
|
-
|
506
|
-
@node_uuid ||= SecureRandom.uuid
|
233
|
+
def start_pipelines
|
234
|
+
@pipelines.each { |id, _| start_pipeline(id) }
|
507
235
|
end
|
508
236
|
|
509
|
-
|
510
|
-
|
511
|
-
|
237
|
+
def shutdown_pipelines
|
238
|
+
@pipelines.each { |id, _| stop_pipeline(id) }
|
239
|
+
end
|
512
240
|
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
end
|
518
|
-
end # def show_version
|
519
|
-
|
520
|
-
def show_version_logstash
|
521
|
-
require "logstash/version"
|
522
|
-
puts "logstash #{LOGSTASH_VERSION}"
|
523
|
-
end # def show_version_logstash
|
524
|
-
|
525
|
-
def show_version_ruby
|
526
|
-
puts RUBY_DESCRIPTION
|
527
|
-
end # def show_version_ruby
|
528
|
-
|
529
|
-
def show_version_java
|
530
|
-
properties = java.lang.System.getProperties
|
531
|
-
puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
|
532
|
-
puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
|
533
|
-
end # def show_version_java
|
534
|
-
|
535
|
-
def show_gems
|
536
|
-
require "rubygems"
|
537
|
-
Gem::Specification.each do |spec|
|
538
|
-
puts "gem #{spec.name} #{spec.version}"
|
539
|
-
end
|
540
|
-
end # def show_gems
|
241
|
+
def running_pipeline?(pipeline_id)
|
242
|
+
thread = @pipelines[pipeline_id].thread
|
243
|
+
thread.is_a?(Thread) && thread.alive?
|
244
|
+
end
|
541
245
|
|
246
|
+
def upgrade_pipeline(pipeline_id, new_pipeline)
|
247
|
+
stop_pipeline(pipeline_id)
|
248
|
+
@pipelines[pipeline_id] = new_pipeline
|
249
|
+
start_pipeline(pipeline_id)
|
250
|
+
end
|
251
|
+
|
252
|
+
def clean_state?
|
253
|
+
@pipelines.empty?
|
254
|
+
end
|
542
255
|
end # class LogStash::Agent
|