logstash-core 2.3.4.snapshot1-java → 2.4.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of logstash-core might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/agent.rb +28 -10
- data/lib/logstash/codecs/base.rb +29 -1
- data/lib/logstash/config/mixin.rb +62 -23
- data/lib/logstash/inputs/base.rb +1 -1
- data/lib/logstash/instrument/null_metric.rb +45 -0
- data/lib/logstash/logging/json.rb +21 -0
- data/lib/logstash/output_delegator.rb +3 -3
- data/lib/logstash/outputs/base.rb +32 -1
- data/lib/logstash/pipeline.rb +1 -1
- data/lib/logstash/plugin.rb +11 -1
- data/lib/logstash/util/safe_uri.rb +50 -0
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +4 -0
- data/logstash-core.gemspec +3 -2
- data/spec/logstash/agent_spec.rb +8 -3
- data/spec/logstash/codecs/base_spec.rb +74 -0
- data/spec/logstash/config/mixin_spec.rb +157 -0
- data/spec/logstash/instrument/null_metric_spec.rb +63 -0
- data/spec/logstash/output_delegator_spec.rb +1 -0
- data/spec/logstash/outputs/base_spec.rb +107 -0
- data/spec/logstash/plugin_spec.rb +29 -3
- data/spec/logstash/runner_spec.rb +27 -0
- data/spec/logstash/shutdown_watcher_spec.rb +1 -0
- metadata +15 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3ce17d1073206a2d7cfde3dd8e13aba375a8d797
|
4
|
+
data.tar.gz: 966d4c0af56bb4b9fe56d37efec5812037b0c677
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ccb3f5b9f0d2e3a01e2524f8f14f041bbece04ed8f31ee40780950438990dd149032b578735b0a2d6446e4f8ceca5eb691faa44399709104860dc8177aa22f91
|
7
|
+
data.tar.gz: acf3e571a22a304866669cf10964d1843c5d73904c2ddb0b49d19a4af942134fa030f79528d8aa83bc09e22dae6e76e5084af6ac2559f07023680d652b7389e0
|
data/lib/logstash/agent.rb
CHANGED
@@ -91,6 +91,10 @@ class LogStash::Agent < Clamp::Command
|
|
91
91
|
I18n.t("logstash.agent.flag.allow-env"),
|
92
92
|
:attribute_name => :allow_env, :default => false
|
93
93
|
|
94
|
+
option ["--[no-]log-in-json"], :flag,
|
95
|
+
I18n.t("logstash.agent.flag.log-in-json"),
|
96
|
+
:default => false
|
97
|
+
|
94
98
|
def initialize(*params)
|
95
99
|
super(*params)
|
96
100
|
@logger = Cabin::Channel.get(LogStash)
|
@@ -129,11 +133,11 @@ class LogStash::Agent < Clamp::Command
|
|
129
133
|
# Emit a warning message.
|
130
134
|
def warn(message)
|
131
135
|
# For now, all warnings are fatal.
|
132
|
-
|
136
|
+
signal_usage_error(message)
|
133
137
|
end # def warn
|
134
138
|
|
135
139
|
def fail(message)
|
136
|
-
|
140
|
+
signal_usage_error(message)
|
137
141
|
end # def fail
|
138
142
|
|
139
143
|
# Run the agent. This method is invoked after clamp parses the
|
@@ -142,6 +146,7 @@ class LogStash::Agent < Clamp::Command
|
|
142
146
|
require "logstash/pipeline"
|
143
147
|
require "cabin" # gem 'cabin'
|
144
148
|
require "logstash/plugin"
|
149
|
+
require "logstash/logging/json"
|
145
150
|
|
146
151
|
LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
|
147
152
|
LogStash::ShutdownWatcher.logger = @logger
|
@@ -177,8 +182,7 @@ class LogStash::Agent < Clamp::Command
|
|
177
182
|
end
|
178
183
|
|
179
184
|
if config_test?
|
180
|
-
|
181
|
-
config_str = config_loader.format_config(config_path, config_string)
|
185
|
+
config_str = @config_loader.format_config(config_path, config_string)
|
182
186
|
begin
|
183
187
|
# currently the best strategy to validate the configuration
|
184
188
|
# is creating a pipeline instance and checking for exceptions
|
@@ -198,6 +202,8 @@ class LogStash::Agent < Clamp::Command
|
|
198
202
|
:allow_env => allow_env?
|
199
203
|
}))
|
200
204
|
|
205
|
+
@thread = Thread.current # this var is implicilty used by Stud.stop?
|
206
|
+
|
201
207
|
sigint_id = trap_sigint()
|
202
208
|
sigterm_id = trap_sigterm()
|
203
209
|
sighup_id = trap_sighup()
|
@@ -210,8 +216,6 @@ class LogStash::Agent < Clamp::Command
|
|
210
216
|
|
211
217
|
return 1 if clean_state?
|
212
218
|
|
213
|
-
@thread = Thread.current # this var is implicilty used by Stud.stop?
|
214
|
-
|
215
219
|
Stud.stoppable_sleep(reload_interval) # sleep before looping
|
216
220
|
|
217
221
|
if auto_reload?
|
@@ -237,7 +241,10 @@ class LogStash::Agent < Clamp::Command
|
|
237
241
|
end
|
238
242
|
return 1
|
239
243
|
rescue => e
|
240
|
-
|
244
|
+
if show_startup_errors
|
245
|
+
@logger.terminal(e.message)
|
246
|
+
@logger.unsubscribe(stdout_logs)
|
247
|
+
end
|
241
248
|
@logger.warn(I18n.t("oops"), :error => e, :class => e.class.name, :backtrace => e.backtrace)
|
242
249
|
return 1
|
243
250
|
ensure
|
@@ -294,11 +301,22 @@ class LogStash::Agent < Clamp::Command
|
|
294
301
|
|
295
302
|
puts "Sending logstash logs to #{path}."
|
296
303
|
@logger.unsubscribe(@logger_subscription) if @logger_subscription
|
297
|
-
|
304
|
+
if log_in_json?
|
305
|
+
@logger_subscription = @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
|
306
|
+
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
|
307
|
+
else
|
308
|
+
@logger_subscription = @logger.subscribe(@log_fd)
|
309
|
+
@logger.subscribe(STDOUT, :level => :fatal)
|
310
|
+
end
|
298
311
|
else
|
299
|
-
|
312
|
+
if log_in_json?
|
313
|
+
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
|
314
|
+
else
|
315
|
+
@logger.subscribe(STDOUT)
|
316
|
+
end
|
300
317
|
end
|
301
318
|
|
319
|
+
|
302
320
|
# TODO(sissel): redirect stdout/stderr to the log as well
|
303
321
|
# http://jira.codehaus.org/browse/JRUBY-7003
|
304
322
|
end # def configure_logging
|
@@ -472,7 +490,7 @@ class LogStash::Agent < Clamp::Command
|
|
472
490
|
begin
|
473
491
|
pipeline.run
|
474
492
|
rescue => e
|
475
|
-
@logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
|
493
|
+
@logger.error("Pipeline aborted due to error", :exception => e.class.name, :backtrace => e.backtrace)
|
476
494
|
end
|
477
495
|
end
|
478
496
|
sleep 0.01 until pipeline.ready?
|
data/lib/logstash/codecs/base.rb
CHANGED
@@ -13,6 +13,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
|
|
13
13
|
super
|
14
14
|
config_init(@params)
|
15
15
|
register if respond_to?(:register)
|
16
|
+
setup_multi_encode!
|
16
17
|
end
|
17
18
|
|
18
19
|
public
|
@@ -23,10 +24,37 @@ module LogStash::Codecs; class Base < LogStash::Plugin
|
|
23
24
|
alias_method :<<, :decode
|
24
25
|
|
25
26
|
public
|
27
|
+
# DEPRECATED: Prefer defining encode_sync or multi_encode
|
26
28
|
def encode(event)
|
27
|
-
|
29
|
+
encoded = multi_encode([event])
|
30
|
+
encoded.each {|event,data| @on_event.call(event,data) }
|
28
31
|
end # def encode
|
29
32
|
|
33
|
+
public
|
34
|
+
# Relies on the codec being synchronous (which they all are!)
|
35
|
+
# We need a better long term design here, but this is an improvement
|
36
|
+
# over the current API for shared plugins
|
37
|
+
# It is best if the codec implements this directly
|
38
|
+
def multi_encode(events)
|
39
|
+
if @has_encode_sync
|
40
|
+
events.map {|event| [event, self.encode_sync(event)]}
|
41
|
+
else
|
42
|
+
batch = Thread.current[:logstash_output_codec_batch] ||= []
|
43
|
+
batch.clear
|
44
|
+
|
45
|
+
events.each {|event| self.encode(event) }
|
46
|
+
batch
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def setup_multi_encode!
|
51
|
+
@has_encode_sync = self.methods.include?(:encode_sync)
|
52
|
+
|
53
|
+
on_event do |event, data|
|
54
|
+
Thread.current[:logstash_output_codec_batch] << [event, data]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
30
58
|
public
|
31
59
|
def close; end;
|
32
60
|
|
@@ -3,6 +3,7 @@ require "logstash/namespace"
|
|
3
3
|
require "logstash/config/registry"
|
4
4
|
require "logstash/logging"
|
5
5
|
require "logstash/util/password"
|
6
|
+
require "logstash/util/safe_uri"
|
6
7
|
require "logstash/version"
|
7
8
|
require "logstash/environment"
|
8
9
|
require "logstash/util/plugin_version"
|
@@ -333,58 +334,89 @@ module LogStash::Config::Mixin
|
|
333
334
|
return true
|
334
335
|
end # def validate_check_invalid_parameter_names
|
335
336
|
|
337
|
+
def validate_check_required_parameter(config_key, config_opts, k, v)
|
338
|
+
if config_key.is_a?(Regexp)
|
339
|
+
(k =~ config_key && v)
|
340
|
+
elsif config_key.is_a?(String)
|
341
|
+
k && v
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
336
345
|
def validate_check_required_parameter_names(params)
|
337
346
|
is_valid = true
|
338
347
|
|
339
348
|
@config.each do |config_key, config|
|
340
349
|
next unless config[:required]
|
341
350
|
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
351
|
+
value = params[config_key]
|
352
|
+
|
353
|
+
if config_key.is_a?(Regexp) && !params.keys.any? { |k| k =~ config_key }
|
354
|
+
is_valid = false
|
355
|
+
end
|
356
|
+
|
357
|
+
if value.nil? || (config[:list] && Array(value).empty?)
|
358
|
+
@logger.error(I18n.t("logstash.agent.configuration.setting_missing",
|
359
|
+
:setting => config_key, :plugin => @plugin_name,
|
360
|
+
:type => @plugin_type))
|
361
|
+
is_valid = false
|
346
362
|
end
|
347
|
-
@logger.error(I18n.t("logstash.agent.configuration.setting_missing",
|
348
|
-
:setting => config_key, :plugin => @plugin_name,
|
349
|
-
:type => @plugin_type))
|
350
|
-
is_valid = false
|
351
363
|
end
|
352
364
|
|
353
365
|
return is_valid
|
354
366
|
end
|
355
367
|
|
368
|
+
def process_parameter_value(value, config_settings)
|
369
|
+
config_val = config_settings[:validate]
|
370
|
+
|
371
|
+
if config_settings[:list]
|
372
|
+
value = Array(value) # coerce scalars to lists
|
373
|
+
# Empty lists are converted to nils
|
374
|
+
return true, nil if value.empty?
|
375
|
+
|
376
|
+
validated_items = value.map {|v| validate_value(v, config_val)}
|
377
|
+
is_valid = validated_items.all? {|sr| sr[0] }
|
378
|
+
processed_value = validated_items.map {|sr| sr[1]}
|
379
|
+
else
|
380
|
+
is_valid, processed_value = validate_value(value, config_val)
|
381
|
+
end
|
382
|
+
|
383
|
+
return [is_valid, processed_value]
|
384
|
+
end
|
385
|
+
|
356
386
|
def validate_check_parameter_values(params)
|
357
387
|
# Filter out parametrs that match regexp keys.
|
358
388
|
# These are defined in plugins like this:
|
359
389
|
# config /foo.*/ => ...
|
360
|
-
|
390
|
+
all_params_valid = true
|
361
391
|
|
362
392
|
params.each do |key, value|
|
363
393
|
@config.keys.each do |config_key|
|
364
394
|
next unless (config_key.is_a?(Regexp) && key =~ config_key) \
|
365
395
|
|| (config_key.is_a?(String) && key == config_key)
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
396
|
+
|
397
|
+
config_settings = @config[config_key]
|
398
|
+
|
399
|
+
is_valid, processed_value = process_parameter_value(value, config_settings)
|
400
|
+
|
401
|
+
if is_valid
|
402
|
+
# Accept coerced value if valid
|
371
403
|
# Used for converting values in the config to proper objects.
|
372
|
-
params[key] =
|
404
|
+
params[key] = processed_value
|
373
405
|
else
|
374
406
|
@logger.error(I18n.t("logstash.agent.configuration.setting_invalid",
|
375
407
|
:plugin => @plugin_name, :type => @plugin_type,
|
376
408
|
:setting => key, :value => value.inspect,
|
377
|
-
:value_type =>
|
378
|
-
:note =>
|
409
|
+
:value_type => config_settings[:validate],
|
410
|
+
:note => processed_value))
|
379
411
|
end
|
380
|
-
|
381
|
-
|
412
|
+
|
413
|
+
all_params_valid &&= is_valid
|
382
414
|
|
383
415
|
break # done with this param key
|
384
416
|
end # config.each
|
385
417
|
end # params.each
|
386
418
|
|
387
|
-
return
|
419
|
+
return all_params_valid
|
388
420
|
end # def validate_check_parameter_values
|
389
421
|
|
390
422
|
def validator_find(key)
|
@@ -405,7 +437,7 @@ module LogStash::Config::Mixin
|
|
405
437
|
result = nil
|
406
438
|
|
407
439
|
if validator.nil?
|
408
|
-
return true
|
440
|
+
return true, value
|
409
441
|
elsif validator.is_a?(Array)
|
410
442
|
value = [*value]
|
411
443
|
if value.size > 1
|
@@ -519,6 +551,12 @@ module LogStash::Config::Mixin
|
|
519
551
|
end
|
520
552
|
|
521
553
|
result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
|
554
|
+
when :uri
|
555
|
+
if value.size > 1
|
556
|
+
return false, "Expected uri (one value), got #{value.size} values?"
|
557
|
+
end
|
558
|
+
|
559
|
+
result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
|
522
560
|
when :path
|
523
561
|
if value.size > 1 # Only 1 value wanted
|
524
562
|
return false, "Expected path (one value), got #{value.size} values?"
|
@@ -554,8 +592,9 @@ module LogStash::Config::Mixin
|
|
554
592
|
|
555
593
|
def secure_params!(params)
|
556
594
|
params.each do |key, value|
|
557
|
-
if @config[key][:validate]
|
558
|
-
|
595
|
+
if [:uri, :password].include? @config[key][:validate]
|
596
|
+
is_valid, processed_value = process_parameter_value(value, @config[key])
|
597
|
+
params[key] = processed_value
|
559
598
|
end
|
560
599
|
end
|
561
600
|
end
|
data/lib/logstash/inputs/base.rb
CHANGED
@@ -0,0 +1,45 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module LogStash module Instrument
|
4
|
+
# This class is used in the context when we disable the metric collection
|
5
|
+
# for specific plugin to replace the `NamespacedMetric` class with this one
|
6
|
+
# which doesn't produce any metric to the collector.
|
7
|
+
class NullMetric
|
8
|
+
attr_reader :namespace_name, :collector
|
9
|
+
|
10
|
+
def increment(key, value = 1)
|
11
|
+
end
|
12
|
+
|
13
|
+
def decrement(key, value = 1)
|
14
|
+
end
|
15
|
+
|
16
|
+
def gauge(key, value)
|
17
|
+
end
|
18
|
+
|
19
|
+
def report_time(key, duration)
|
20
|
+
end
|
21
|
+
|
22
|
+
# We have to manually redefine this method since it can return an
|
23
|
+
# object this object also has to be implemented as a NullObject
|
24
|
+
def time(key, &block)
|
25
|
+
if block_given?
|
26
|
+
yield
|
27
|
+
else
|
28
|
+
NullTimedExecution
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def namespace(name)
|
33
|
+
self.class.new
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
# Null implementation of the internal timer class
|
38
|
+
#
|
39
|
+
# @see LogStash::Instrument::TimedExecution`
|
40
|
+
class NullTimedExecution
|
41
|
+
def self.stop
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end; end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/namespace"
|
3
|
+
require "logstash/logging"
|
4
|
+
require "logstash/json"
|
5
|
+
|
6
|
+
module LogStash; class Logging; class JSON
|
7
|
+
def initialize(io)
|
8
|
+
raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
|
9
|
+
|
10
|
+
@io = io
|
11
|
+
@lock = Mutex.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def <<(obj)
|
15
|
+
serialized = LogStash::Json.dump(obj)
|
16
|
+
@lock.synchronize do
|
17
|
+
@io.puts(serialized)
|
18
|
+
@io.flush
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end; end; end
|
@@ -74,7 +74,7 @@ module LogStash class OutputDelegator
|
|
74
74
|
@workers << @klass.new(@config)
|
75
75
|
@workers.first.register # Needed in case register calls `workers_not_supported`
|
76
76
|
|
77
|
-
@logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
|
77
|
+
@logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
|
78
78
|
|
79
79
|
# Threadsafe versions don't need additional workers
|
80
80
|
setup_additional_workers!(target_worker_count) unless @threadsafe
|
@@ -134,7 +134,7 @@ module LogStash class OutputDelegator
|
|
134
134
|
end
|
135
135
|
|
136
136
|
def do_close
|
137
|
-
@logger.debug("closing output delegator", :klass => @klass)
|
137
|
+
@logger.debug("closing output delegator", :klass => @klass.name)
|
138
138
|
|
139
139
|
if @threadsafe
|
140
140
|
@workers.each(&:do_close)
|
@@ -169,4 +169,4 @@ module LogStash class OutputDelegator
|
|
169
169
|
private
|
170
170
|
# Needed for testing, so private
|
171
171
|
attr_reader :threadsafe_worker, :worker_queue
|
172
|
-
end end
|
172
|
+
end end
|
@@ -27,6 +27,22 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
27
27
|
|
28
28
|
attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
|
29
29
|
|
30
|
+
# Set or return concurrency type
|
31
|
+
def self.concurrency(type=nil)
|
32
|
+
if type
|
33
|
+
@concurrency = type
|
34
|
+
|
35
|
+
if type == :shared
|
36
|
+
declare_threadsafe!
|
37
|
+
elsif type == :single
|
38
|
+
declare_workers_not_supported!("This plugin only supports one worker!")
|
39
|
+
end
|
40
|
+
|
41
|
+
else
|
42
|
+
@concurrency || :legacy # default is :legacyo
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
30
46
|
def self.declare_threadsafe!
|
31
47
|
declare_workers_not_supported!
|
32
48
|
@threadsafe = true
|
@@ -65,6 +81,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
65
81
|
# If we're running with a single thread we must enforce single-threaded concurrency by default
|
66
82
|
# Maybe in a future version we'll assume output plugins are threadsafe
|
67
83
|
@single_worker_mutex = Mutex.new
|
84
|
+
|
85
|
+
@receives_encoded = self.methods.include?(:multi_receive_encoded)
|
68
86
|
end
|
69
87
|
|
70
88
|
public
|
@@ -77,10 +95,23 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
77
95
|
raise "#{self.class}#receive must be overidden"
|
78
96
|
end # def receive
|
79
97
|
|
98
|
+
public
|
99
|
+
def concurrency
|
100
|
+
self.class.concurrency
|
101
|
+
end
|
102
|
+
|
80
103
|
public
|
81
104
|
# To be overriden in implementations
|
82
105
|
def multi_receive(events)
|
83
|
-
|
106
|
+
if @receives_encoded
|
107
|
+
self.multi_receive_encoded(codec.multi_encode(events))
|
108
|
+
else
|
109
|
+
events.each {|event| receive(event) }
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def codec
|
114
|
+
params["codec"]
|
84
115
|
end
|
85
116
|
|
86
117
|
private
|