logstash-core 2.2.4.snapshot1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (83) hide show
  1. checksums.yaml +7 -0
  2. data/lib/logstash-core.rb +1 -0
  3. data/lib/logstash-core/logstash-core.rb +3 -0
  4. data/lib/logstash-core/version.rb +8 -0
  5. data/lib/logstash/agent.rb +391 -0
  6. data/lib/logstash/codecs/base.rb +50 -0
  7. data/lib/logstash/config/config_ast.rb +550 -0
  8. data/lib/logstash/config/cpu_core_strategy.rb +32 -0
  9. data/lib/logstash/config/defaults.rb +12 -0
  10. data/lib/logstash/config/file.rb +39 -0
  11. data/lib/logstash/config/grammar.rb +3503 -0
  12. data/lib/logstash/config/mixin.rb +518 -0
  13. data/lib/logstash/config/registry.rb +13 -0
  14. data/lib/logstash/environment.rb +98 -0
  15. data/lib/logstash/errors.rb +12 -0
  16. data/lib/logstash/filters/base.rb +205 -0
  17. data/lib/logstash/inputs/base.rb +116 -0
  18. data/lib/logstash/inputs/threadable.rb +18 -0
  19. data/lib/logstash/java_integration.rb +116 -0
  20. data/lib/logstash/json.rb +61 -0
  21. data/lib/logstash/logging.rb +91 -0
  22. data/lib/logstash/namespace.rb +13 -0
  23. data/lib/logstash/output_delegator.rb +172 -0
  24. data/lib/logstash/outputs/base.rb +91 -0
  25. data/lib/logstash/patches.rb +5 -0
  26. data/lib/logstash/patches/bugfix_jruby_2558.rb +51 -0
  27. data/lib/logstash/patches/cabin.rb +35 -0
  28. data/lib/logstash/patches/profile_require_calls.rb +47 -0
  29. data/lib/logstash/patches/rubygems.rb +38 -0
  30. data/lib/logstash/patches/stronger_openssl_defaults.rb +68 -0
  31. data/lib/logstash/pipeline.rb +499 -0
  32. data/lib/logstash/pipeline_reporter.rb +114 -0
  33. data/lib/logstash/plugin.rb +120 -0
  34. data/lib/logstash/program.rb +14 -0
  35. data/lib/logstash/runner.rb +124 -0
  36. data/lib/logstash/shutdown_watcher.rb +100 -0
  37. data/lib/logstash/util.rb +203 -0
  38. data/lib/logstash/util/buftok.rb +139 -0
  39. data/lib/logstash/util/charset.rb +35 -0
  40. data/lib/logstash/util/decorators.rb +52 -0
  41. data/lib/logstash/util/defaults_printer.rb +31 -0
  42. data/lib/logstash/util/filetools.rb +186 -0
  43. data/lib/logstash/util/java_version.rb +66 -0
  44. data/lib/logstash/util/password.rb +25 -0
  45. data/lib/logstash/util/plugin_version.rb +56 -0
  46. data/lib/logstash/util/prctl.rb +10 -0
  47. data/lib/logstash/util/retryable.rb +40 -0
  48. data/lib/logstash/util/socket_peer.rb +7 -0
  49. data/lib/logstash/util/unicode_trimmer.rb +81 -0
  50. data/lib/logstash/util/worker_threads_default_printer.rb +29 -0
  51. data/lib/logstash/util/wrapped_synchronous_queue.rb +41 -0
  52. data/lib/logstash/version.rb +14 -0
  53. data/locales/en.yml +204 -0
  54. data/logstash-core.gemspec +58 -0
  55. data/spec/conditionals_spec.rb +429 -0
  56. data/spec/logstash/agent_spec.rb +85 -0
  57. data/spec/logstash/config/config_ast_spec.rb +146 -0
  58. data/spec/logstash/config/cpu_core_strategy_spec.rb +123 -0
  59. data/spec/logstash/config/defaults_spec.rb +10 -0
  60. data/spec/logstash/config/mixin_spec.rb +158 -0
  61. data/spec/logstash/environment_spec.rb +56 -0
  62. data/spec/logstash/filters/base_spec.rb +251 -0
  63. data/spec/logstash/inputs/base_spec.rb +74 -0
  64. data/spec/logstash/java_integration_spec.rb +304 -0
  65. data/spec/logstash/json_spec.rb +96 -0
  66. data/spec/logstash/output_delegator_spec.rb +144 -0
  67. data/spec/logstash/outputs/base_spec.rb +40 -0
  68. data/spec/logstash/patches_spec.rb +90 -0
  69. data/spec/logstash/pipeline_reporter_spec.rb +85 -0
  70. data/spec/logstash/pipeline_spec.rb +455 -0
  71. data/spec/logstash/plugin_spec.rb +169 -0
  72. data/spec/logstash/runner_spec.rb +68 -0
  73. data/spec/logstash/shutdown_watcher_spec.rb +113 -0
  74. data/spec/logstash/util/buftok_spec.rb +31 -0
  75. data/spec/logstash/util/charset_spec.rb +74 -0
  76. data/spec/logstash/util/defaults_printer_spec.rb +50 -0
  77. data/spec/logstash/util/java_version_spec.rb +79 -0
  78. data/spec/logstash/util/plugin_version_spec.rb +64 -0
  79. data/spec/logstash/util/unicode_trimmer_spec.rb +55 -0
  80. data/spec/logstash/util/worker_threads_default_printer_spec.rb +45 -0
  81. data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +28 -0
  82. data/spec/logstash/util_spec.rb +35 -0
  83. metadata +364 -0
@@ -0,0 +1,518 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "logstash/config/registry"
4
+ require "logstash/logging"
5
+ require "logstash/util/password"
6
+ require "logstash/version"
7
+ require "logstash/environment"
8
+ require "logstash/util/plugin_version"
9
+ require "filesize"
10
+
11
+ LogStash::Environment.load_locale!
12
+
13
+ # This module is meant as a mixin to classes wishing to be configurable from
14
+ # config files
15
+ #
16
+ # The idea is that you can do this:
17
+ #
18
+ # class Foo < LogStash::Config
19
+ # # Add config file settings
20
+ # config "path" => ...
21
+ # config "tag" => ...
22
+ #
23
+ # # Add global flags (becomes --foo-bar)
24
+ # flag "bar" => ...
25
+ # end
26
+ #
27
+ # And the config file should let you do:
28
+ #
29
+ # foo {
30
+ # "path" => ...
31
+ # "tag" => ...
32
+ # }
33
+ #
34
+ module LogStash::Config::Mixin
35
+ attr_accessor :config
36
+ attr_accessor :original_params
37
+
38
+ PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
39
+ PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
40
+
41
+ # This method is called when someone does 'include LogStash::Config'
42
+ def self.included(base)
43
+ # Add the DSL methods to the 'base' given.
44
+ base.extend(LogStash::Config::Mixin::DSL)
45
+ end
46
+
47
+ def config_init(params)
48
+ # Validation will modify the values inside params if necessary.
49
+ # For example: converting a string to a number, etc.
50
+
51
+ # Keep a copy of the original config params so that we can later
52
+ # differentiate between explicit configuration and implicit (default)
53
+ # configuration.
54
+ original_params = params.clone
55
+
56
+ # store the plugin type, turns LogStash::Inputs::Base into 'input'
57
+ @plugin_type = self.class.ancestors.find { |a| a.name =~ /::Base$/ }.config_name
58
+
59
+ # warn about deprecated variable use
60
+ params.each do |name, value|
61
+ opts = self.class.get_config[name]
62
+ if opts && opts[:deprecated]
63
+ extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
64
+ extra.gsub!("%PLUGIN%", self.class.config_name)
65
+ @logger.warn("You are using a deprecated config setting " +
66
+ "#{name.inspect} set in #{self.class.config_name}. " +
67
+ "Deprecated settings will continue to work, " +
68
+ "but are scheduled for removal from logstash " +
69
+ "in the future. #{extra} If you have any questions " +
70
+ "about this, please visit the #logstash channel " +
71
+ "on freenode irc.", :name => name, :plugin => self)
72
+ end
73
+ if opts && opts[:obsolete]
74
+ extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
75
+ extra.gsub!("%PLUGIN%", self.class.config_name)
76
+ raise LogStash::ConfigurationError,
77
+ I18n.t("logstash.agent.configuration.obsolete", :name => name,
78
+ :plugin => self.class.config_name, :extra => extra)
79
+ end
80
+ end
81
+
82
+ # Set defaults from 'config :foo, :default => somevalue'
83
+ self.class.get_config.each do |name, opts|
84
+ next if params.include?(name.to_s)
85
+ if opts.include?(:default) and (name.is_a?(Symbol) or name.is_a?(String))
86
+ # default values should be cloned if possible
87
+ # cloning prevents
88
+ case opts[:default]
89
+ when FalseClass, TrueClass, NilClass, Numeric
90
+ params[name.to_s] = opts[:default]
91
+ else
92
+ params[name.to_s] = opts[:default].clone
93
+ end
94
+ end
95
+
96
+ # Allow plugins to override default values of config settings
97
+ if self.class.default?(name)
98
+ params[name.to_s] = self.class.get_default(name)
99
+ end
100
+ end
101
+
102
+ if !self.class.validate(params)
103
+ raise LogStash::ConfigurationError,
104
+ I18n.t("logstash.agent.configuration.invalid_plugin_settings")
105
+ end
106
+
107
+ # We remove any config options marked as obsolete,
108
+ # no code should be associated to them and their values should not bleed
109
+ # to the plugin context.
110
+ #
111
+ # This need to be done after fetching the options from the parents classed
112
+ params.reject! do |name, value|
113
+ opts = self.class.get_config[name]
114
+ opts.include?(:obsolete)
115
+ end
116
+
117
+ # set instance variables like '@foo' for each config value given.
118
+ params.each do |key, value|
119
+ next if key[0, 1] == "@"
120
+
121
+ # Set this key as an instance variable only if it doesn't start with an '@'
122
+ @logger.debug("config #{self.class.name}/@#{key} = #{value.inspect}")
123
+ instance_variable_set("@#{key}", value)
124
+ end
125
+
126
+ # now that we know the parameters are valid, we can obfuscate the original copy
127
+ # of the parameters before storing them as an instance variable
128
+ self.class.secure_params!(original_params)
129
+ @original_params = original_params
130
+
131
+ @config = params
132
+ end # def config_init
133
+
134
+ module DSL
135
+ attr_accessor :flags
136
+
137
+ # If name is given, set the name and return it.
138
+ # If no name given (nil), return the current name.
139
+ def config_name(name = nil)
140
+ @config_name = name if !name.nil?
141
+ LogStash::Config::Registry.registry[@config_name] = self
142
+ return @config_name
143
+ end
144
+
145
+ # Deprecated: Declare the version of the plugin
146
+ # inside the gemspec.
147
+ def plugin_status(status = nil)
148
+ milestone(status)
149
+ end
150
+
151
+ # Deprecated: Declare the version of the plugin
152
+ # inside the gemspec.
153
+ def milestone(m = nil)
154
+ @logger = Cabin::Channel.get(LogStash)
155
+ @logger.warn(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
156
+ end
157
+
158
+ # Define a new configuration setting
159
+ def config(name, opts={})
160
+ @config ||= Hash.new
161
+ # TODO(sissel): verify 'name' is of type String, Symbol, or Regexp
162
+
163
+ name = name.to_s if name.is_a?(Symbol)
164
+ @config[name] = opts # ok if this is empty
165
+
166
+ if name.is_a?(String)
167
+ define_method(name) { instance_variable_get("@#{name}") }
168
+ define_method("#{name}=") { |v| instance_variable_set("@#{name}", v) }
169
+ end
170
+ end # def config
171
+
172
+ def default(name, value)
173
+ @defaults ||= {}
174
+ @defaults[name.to_s] = value
175
+ end
176
+
177
+ def get_config
178
+ return @config
179
+ end # def get_config
180
+
181
+ def get_default(name)
182
+ return @defaults && @defaults[name]
183
+ end
184
+
185
+ def default?(name)
186
+ return @defaults && @defaults.include?(name)
187
+ end
188
+
189
+ def options(opts)
190
+ # add any options from this class
191
+ prefix = self.name.split("::").last.downcase
192
+ @flags.each do |flag|
193
+ flagpart = flag[:args].first.gsub(/^--/,"")
194
+ # TODO(sissel): logger things here could help debugging.
195
+
196
+ opts.on("--#{prefix}-#{flagpart}", *flag[:args][1..-1], &flag[:block])
197
+ end
198
+ end # def options
199
+
200
+ # This is called whenever someone subclasses a class that has this mixin.
201
+ def inherited(subclass)
202
+ # Copy our parent's config to a subclass.
203
+ # This method is invoked whenever someone subclasses us, like:
204
+ # class Foo < Bar ...
205
+ subconfig = Hash.new
206
+ if !@config.nil?
207
+ @config.each do |key, val|
208
+ subconfig[key] = val
209
+ end
210
+ end
211
+ subclass.instance_variable_set("@config", subconfig)
212
+ @@version_notice_given = false
213
+ end # def inherited
214
+
215
+ def validate(params)
216
+ @plugin_name = config_name
217
+ @plugin_type = ancestors.find { |a| a.name =~ /::Base$/ }.config_name
218
+ @logger = Cabin::Channel.get(LogStash)
219
+ is_valid = true
220
+
221
+ print_version_notice
222
+
223
+ is_valid &&= validate_check_invalid_parameter_names(params)
224
+ is_valid &&= validate_check_required_parameter_names(params)
225
+ is_valid &&= validate_check_parameter_values(params)
226
+
227
+ return is_valid
228
+ end # def validate
229
+
230
+ def print_version_notice
231
+ return if @@version_notice_given
232
+
233
+ begin
234
+ plugin_version = LogStash::Util::PluginVersion.find_plugin_version!(@plugin_type, @config_name)
235
+
236
+ if plugin_version < PLUGIN_VERSION_1_0_0
237
+ if plugin_version < PLUGIN_VERSION_0_9_0
238
+ @logger.info(I18n.t("logstash.plugin.version.0-1-x",
239
+ :type => @plugin_type,
240
+ :name => @config_name,
241
+ :LOGSTASH_VERSION => LOGSTASH_VERSION))
242
+ else
243
+ @logger.info(I18n.t("logstash.plugin.version.0-9-x",
244
+ :type => @plugin_type,
245
+ :name => @config_name,
246
+ :LOGSTASH_VERSION => LOGSTASH_VERSION))
247
+ end
248
+ end
249
+ rescue LogStash::PluginNoVersionError
250
+ # If we cannot find a version in the currently installed gems we
251
+ # will display this message. This could happen in the test, if you
252
+ # create an anonymous class to test a plugin.
253
+ @logger.warn(I18n.t("logstash.plugin.no_version",
254
+ :type => @plugin_type,
255
+ :name => @config_name,
256
+ :LOGSTASH_VERSION => LOGSTASH_VERSION))
257
+ ensure
258
+ @@version_notice_given = true
259
+ end
260
+ end
261
+
262
+ def validate_check_invalid_parameter_names(params)
263
+ invalid_params = params.keys
264
+ # Filter out parameters that match regexp keys.
265
+ # These are defined in plugins like this:
266
+ # config /foo.*/ => ...
267
+ @config.each_key do |config_key|
268
+ if config_key.is_a?(Regexp)
269
+ invalid_params.reject! { |k| k =~ config_key }
270
+ elsif config_key.is_a?(String)
271
+ invalid_params.reject! { |k| k == config_key }
272
+ end
273
+ end
274
+
275
+ if invalid_params.size > 0
276
+ invalid_params.each do |name|
277
+ @logger.error("Unknown setting '#{name}' for #{@plugin_name}")
278
+ end
279
+ return false
280
+ end # if invalid_params.size > 0
281
+ return true
282
+ end # def validate_check_invalid_parameter_names
283
+
284
+ def validate_check_required_parameter_names(params)
285
+ is_valid = true
286
+
287
+ @config.each do |config_key, config|
288
+ next unless config[:required]
289
+
290
+ if config_key.is_a?(Regexp)
291
+ next if params.keys.select { |k| k =~ config_key }.length > 0
292
+ elsif config_key.is_a?(String)
293
+ next if params.keys.member?(config_key)
294
+ end
295
+ @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
296
+ :setting => config_key, :plugin => @plugin_name,
297
+ :type => @plugin_type))
298
+ is_valid = false
299
+ end
300
+
301
+ return is_valid
302
+ end
303
+
304
+ def validate_check_parameter_values(params)
305
+ # Filter out parametrs that match regexp keys.
306
+ # These are defined in plugins like this:
307
+ # config /foo.*/ => ...
308
+ is_valid = true
309
+
310
+ params.each do |key, value|
311
+ @config.keys.each do |config_key|
312
+ next unless (config_key.is_a?(Regexp) && key =~ config_key) \
313
+ || (config_key.is_a?(String) && key == config_key)
314
+ config_val = @config[config_key][:validate]
315
+ #puts " Key matches."
316
+ success, result = validate_value(value, config_val)
317
+ if success
318
+ # Accept coerced value if success
319
+ # Used for converting values in the config to proper objects.
320
+ params[key] = result if !result.nil?
321
+ else
322
+ @logger.error(I18n.t("logstash.agent.configuration.setting_invalid",
323
+ :plugin => @plugin_name, :type => @plugin_type,
324
+ :setting => key, :value => value.inspect,
325
+ :value_type => config_val,
326
+ :note => result))
327
+ end
328
+ #puts "Result: #{key} / #{result.inspect} / #{success}"
329
+ is_valid &&= success
330
+
331
+ break # done with this param key
332
+ end # config.each
333
+ end # params.each
334
+
335
+ return is_valid
336
+ end # def validate_check_parameter_values
337
+
338
+ def validator_find(key)
339
+ @config.each do |config_key, config_val|
340
+ if (config_key.is_a?(Regexp) && key =~ config_key) \
341
+ || (config_key.is_a?(String) && key == config_key)
342
+ return config_val
343
+ end
344
+ end # @config.each
345
+ return nil
346
+ end
347
+
348
+ def validate_value(value, validator)
349
+ # Validator comes from the 'config' pieces of plugins.
350
+ # They look like this
351
+ # config :mykey => lambda do |value| ... end
352
+ # (see LogStash::Inputs::File for example)
353
+ result = nil
354
+
355
+ if validator.nil?
356
+ return true
357
+ elsif validator.is_a?(Array)
358
+ value = [*value]
359
+ if value.size > 1
360
+ return false, "Expected one of #{validator.inspect}, got #{value.inspect}"
361
+ end
362
+
363
+ if !validator.include?(value.first)
364
+ return false, "Expected one of #{validator.inspect}, got #{value.inspect}"
365
+ end
366
+ result = value.first
367
+ elsif validator.is_a?(Symbol)
368
+ # TODO(sissel): Factor this out into a coersion method?
369
+ # TODO(sissel): Document this stuff.
370
+ value = hash_or_array(value)
371
+
372
+ case validator
373
+ when :codec
374
+ if value.first.is_a?(String)
375
+ value = LogStash::Plugin.lookup("codec", value.first).new
376
+ return true, value
377
+ else
378
+ value = value.first
379
+ return true, value
380
+ end
381
+ when :hash
382
+ if value.is_a?(Hash)
383
+ return true, value
384
+ end
385
+
386
+ if value.size % 2 == 1
387
+ return false, "This field must contain an even number of items, got #{value.size}"
388
+ end
389
+
390
+ # Convert the array the config parser produces into a hash.
391
+ result = {}
392
+ value.each_slice(2) do |key, value|
393
+ entry = result[key]
394
+ if entry.nil?
395
+ result[key] = value
396
+ else
397
+ if entry.is_a?(Array)
398
+ entry << value
399
+ else
400
+ result[key] = [entry, value]
401
+ end
402
+ end
403
+ end
404
+ when :array
405
+ result = value
406
+ when :string
407
+ if value.size > 1 # only one value wanted
408
+ return false, "Expected string, got #{value.inspect}"
409
+ end
410
+ result = value.first
411
+ when :number
412
+ if value.size > 1 # only one value wanted
413
+ return false, "Expected number, got #{value.inspect} (type #{value.class})"
414
+ end
415
+
416
+ v = value.first
417
+ case v
418
+ when Numeric
419
+ result = v
420
+ when String
421
+ if v.to_s.to_f.to_s != v.to_s \
422
+ && v.to_s.to_i.to_s != v.to_s
423
+ return false, "Expected number, got #{v.inspect} (type #{v})"
424
+ end
425
+ if v.include?(".")
426
+ # decimal value, use float.
427
+ result = v.to_f
428
+ else
429
+ result = v.to_i
430
+ end
431
+ end # case v
432
+ when :boolean
433
+ if value.size > 1 # only one value wanted
434
+ return false, "Expected boolean, got #{value.inspect}"
435
+ end
436
+
437
+ bool_value = value.first
438
+ if !!bool_value == bool_value
439
+ # is_a does not work for booleans
440
+ # we have Boolean and not a string
441
+ result = bool_value
442
+ else
443
+ if bool_value !~ /^(true|false)$/
444
+ return false, "Expected boolean 'true' or 'false', got #{bool_value.inspect}"
445
+ end
446
+
447
+ result = (bool_value == "true")
448
+ end
449
+ when :ipaddr
450
+ if value.size > 1 # only one value wanted
451
+ return false, "Expected IPaddr, got #{value.inspect}"
452
+ end
453
+
454
+ octets = value.split(".")
455
+ if octets.length != 4
456
+ return false, "Expected IPaddr, got #{value.inspect}"
457
+ end
458
+ octets.each do |o|
459
+ if o.to_i < 0 or o.to_i > 255
460
+ return false, "Expected IPaddr, got #{value.inspect}"
461
+ end
462
+ end
463
+ result = value.first
464
+ when :password
465
+ if value.size > 1
466
+ return false, "Expected password (one value), got #{value.size} values?"
467
+ end
468
+
469
+ result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
470
+ when :path
471
+ if value.size > 1 # Only 1 value wanted
472
+ return false, "Expected path (one value), got #{value.size} values?"
473
+ end
474
+
475
+ # Paths must be absolute
476
+ #if !Pathname.new(value.first).absolute?
477
+ #return false, "Require absolute path, got relative path #{value.first}?"
478
+ #end
479
+
480
+ if !File.exists?(value.first) # Check if the file exists
481
+ return false, "File does not exist or cannot be opened #{value.first}"
482
+ end
483
+
484
+ result = value.first
485
+ when :bytes
486
+ begin
487
+ bytes = Integer(value.first) rescue nil
488
+ result = bytes || Filesize.from(value.first).to_i
489
+ rescue ArgumentError
490
+ return false, "Unparseable filesize: #{value.first}. possible units (KiB, MiB, ...) e.g. '10 KiB'. doc reference: http://www.elastic.co/guide/en/logstash/current/configuration.html#bytes"
491
+ end
492
+ else
493
+ return false, "Unknown validator symbol #{validator}"
494
+ end # case validator
495
+ else
496
+ return false, "Unknown validator #{validator.class}"
497
+ end
498
+
499
+ # Return the validator for later use, like with type coercion.
500
+ return true, result
501
+ end # def validate_value
502
+
503
+ def secure_params!(params)
504
+ params.each do |key, value|
505
+ if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
506
+ params[key] = ::LogStash::Util::Password.new(value)
507
+ end
508
+ end
509
+ end
510
+
511
+ def hash_or_array(value)
512
+ if !value.is_a?(Hash)
513
+ value = [*value] # coerce scalar to array if necessary
514
+ end
515
+ return value
516
+ end
517
+ end # module LogStash::Config::DSL
518
+ end # module LogStash::Config