logstash-core 2.3.4.snapshot1-java → 2.4.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 54ad2dca97892c52abdd7681a54fe06463a5ec40
4
- data.tar.gz: 9a6231f80dc1c611b6465cdd11e6a9c0a65ffa3e
3
+ metadata.gz: 3ce17d1073206a2d7cfde3dd8e13aba375a8d797
4
+ data.tar.gz: 966d4c0af56bb4b9fe56d37efec5812037b0c677
5
5
  SHA512:
6
- metadata.gz: 15db28073a86f794a57487161e23701146dd94d71f38893b693026981d18bf6a30871fed36bd0ae15dc3d98b64302c1e051c44517dac7b973c906f76fbf14d74
7
- data.tar.gz: f382a8b4505459643327c96ce0857d909fbeca6a3f9779427171d9c020b3d93ed57b9f68e2df7cad3a5b16cb3e1e5bb060ded2722b9d004b205dff011af19c86
6
+ metadata.gz: ccb3f5b9f0d2e3a01e2524f8f14f041bbece04ed8f31ee40780950438990dd149032b578735b0a2d6446e4f8ceca5eb691faa44399709104860dc8177aa22f91
7
+ data.tar.gz: acf3e571a22a304866669cf10964d1843c5d73904c2ddb0b49d19a4af942134fa030f79528d8aa83bc09e22dae6e76e5084af6ac2559f07023680d652b7389e0
@@ -5,4 +5,4 @@
5
5
  # Note to authors: this should not include dashes because 'gem' barfs if
6
6
  # you include a dash in the version string.
7
7
 
8
- LOGSTASH_CORE_VERSION = "2.3.4.snapshot1"
8
+ LOGSTASH_CORE_VERSION = "2.4.0"
@@ -91,6 +91,10 @@ class LogStash::Agent < Clamp::Command
91
91
  I18n.t("logstash.agent.flag.allow-env"),
92
92
  :attribute_name => :allow_env, :default => false
93
93
 
94
+ option ["--[no-]log-in-json"], :flag,
95
+ I18n.t("logstash.agent.flag.log-in-json"),
96
+ :default => false
97
+
94
98
  def initialize(*params)
95
99
  super(*params)
96
100
  @logger = Cabin::Channel.get(LogStash)
@@ -129,11 +133,11 @@ class LogStash::Agent < Clamp::Command
129
133
  # Emit a warning message.
130
134
  def warn(message)
131
135
  # For now, all warnings are fatal.
132
- raise LogStash::ConfigurationError, message
136
+ signal_usage_error(message)
133
137
  end # def warn
134
138
 
135
139
  def fail(message)
136
- raise LogStash::ConfigurationError, message
140
+ signal_usage_error(message)
137
141
  end # def fail
138
142
 
139
143
  # Run the agent. This method is invoked after clamp parses the
@@ -142,6 +146,7 @@ class LogStash::Agent < Clamp::Command
142
146
  require "logstash/pipeline"
143
147
  require "cabin" # gem 'cabin'
144
148
  require "logstash/plugin"
149
+ require "logstash/logging/json"
145
150
 
146
151
  LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
147
152
  LogStash::ShutdownWatcher.logger = @logger
@@ -177,8 +182,7 @@ class LogStash::Agent < Clamp::Command
177
182
  end
178
183
 
179
184
  if config_test?
180
- config_loader = LogStash::Config::Loader.new(@logger)
181
- config_str = config_loader.format_config(config_path, config_string)
185
+ config_str = @config_loader.format_config(config_path, config_string)
182
186
  begin
183
187
  # currently the best strategy to validate the configuration
184
188
  # is creating a pipeline instance and checking for exceptions
@@ -198,6 +202,8 @@ class LogStash::Agent < Clamp::Command
198
202
  :allow_env => allow_env?
199
203
  }))
200
204
 
205
+ @thread = Thread.current # this var is implicilty used by Stud.stop?
206
+
201
207
  sigint_id = trap_sigint()
202
208
  sigterm_id = trap_sigterm()
203
209
  sighup_id = trap_sighup()
@@ -210,8 +216,6 @@ class LogStash::Agent < Clamp::Command
210
216
 
211
217
  return 1 if clean_state?
212
218
 
213
- @thread = Thread.current # this var is implicilty used by Stud.stop?
214
-
215
219
  Stud.stoppable_sleep(reload_interval) # sleep before looping
216
220
 
217
221
  if auto_reload?
@@ -237,7 +241,10 @@ class LogStash::Agent < Clamp::Command
237
241
  end
238
242
  return 1
239
243
  rescue => e
240
- @logger.unsubscribe(stdout_logs) if show_startup_errors
244
+ if show_startup_errors
245
+ @logger.terminal(e.message)
246
+ @logger.unsubscribe(stdout_logs)
247
+ end
241
248
  @logger.warn(I18n.t("oops"), :error => e, :class => e.class.name, :backtrace => e.backtrace)
242
249
  return 1
243
250
  ensure
@@ -294,11 +301,22 @@ class LogStash::Agent < Clamp::Command
294
301
 
295
302
  puts "Sending logstash logs to #{path}."
296
303
  @logger.unsubscribe(@logger_subscription) if @logger_subscription
297
- @logger_subscription = @logger.subscribe(@log_fd)
304
+ if log_in_json?
305
+ @logger_subscription = @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
306
+ @logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
307
+ else
308
+ @logger_subscription = @logger.subscribe(@log_fd)
309
+ @logger.subscribe(STDOUT, :level => :fatal)
310
+ end
298
311
  else
299
- @logger.subscribe(STDOUT)
312
+ if log_in_json?
313
+ @logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
314
+ else
315
+ @logger.subscribe(STDOUT)
316
+ end
300
317
  end
301
318
 
319
+
302
320
  # TODO(sissel): redirect stdout/stderr to the log as well
303
321
  # http://jira.codehaus.org/browse/JRUBY-7003
304
322
  end # def configure_logging
@@ -472,7 +490,7 @@ class LogStash::Agent < Clamp::Command
472
490
  begin
473
491
  pipeline.run
474
492
  rescue => e
475
- @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
493
+ @logger.error("Pipeline aborted due to error", :exception => e.class.name, :backtrace => e.backtrace)
476
494
  end
477
495
  end
478
496
  sleep 0.01 until pipeline.ready?
@@ -13,6 +13,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
13
13
  super
14
14
  config_init(@params)
15
15
  register if respond_to?(:register)
16
+ setup_multi_encode!
16
17
  end
17
18
 
18
19
  public
@@ -23,10 +24,37 @@ module LogStash::Codecs; class Base < LogStash::Plugin
23
24
  alias_method :<<, :decode
24
25
 
25
26
  public
27
+ # DEPRECATED: Prefer defining encode_sync or multi_encode
26
28
  def encode(event)
27
- raise "#{self.class}#encode must be overidden"
29
+ encoded = multi_encode([event])
30
+ encoded.each {|event,data| @on_event.call(event,data) }
28
31
  end # def encode
29
32
 
33
+ public
34
+ # Relies on the codec being synchronous (which they all are!)
35
+ # We need a better long term design here, but this is an improvement
36
+ # over the current API for shared plugins
37
+ # It is best if the codec implements this directly
38
+ def multi_encode(events)
39
+ if @has_encode_sync
40
+ events.map {|event| [event, self.encode_sync(event)]}
41
+ else
42
+ batch = Thread.current[:logstash_output_codec_batch] ||= []
43
+ batch.clear
44
+
45
+ events.each {|event| self.encode(event) }
46
+ batch
47
+ end
48
+ end
49
+
50
+ def setup_multi_encode!
51
+ @has_encode_sync = self.methods.include?(:encode_sync)
52
+
53
+ on_event do |event, data|
54
+ Thread.current[:logstash_output_codec_batch] << [event, data]
55
+ end
56
+ end
57
+
30
58
  public
31
59
  def close; end;
32
60
 
@@ -3,6 +3,7 @@ require "logstash/namespace"
3
3
  require "logstash/config/registry"
4
4
  require "logstash/logging"
5
5
  require "logstash/util/password"
6
+ require "logstash/util/safe_uri"
6
7
  require "logstash/version"
7
8
  require "logstash/environment"
8
9
  require "logstash/util/plugin_version"
@@ -333,58 +334,89 @@ module LogStash::Config::Mixin
333
334
  return true
334
335
  end # def validate_check_invalid_parameter_names
335
336
 
337
+ def validate_check_required_parameter(config_key, config_opts, k, v)
338
+ if config_key.is_a?(Regexp)
339
+ (k =~ config_key && v)
340
+ elsif config_key.is_a?(String)
341
+ k && v
342
+ end
343
+ end
344
+
336
345
  def validate_check_required_parameter_names(params)
337
346
  is_valid = true
338
347
 
339
348
  @config.each do |config_key, config|
340
349
  next unless config[:required]
341
350
 
342
- if config_key.is_a?(Regexp)
343
- next if params.keys.select { |k| k =~ config_key }.length > 0
344
- elsif config_key.is_a?(String)
345
- next if params.keys.member?(config_key)
351
+ value = params[config_key]
352
+
353
+ if config_key.is_a?(Regexp) && !params.keys.any? { |k| k =~ config_key }
354
+ is_valid = false
355
+ end
356
+
357
+ if value.nil? || (config[:list] && Array(value).empty?)
358
+ @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
359
+ :setting => config_key, :plugin => @plugin_name,
360
+ :type => @plugin_type))
361
+ is_valid = false
346
362
  end
347
- @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
348
- :setting => config_key, :plugin => @plugin_name,
349
- :type => @plugin_type))
350
- is_valid = false
351
363
  end
352
364
 
353
365
  return is_valid
354
366
  end
355
367
 
368
+ def process_parameter_value(value, config_settings)
369
+ config_val = config_settings[:validate]
370
+
371
+ if config_settings[:list]
372
+ value = Array(value) # coerce scalars to lists
373
+ # Empty lists are converted to nils
374
+ return true, nil if value.empty?
375
+
376
+ validated_items = value.map {|v| validate_value(v, config_val)}
377
+ is_valid = validated_items.all? {|sr| sr[0] }
378
+ processed_value = validated_items.map {|sr| sr[1]}
379
+ else
380
+ is_valid, processed_value = validate_value(value, config_val)
381
+ end
382
+
383
+ return [is_valid, processed_value]
384
+ end
385
+
356
386
  def validate_check_parameter_values(params)
357
387
  # Filter out parametrs that match regexp keys.
358
388
  # These are defined in plugins like this:
359
389
  # config /foo.*/ => ...
360
- is_valid = true
390
+ all_params_valid = true
361
391
 
362
392
  params.each do |key, value|
363
393
  @config.keys.each do |config_key|
364
394
  next unless (config_key.is_a?(Regexp) && key =~ config_key) \
365
395
  || (config_key.is_a?(String) && key == config_key)
366
- config_val = @config[config_key][:validate]
367
- #puts " Key matches."
368
- success, result = validate_value(value, config_val)
369
- if success
370
- # Accept coerced value if success
396
+
397
+ config_settings = @config[config_key]
398
+
399
+ is_valid, processed_value = process_parameter_value(value, config_settings)
400
+
401
+ if is_valid
402
+ # Accept coerced value if valid
371
403
  # Used for converting values in the config to proper objects.
372
- params[key] = result if !result.nil?
404
+ params[key] = processed_value
373
405
  else
374
406
  @logger.error(I18n.t("logstash.agent.configuration.setting_invalid",
375
407
  :plugin => @plugin_name, :type => @plugin_type,
376
408
  :setting => key, :value => value.inspect,
377
- :value_type => config_val,
378
- :note => result))
409
+ :value_type => config_settings[:validate],
410
+ :note => processed_value))
379
411
  end
380
- #puts "Result: #{key} / #{result.inspect} / #{success}"
381
- is_valid &&= success
412
+
413
+ all_params_valid &&= is_valid
382
414
 
383
415
  break # done with this param key
384
416
  end # config.each
385
417
  end # params.each
386
418
 
387
- return is_valid
419
+ return all_params_valid
388
420
  end # def validate_check_parameter_values
389
421
 
390
422
  def validator_find(key)
@@ -405,7 +437,7 @@ module LogStash::Config::Mixin
405
437
  result = nil
406
438
 
407
439
  if validator.nil?
408
- return true
440
+ return true, value
409
441
  elsif validator.is_a?(Array)
410
442
  value = [*value]
411
443
  if value.size > 1
@@ -519,6 +551,12 @@ module LogStash::Config::Mixin
519
551
  end
520
552
 
521
553
  result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
554
+ when :uri
555
+ if value.size > 1
556
+ return false, "Expected uri (one value), got #{value.size} values?"
557
+ end
558
+
559
+ result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
522
560
  when :path
523
561
  if value.size > 1 # Only 1 value wanted
524
562
  return false, "Expected path (one value), got #{value.size} values?"
@@ -554,8 +592,9 @@ module LogStash::Config::Mixin
554
592
 
555
593
  def secure_params!(params)
556
594
  params.each do |key, value|
557
- if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
558
- params[key] = ::LogStash::Util::Password.new(value)
595
+ if [:uri, :password].include? @config[key][:validate]
596
+ is_valid, processed_value = process_parameter_value(value, @config[key])
597
+ params[key] = processed_value
559
598
  end
560
599
  end
561
600
  end
@@ -78,7 +78,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
78
78
 
79
79
  public
80
80
  def do_stop
81
- @logger.debug("stopping", :plugin => self)
81
+ @logger.debug("stopping", :plugin => self.class.name)
82
82
  @stop_called.make_true
83
83
  stop
84
84
  end
@@ -0,0 +1,45 @@
1
+ # encoding: utf-8
2
+
3
+ module LogStash module Instrument
4
+ # This class is used in the context when we disable the metric collection
5
+ # for specific plugin to replace the `NamespacedMetric` class with this one
6
+ # which doesn't produce any metric to the collector.
7
+ class NullMetric
8
+ attr_reader :namespace_name, :collector
9
+
10
+ def increment(key, value = 1)
11
+ end
12
+
13
+ def decrement(key, value = 1)
14
+ end
15
+
16
+ def gauge(key, value)
17
+ end
18
+
19
+ def report_time(key, duration)
20
+ end
21
+
22
+ # We have to manually redefine this method since it can return an
23
+ # object this object also has to be implemented as a NullObject
24
+ def time(key, &block)
25
+ if block_given?
26
+ yield
27
+ else
28
+ NullTimedExecution
29
+ end
30
+ end
31
+
32
+ def namespace(name)
33
+ self.class.new
34
+ end
35
+
36
+ private
37
+ # Null implementation of the internal timer class
38
+ #
39
+ # @see LogStash::Instrument::TimedExecution`
40
+ class NullTimedExecution
41
+ def self.stop
42
+ end
43
+ end
44
+ end
45
+ end; end
@@ -0,0 +1,21 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "logstash/logging"
4
+ require "logstash/json"
5
+
6
+ module LogStash; class Logging; class JSON
7
+ def initialize(io)
8
+ raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
9
+
10
+ @io = io
11
+ @lock = Mutex.new
12
+ end
13
+
14
+ def <<(obj)
15
+ serialized = LogStash::Json.dump(obj)
16
+ @lock.synchronize do
17
+ @io.puts(serialized)
18
+ @io.flush
19
+ end
20
+ end
21
+ end; end; end
@@ -74,7 +74,7 @@ module LogStash class OutputDelegator
74
74
  @workers << @klass.new(@config)
75
75
  @workers.first.register # Needed in case register calls `workers_not_supported`
76
76
 
77
- @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
77
+ @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
78
78
 
79
79
  # Threadsafe versions don't need additional workers
80
80
  setup_additional_workers!(target_worker_count) unless @threadsafe
@@ -134,7 +134,7 @@ module LogStash class OutputDelegator
134
134
  end
135
135
 
136
136
  def do_close
137
- @logger.debug("closing output delegator", :klass => @klass)
137
+ @logger.debug("closing output delegator", :klass => @klass.name)
138
138
 
139
139
  if @threadsafe
140
140
  @workers.each(&:do_close)
@@ -169,4 +169,4 @@ module LogStash class OutputDelegator
169
169
  private
170
170
  # Needed for testing, so private
171
171
  attr_reader :threadsafe_worker, :worker_queue
172
- end end
172
+ end end
@@ -27,6 +27,22 @@ class LogStash::Outputs::Base < LogStash::Plugin
27
27
 
28
28
  attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
29
29
 
30
+ # Set or return concurrency type
31
+ def self.concurrency(type=nil)
32
+ if type
33
+ @concurrency = type
34
+
35
+ if type == :shared
36
+ declare_threadsafe!
37
+ elsif type == :single
38
+ declare_workers_not_supported!("This plugin only supports one worker!")
39
+ end
40
+
41
+ else
42
+ @concurrency || :legacy # default is :legacyo
43
+ end
44
+ end
45
+
30
46
  def self.declare_threadsafe!
31
47
  declare_workers_not_supported!
32
48
  @threadsafe = true
@@ -65,6 +81,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
65
81
  # If we're running with a single thread we must enforce single-threaded concurrency by default
66
82
  # Maybe in a future version we'll assume output plugins are threadsafe
67
83
  @single_worker_mutex = Mutex.new
84
+
85
+ @receives_encoded = self.methods.include?(:multi_receive_encoded)
68
86
  end
69
87
 
70
88
  public
@@ -77,10 +95,23 @@ class LogStash::Outputs::Base < LogStash::Plugin
77
95
  raise "#{self.class}#receive must be overidden"
78
96
  end # def receive
79
97
 
98
+ public
99
+ def concurrency
100
+ self.class.concurrency
101
+ end
102
+
80
103
  public
81
104
  # To be overriden in implementations
82
105
  def multi_receive(events)
83
- events.each {|event| receive(event) }
106
+ if @receives_encoded
107
+ self.multi_receive_encoded(codec.multi_encode(events))
108
+ else
109
+ events.each {|event| receive(event) }
110
+ end
111
+ end
112
+
113
+ def codec
114
+ params["codec"]
84
115
  end
85
116
 
86
117
  private