fluentd 1.10.1-x86-mingw32 → 1.11.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +112 -1
  3. data/CONTRIBUTING.md +1 -1
  4. data/lib/fluent/command/debug.rb +1 -0
  5. data/lib/fluent/command/fluentd.rb +12 -1
  6. data/lib/fluent/config.rb +1 -0
  7. data/lib/fluent/log.rb +45 -6
  8. data/lib/fluent/match.rb +1 -1
  9. data/lib/fluent/plugin/in_dummy.rb +2 -2
  10. data/lib/fluent/plugin/in_forward.rb +2 -2
  11. data/lib/fluent/plugin/in_gc_stat.rb +16 -0
  12. data/lib/fluent/plugin/in_http.rb +146 -75
  13. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  14. data/lib/fluent/plugin/in_syslog.rb +4 -4
  15. data/lib/fluent/plugin/in_tail.rb +4 -4
  16. data/lib/fluent/plugin/in_unix.rb +77 -77
  17. data/lib/fluent/plugin/out_copy.rb +1 -1
  18. data/lib/fluent/plugin/out_file.rb +1 -1
  19. data/lib/fluent/plugin/out_forward.rb +23 -18
  20. data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
  21. data/lib/fluent/plugin/out_http.rb +15 -2
  22. data/lib/fluent/plugin/parser_multiline.rb +1 -1
  23. data/lib/fluent/plugin/parser_syslog.rb +215 -54
  24. data/lib/fluent/plugin_helper/child_process.rb +3 -2
  25. data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
  26. data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
  27. data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
  28. data/lib/fluent/plugin_helper/socket.rb +20 -2
  29. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  30. data/lib/fluent/supervisor.rb +21 -9
  31. data/lib/fluent/system_config.rb +2 -1
  32. data/lib/fluent/test/filter_test.rb +2 -2
  33. data/lib/fluent/test/output_test.rb +3 -3
  34. data/lib/fluent/version.rb +1 -1
  35. data/test/command/test_fluentd.rb +57 -10
  36. data/test/config/test_system_config.rb +2 -0
  37. data/test/plugin/out_forward/test_load_balancer.rb +46 -0
  38. data/test/plugin/test_in_gc_stat.rb +24 -1
  39. data/test/plugin/test_in_http.rb +57 -0
  40. data/test/plugin/test_in_syslog.rb +1 -1
  41. data/test/plugin/test_in_tail.rb +20 -16
  42. data/test/plugin/test_in_unix.rb +128 -73
  43. data/test/plugin/test_out_forward.rb +11 -2
  44. data/test/plugin/test_out_http.rb +38 -0
  45. data/test/plugin/test_out_null.rb +1 -1
  46. data/test/plugin/test_output_as_buffered_retries.rb +12 -4
  47. data/test/plugin/test_parser_syslog.rb +66 -29
  48. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  49. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  50. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  51. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  52. data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
  53. data/test/plugin_helper/http_server/test_app.rb +1 -1
  54. data/test/plugin_helper/http_server/test_route.rb +1 -1
  55. data/test/plugin_helper/test_child_process.rb +15 -0
  56. data/test/plugin_helper/test_http_server_helper.rb +2 -2
  57. data/test/plugin_helper/test_record_accessor.rb +41 -0
  58. data/test/plugin_helper/test_server.rb +1 -1
  59. data/test/plugin_helper/test_service_discovery.rb +37 -4
  60. data/test/plugin_helper/test_socket.rb +131 -0
  61. data/test/test_log.rb +44 -0
  62. metadata +12 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f0cc7030846081961256c91d06ccf96666d771e21b9b97718b35ec176a051462
4
- data.tar.gz: 3718a462fc5dbf3b2c151e912d90faa298437ff5d657aba20cfc5210a522c990
3
+ metadata.gz: c5c0bb8a881ac7bdbad86f9261310b8d3e7f505c3ec085594237a6f01b6bd2b3
4
+ data.tar.gz: 5436e5d40545b9c3131abd63eeb33a94663ea8c98bd492b3b1d1d92ddfd616c7
5
5
  SHA512:
6
- metadata.gz: 81a3c1a791d4de6ff0d1b965c5fd1ac685e46cdb5b6780683a85633574c6626d4e2b93e32c51252ecc9d5fc5db662937c70521aabc0272fd75e5bea1bfb2a9fe
7
- data.tar.gz: 5c75619ed35a92696b0a3dcb425aef1d2f684c381f8870d00ac4e9e8112911b4579cffd46db5fd003078d8bc6e07db91ac405e4385aaaf2f9db43cb7b9461efd
6
+ metadata.gz: 0ba7de652d79100f15bd5c921d233260fa8ce97d4c89dde51a2c4c1ea76bc8ef363e76a51bb179634e0d9e30edea0bbf2f80f4fe8cca0baea1e2d1c4964664f3
7
+ data.tar.gz: 7273f92710cab57e9368623641efcd31bcd9aced03db45eb5f9316466ffaae0209725800d35abc1c522428ebf7b4280a9a8a812793dfa1a2909ab2db74eb80a4
@@ -1,5 +1,116 @@
1
+ # v1.11
2
+
3
+ ## Release v1.11.1 - 2020/06/22
4
+
5
+ ### Enhancement
6
+
7
+ * in_http: Add `dump_error_log` parameter
8
+ https://github.com/fluent/fluentd/pull/3035
9
+ * in_http: Improve time field handling
10
+ https://github.com/fluent/fluentd/pull/3046
11
+ * Refactoring code
12
+ https://github.com/fluent/fluentd/pull/3047
13
+
14
+ ### Bug fix
15
+
16
+ * in_tail: Use actual path instead of based pattern for ignore list
17
+ https://github.com/fluent/fluentd/pull/3042
18
+ * child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout
19
+ https://github.com/fluent/fluentd/pull/3044
20
+
21
+ ## Release v1.11.0 - 2020/06/04
22
+
23
+ ### New feature
24
+
25
+ * in_unix: Use v1 API
26
+ https://github.com/fluent/fluentd/pull/2992
27
+
28
+ ### Enhancement
29
+
30
+ * parser_syslog: Support any `time_format` for RFC3164 string parser
31
+ https://github.com/fluent/fluentd/pull/3014
32
+ * parser_syslog: Add new parser for RFC5424
33
+ https://github.com/fluent/fluentd/pull/3015
34
+ * Refactoring code
35
+ https://github.com/fluent/fluentd/pull/3019
36
+
37
+ ### Bug fix
38
+
39
+ * in_gc_stat: Add `use_symbol_keys` parameter to emit string key record
40
+ https://github.com/fluent/fluentd/pull/3008
41
+
1
42
  # v1.10
2
43
 
44
+ ## Release v1.10.4 - 2020/05/12
45
+
46
+ ### Enhancement
47
+
48
+ * out_http: Support single json array payload
49
+ https://github.com/fluent/fluentd/pull/2973
50
+ * Refactoring
51
+ https://github.com/fluent/fluentd/pull/2988
52
+
53
+ ### Bug fix
54
+
55
+ * supervisor: Call `File.umask(0)` for standalone worker
56
+ https://github.com/fluent/fluentd/pull/2987
57
+ * out_forward: Fix ZeroDivisionError issue with `weight 0`
58
+ https://github.com/fluent/fluentd/pull/2989
59
+
60
+ ## Release v1.10.3 - 2020/05/01
61
+
62
+ ### Enhancement
63
+
64
+ * record_accessor: Add `set` method
65
+ https://github.com/fluent/fluentd/pull/2977
66
+ * config: Ruby DSL format is deprecated
67
+ https://github.com/fluent/fluentd/pull/2958
68
+ * Refactor code
69
+ https://github.com/fluent/fluentd/pull/2961
70
+ https://github.com/fluent/fluentd/pull/2962
71
+ https://github.com/fluent/fluentd/pull/2965
72
+ https://github.com/fluent/fluentd/pull/2966
73
+ https://github.com/fluent/fluentd/pull/2978
74
+
75
+ ### Bug fix
76
+
77
+ * out_forward: Disable `linger_timeout` setting on Windows
78
+ https://github.com/fluent/fluentd/pull/2959
79
+ * out_forward: Fix warning of service discovery manager when fluentd stops
80
+ https://github.com/fluent/fluentd/pull/2974
81
+
82
+ ## Release v1.10.2 - 2020/04/15
83
+
84
+ ### Enhancement
85
+
86
+ * out_copy: Add plugin_id to log message
87
+ https://github.com/fluent/fluentd/pull/2934
88
+ * socket: Allow cert chains in mutual auth
89
+ https://github.com/fluent/fluentd/pull/2930
90
+ * system: Add ignore_repeated_log_interval parameter
91
+ https://github.com/fluent/fluentd/pull/2937
92
+ * windows: Allow to launch fluentd from whitespace included path
93
+ https://github.com/fluent/fluentd/pull/2920
94
+ * Refactor code
95
+ https://github.com/fluent/fluentd/pull/2935
96
+ https://github.com/fluent/fluentd/pull/2936
97
+ https://github.com/fluent/fluentd/pull/2938
98
+ https://github.com/fluent/fluentd/pull/2939
99
+ https://github.com/fluent/fluentd/pull/2946
100
+
101
+ ### Bug fix
102
+
103
+ * in_syslog: Fix octet-counting mode bug
104
+ https://github.com/fluent/fluentd/pull/2942
105
+ * out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set
106
+ https://github.com/fluent/fluentd/pull/2943
107
+ * out_forward: Need authentication when sending tcp heartbeat with keepalive
108
+ https://github.com/fluent/fluentd/pull/2945
109
+ * command: Fix fluent-debug start failure
110
+ https://github.com/fluent/fluentd/pull/2948
111
+ * command: Fix regression of supervisor's worker and `--daemon` combo
112
+ https://github.com/fluent/fluentd/pull/2950
113
+
3
114
  ## Release v1.10.1 - 2020/04/02
4
115
 
5
116
  ### Enhancement
@@ -27,7 +138,7 @@
27
138
 
28
139
  ### New feature
29
140
 
30
- * sd plugin: Add RSV record plugin
141
+ * sd plugin: Add SRV record plugin
31
142
  https://github.com/fluent/fluentd/pull/2876
32
143
 
33
144
  ### Enhancement
@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
24
24
  submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
25
25
 
26
26
  * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
27
- * **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
27
+ * **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
28
28
 
29
29
  If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
30
30
  And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.
@@ -61,6 +61,7 @@ else
61
61
  end
62
62
 
63
63
  require 'fluent/log'
64
+ require 'fluent/env'
64
65
  require 'fluent/engine'
65
66
  require 'fluent/system_config'
66
67
  require 'serverengine'
@@ -185,6 +185,10 @@ if Fluent.windows?
185
185
  opts[:regwinsvcautostart] = s
186
186
  }
187
187
 
188
+ op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
189
+ opts[:regwinsvcdelaystart] = s
190
+ }
191
+
188
192
  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
189
193
  opts[:fluentdopt] = s
190
194
  }
@@ -285,6 +289,13 @@ if winsvcinstmode = opts[:regwinsvc]
285
289
  dependencies: [""],
286
290
  display_name: opts[:winsvc_display_name]
287
291
  )
292
+
293
+ if opts[:regwinsvcdelaystart]
294
+ Service.configure(
295
+ service_name: opts[:winsvc_name],
296
+ delayed_start: true
297
+ )
298
+ end
288
299
  when 'u'
289
300
  if Service.status(opts[:winsvc_name]).current_state != 'stopped'
290
301
  begin
@@ -336,7 +347,7 @@ else
336
347
  worker = Fluent::Supervisor.new(opts)
337
348
  worker.configure
338
349
 
339
- if opts[:daemonize]
350
+ if opts[:daemonize] && opts[:standalone_worker]
340
351
  require 'fluent/daemonizer'
341
352
  args = ARGV.dup
342
353
  i = args.index('--daemon')
@@ -62,6 +62,7 @@ module Fluent
62
62
  Parser.parse(str, fname, basepath)
63
63
  when :ruby
64
64
  require 'fluent/config/dsl'
65
+ $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
65
66
  Config::DSL::Parser.parse(str, File.join(basepath, fname))
66
67
  else
67
68
  raise "[BUG] unknown configuration parser specification:'#{parser}'"
@@ -112,6 +112,7 @@ module Fluent
112
112
  @optional_attrs = nil
113
113
 
114
114
  @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
115
+ @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
115
116
 
116
117
  @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
117
118
  @process_type ||= :standalone # to keep behavior of existing code
@@ -139,7 +140,8 @@ module Fluent
139
140
  dl_opts = {}
140
141
  dl_opts[:log_level] = @level - 1
141
142
  logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
142
- clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
143
+ clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
144
+ worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
143
145
  clone.format = @format
144
146
  clone.time_format = @time_format
145
147
  clone.log_event_enabled = @log_event_enabled
@@ -149,7 +151,7 @@ module Fluent
149
151
 
150
152
  attr_reader :format
151
153
  attr_reader :time_format
152
- attr_accessor :log_event_enabled
154
+ attr_accessor :log_event_enabled, :ignore_repeated_log_interval
153
155
  attr_accessor :out
154
156
  attr_accessor :level
155
157
  attr_accessor :optional_header, :optional_attrs
@@ -278,6 +280,7 @@ module Fluent
278
280
  return if skipped_type?(type)
279
281
  args << block.call if block
280
282
  time, msg = event(:trace, args)
283
+ return if time.nil?
281
284
  puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
282
285
  rescue
283
286
  # logger should not raise an exception. This rescue prevents unexpected behaviour.
@@ -299,6 +302,7 @@ module Fluent
299
302
  return if skipped_type?(type)
300
303
  args << block.call if block
301
304
  time, msg = event(:debug, args)
305
+ return if time.nil?
302
306
  puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
303
307
  rescue
304
308
  end
@@ -319,6 +323,7 @@ module Fluent
319
323
  return if skipped_type?(type)
320
324
  args << block.call if block
321
325
  time, msg = event(:info, args)
326
+ return if time.nil?
322
327
  puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
323
328
  rescue
324
329
  end
@@ -339,6 +344,7 @@ module Fluent
339
344
  return if skipped_type?(type)
340
345
  args << block.call if block
341
346
  time, msg = event(:warn, args)
347
+ return if time.nil?
342
348
  puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
343
349
  rescue
344
350
  end
@@ -359,6 +365,7 @@ module Fluent
359
365
  return if skipped_type?(type)
360
366
  args << block.call if block
361
367
  time, msg = event(:error, args)
368
+ return if time.nil?
362
369
  puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
363
370
  rescue
364
371
  end
@@ -379,6 +386,7 @@ module Fluent
379
386
  return if skipped_type?(type)
380
387
  args << block.call if block
381
388
  time, msg = event(:fatal, args)
389
+ return if time.nil?
382
390
  puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
383
391
  rescue
384
392
  end
@@ -412,6 +420,20 @@ module Fluent
412
420
  @out.reset if @out.respond_to?(:reset)
413
421
  end
414
422
 
423
+ CachedLog = Struct.new(:msg, :time)
424
+
425
+ def ignore_repeated_log?(key, time, message)
426
+ cached_log = Thread.current[key]
427
+ return false if cached_log.nil?
428
+ (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
429
+ end
430
+
431
+ def suppress_stacktrace?(backtrace)
432
+ cached_log = Thread.current[:last_repeated_stacktrace]
433
+ return false if cached_log.nil?
434
+ cached_log.msg == backtrace
435
+ end
436
+
415
437
  def dump_stacktrace(type, backtrace, level)
416
438
  return if @level > level
417
439
 
@@ -419,13 +441,16 @@ module Fluent
419
441
 
420
442
  if @format == :text
421
443
  line = caller_line(type, time, 5, level)
422
- if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
444
+ if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
445
+ return
446
+ elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
423
447
  puts [" ", line, 'suppressed same stacktrace'].join
448
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
424
449
  else
425
450
  backtrace.each { |msg|
426
451
  puts [" ", line, msg].join
427
452
  }
428
- Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
453
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
429
454
  end
430
455
  else
431
456
  r = {
@@ -436,11 +461,14 @@ module Fluent
436
461
  r['worker_id'] = wid
437
462
  end
438
463
 
439
- if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
464
+ if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
465
+ return
466
+ elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
440
467
  r['message'] = 'suppressed same stacktrace'
468
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
441
469
  else
442
470
  r['message'] = backtrace.join("\n")
443
- Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
471
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
444
472
  end
445
473
 
446
474
  puts Yajl.dump(r)
@@ -479,6 +507,14 @@ module Fluent
479
507
  end
480
508
  }
481
509
 
510
+ if @ignore_repeated_log_interval
511
+ if ignore_repeated_log?(:last_repeated_log, time, message)
512
+ return nil, nil
513
+ else
514
+ Thread.current[:last_repeated_log] = CachedLog.new(message, time)
515
+ end
516
+ end
517
+
482
518
  if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
483
519
  record = map.dup
484
520
  record.keys.each {|key|
@@ -530,6 +566,9 @@ module Fluent
530
566
  if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
531
567
  @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
532
568
  end
569
+ if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
570
+ @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
571
+ end
533
572
 
534
573
  self.format = @logger.format
535
574
  self.time_format = @logger.time_format
@@ -125,7 +125,7 @@ module Fluent
125
125
  end
126
126
 
127
127
  def match(str)
128
- @regex.match(str) != nil
128
+ @regex.match?(str)
129
129
  end
130
130
  end
131
131
 
@@ -105,10 +105,10 @@ module Fluent::Plugin
105
105
  begin
106
106
  if @size > 1
107
107
  num.times do
108
- router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
108
+ router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
109
109
  end
110
110
  else
111
- num.times { router.emit(@tag, Fluent::Engine.now, generate) }
111
+ num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
112
112
  end
113
113
  rescue => _
114
114
  # ignore all errors not to stop emits by emit errors
@@ -327,7 +327,7 @@ module Fluent::Plugin
327
327
  record = e[1]
328
328
  next if record.nil?
329
329
  time = e[0]
330
- time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
330
+ time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
331
331
  es.add(time, record)
332
332
  }
333
333
  es
@@ -347,7 +347,7 @@ module Fluent::Plugin
347
347
  return msg[3] # retry never succeeded so return ack and drop incoming event.
348
348
  end
349
349
  return if record.nil?
350
- time = Fluent::Engine.now if time.to_i == 0
350
+ time = Fluent::EventTime.now if time.to_i == 0
351
351
  if @enable_field_injection
352
352
  record[@source_address_key] = conn.remote_addr if @source_address_key
353
353
  record[@source_hostname_key] = conn.remote_host if @source_hostname_key
@@ -24,13 +24,22 @@ module Fluent::Plugin
24
24
 
25
25
  def initialize
26
26
  super
27
+ @key_map = nil
27
28
  end
28
29
 
29
30
  config_param :emit_interval, :time, default: 60
31
+ config_param :use_symbol_keys, :bool, default: true
30
32
  config_param :tag, :string
31
33
 
32
34
  def configure(conf)
33
35
  super
36
+
37
+ unless @use_symbol_keys
38
+ @key_map = {}
39
+ GC.stat.each_key { |key|
40
+ @key_map[key] = key.to_s
41
+ }
42
+ end
34
43
  end
35
44
 
36
45
  def multi_workers_ready?
@@ -50,6 +59,13 @@ module Fluent::Plugin
50
59
  def on_timer
51
60
  now = Fluent::EventTime.now
52
61
  record = GC.stat
62
+ unless @use_symbol_keys
63
+ new_record = {}
64
+ record.each_pair { |k, v|
65
+ new_record[@key_map[k]] = v
66
+ }
67
+ record = new_record
68
+ end
53
69
  router.emit(@tag, now, record)
54
70
  end
55
71
  end
@@ -27,10 +27,28 @@ require 'json'
27
27
  module Fluent::Plugin
28
28
  class InHttpParser < Parser
29
29
  Fluent::Plugin.register_parser('in_http', self)
30
+
31
+ config_set_default :time_key, 'time'
32
+
33
+ def configure(conf)
34
+ super
35
+
36
+ # if no time parser related parameters, use in_http's time convert rule
37
+ @time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')
38
+ time_parser_create
39
+ else
40
+ nil
41
+ end
42
+ end
43
+
30
44
  def parse(text)
31
45
  # this plugin is dummy implementation not to raise error
32
46
  yield nil, nil
33
47
  end
48
+
49
+ def get_time_parser
50
+ @time_parser
51
+ end
34
52
  end
35
53
 
36
54
  class HttpInput < Input
@@ -60,6 +78,8 @@ module Fluent::Plugin
60
78
  config_param :respond_with_empty_img, :bool, default: false
61
79
  desc 'Respond status code with 204.'
62
80
  config_param :use_204_response, :bool, default: false
81
+ desc 'Dump error log or not'
82
+ config_param :dump_error_log, :bool, default: true
63
83
 
64
84
  config_section :parse do
65
85
  config_set_default :@type, 'in_http'
@@ -67,6 +87,24 @@ module Fluent::Plugin
67
87
 
68
88
  EVENT_RECORD_PARAMETER = '_event_record'
69
89
 
90
+ def initialize
91
+ super
92
+
93
+ @km = nil
94
+ @format_name = nil
95
+ @parser_time_key = nil
96
+
97
+ # default parsers
98
+ @parser_msgpack = nil
99
+ @parser_json = nil
100
+ @default_time_parser = nil
101
+ @default_keep_time_key = nil
102
+ @float_time_parser = nil
103
+
104
+ # <parse> configured parser
105
+ @custom_parser = nil
106
+ end
107
+
70
108
  def configure(conf)
71
109
  compat_parameters_convert(conf, :parser)
72
110
 
@@ -74,20 +112,22 @@ module Fluent::Plugin
74
112
 
75
113
  m = if @parser_configs.first['@type'] == 'in_http'
76
114
  @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
115
+ @parser_msgpack.time_key = nil
77
116
  @parser_msgpack.estimate_current_event = false
78
117
  @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
118
+ @parser_json.time_key = nil
79
119
  @parser_json.estimate_current_event = false
120
+
121
+ default_parser = parser_create(usage: '')
80
122
  @format_name = 'default'
81
- @parser_time_key = if parser_config = conf.elements('parse').first
82
- parser_config['time_key'] || 'time'
83
- else
84
- 'time'
85
- end
123
+ @parser_time_key = default_parser.time_key
124
+ @default_time_parser = default_parser.get_time_parser
125
+ @default_keep_time_key = default_parser.keep_time_key
86
126
  method(:parse_params_default)
87
127
  else
88
- @parser = parser_create
128
+ @custom_parser = parser_create
89
129
  @format_name = @parser_configs.first['@type']
90
- @parser_time_key = @parser.time_key
130
+ @parser_time_key = @custom_parser.time_key
91
131
  method(:parse_params_with_parser)
92
132
  end
93
133
  self.singleton_class.module_eval do
@@ -142,6 +182,13 @@ module Fluent::Plugin
142
182
  super
143
183
  end
144
184
 
185
+ RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze
186
+ RESPONSE_200 = ["200 OK".freeze, RES_TEXT_HEADER, "".freeze].freeze
187
+ RESPONSE_204 = ["204 No Content".freeze, {}.freeze].freeze
188
+ RESPONSE_IMG = ["200 OK".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze
189
+ RES_400_STATUS = "400 Bad Request".freeze
190
+ RES_500_STATUS = "500 Internal Server Error".freeze
191
+
145
192
  def on_request(path_info, params)
146
193
  begin
147
194
  path = path_info[1..-1] # remove /
@@ -152,83 +199,77 @@ module Fluent::Plugin
152
199
  if record.nil?
153
200
  log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
154
201
  if @respond_with_empty_img
155
- return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
202
+ return RESPONSE_IMG
156
203
  else
157
204
  if @use_204_response
158
- return ["204 No Content", {}]
205
+ return RESPONSE_204
159
206
  else
160
- return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
207
+ return RESPONSE_200
161
208
  end
162
209
  end
163
210
  end
164
211
 
165
- unless record.is_a?(Array)
166
- if @add_http_headers
167
- params.each_pair { |k,v|
168
- if k.start_with?("HTTP_")
169
- record[k] = v
170
- end
171
- }
172
- end
173
- if @add_remote_addr
174
- record['REMOTE_ADDR'] = params['REMOTE_ADDR']
175
- end
176
- end
177
- time = if param_time = params['time']
178
- param_time = param_time.to_f
179
- param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
180
- else
181
- record_time.nil? ? Fluent::Engine.now : record_time
182
- end
183
- rescue
184
- return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
185
- end
186
-
187
- # TODO server error
188
- begin
212
+ mes = nil
189
213
  # Support batched requests
190
214
  if record.is_a?(Array)
191
215
  mes = Fluent::MultiEventStream.new
192
216
  record.each do |single_record|
193
- if @add_http_headers
194
- params.each_pair { |k,v|
195
- if k.start_with?("HTTP_")
196
- single_record[k] = v
197
- end
198
- }
199
- end
200
- if @add_remote_addr
201
- single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
202
- end
203
-
204
- if defined? @parser
205
- single_time = @parser.parse_time(single_record)
206
- single_time, single_record = @parser.convert_values(single_time, single_record)
217
+ add_params_to_record(single_record, params)
218
+
219
+ if param_time = params['time']
220
+ param_time = param_time.to_f
221
+ single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
222
+ elsif @custom_parser
223
+ single_time = @custom_parser.parse_time(single_record)
224
+ single_time, single_record = @custom_parser.convert_values(single_time, single_record)
207
225
  else
208
- single_time = if t = single_record.delete(@parser_time_key)
209
- Fluent::EventTime.from_time(Time.at(t))
210
- else
211
- time
212
- end
226
+ single_time = convert_time_field(single_record)
213
227
  end
214
228
 
215
229
  mes.add(single_time, single_record)
216
230
  end
231
+ else
232
+ add_params_to_record(record, params)
233
+
234
+ time = if param_time = params['time']
235
+ param_time = param_time.to_f
236
+ param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
237
+ else
238
+ if record_time.nil?
239
+ convert_time_field(record)
240
+ else
241
+ record_time
242
+ end
243
+ end
244
+ end
245
+ rescue => e
246
+ if @dump_error_log
247
+ log.error "failed to process request", error: e
248
+ end
249
+ return [RES_400_STATUS, RES_TEXT_HEADER, "400 Bad Request\n#{e}\n"]
250
+ end
251
+
252
+ # TODO server error
253
+ begin
254
+ if mes
217
255
  router.emit_stream(tag, mes)
218
256
  else
219
257
  router.emit(tag, time, record)
220
258
  end
221
- rescue
222
- return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
259
+ rescue => e
260
+ if @dump_error_log
261
+ log.error "failed to emit data", error: e
262
+ end
263
+ return [RES_500_STATUS, RES_TEXT_HEADER, "500 Internal Server Error\n#{e}\n"]
223
264
  end
224
265
 
225
266
  if @respond_with_empty_img
226
- return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
267
+ return RESPONSE_IMG
227
268
  else
228
269
  if @use_204_response
229
- return ["204 No Content", {}]
270
+ return RESPONSE_204
230
271
  else
231
- return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
272
+ return RESPONSE_200
232
273
  end
233
274
  end
234
275
  end
@@ -267,7 +308,7 @@ module Fluent::Plugin
267
308
 
268
309
  def parse_params_with_parser(params)
269
310
  if content = params[EVENT_RECORD_PARAMETER]
270
- @parser.parse(content) { |time, record|
311
+ @custom_parser.parse(content) { |time, record|
271
312
  raise "Received event is not #{@format_name}: #{content}" if record.nil?
272
313
  return time, record
273
314
  }
@@ -276,6 +317,32 @@ module Fluent::Plugin
276
317
  end
277
318
  end
278
319
 
320
+ def add_params_to_record(record, params)
321
+ if @add_http_headers
322
+ params.each_pair { |k, v|
323
+ if k.start_with?("HTTP_".freeze)
324
+ record[k] = v
325
+ end
326
+ }
327
+ end
328
+
329
+ if @add_remote_addr
330
+ record['REMOTE_ADDR'] = params['REMOTE_ADDR']
331
+ end
332
+ end
333
+
334
+ def convert_time_field(record)
335
+ if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)
336
+ if @default_time_parser
337
+ @default_time_parser.parse(t)
338
+ else
339
+ Fluent::EventTime.from_time(Time.at(t))
340
+ end
341
+ else
342
+ Fluent::EventTime.now
343
+ end
344
+ end
345
+
279
346
  class Handler
280
347
  attr_reader :content_type
281
348
 
@@ -358,7 +425,7 @@ module Fluent::Plugin
358
425
  end
359
426
  }
360
427
  if expect
361
- if expect == '100-continue'
428
+ if expect == '100-continue'.freeze
362
429
  if !size || size < @body_size_limit
363
430
  send_response_nobody("100 Continue", {})
364
431
  else
@@ -380,17 +447,20 @@ module Fluent::Plugin
380
447
  @body << chunk
381
448
  end
382
449
 
450
+ RES_200_STATUS = "200 OK".freeze
451
+ RES_403_STATUS = "403 Forbidden".freeze
452
+
383
453
  # Web browsers can send an OPTIONS request before performing POST
384
454
  # to check if cross-origin requests are supported.
385
455
  def handle_options_request
386
456
  # Is CORS enabled in the first place?
387
457
  if @cors_allow_origins.nil?
388
- return send_response_and_close("403 Forbidden", {}, "")
458
+ return send_response_and_close(RES_403_STATUS, {}, "")
389
459
  end
390
460
 
391
461
  # in_http does not support HTTP methods except POST
392
462
  if @access_control_request_method != 'POST'
393
- return send_response_and_close("403 Forbidden", {}, "")
463
+ return send_response_and_close(RES_403_STATUS, {}, "")
394
464
  end
395
465
 
396
466
  header = {
@@ -401,19 +471,19 @@ module Fluent::Plugin
401
471
  # Check the origin and send back a CORS response
402
472
  if @cors_allow_origins.include?('*')
403
473
  header["Access-Control-Allow-Origin"] = "*"
404
- send_response_and_close("200 OK", header, "")
474
+ send_response_and_close(RES_200_STATUS, header, "")
405
475
  elsif include_cors_allow_origin
406
476
  header["Access-Control-Allow-Origin"] = @origin
407
- send_response_and_close("200 OK", header, "")
477
+ send_response_and_close(RES_200_STATUS, header, "")
408
478
  else
409
- send_response_and_close("403 Forbidden", {}, "")
479
+ send_response_and_close(RES_403_STATUS, {}, "")
410
480
  end
411
481
  end
412
482
 
413
483
  def on_message_complete
414
484
  return if closing?
415
485
 
416
- if @parser.http_method == 'OPTIONS'
486
+ if @parser.http_method == 'OPTIONS'.freeze
417
487
  return handle_options_request()
418
488
  end
419
489
 
@@ -423,7 +493,7 @@ module Fluent::Plugin
423
493
  # restrictions and white listed origins through @cors_allow_origins.
424
494
  unless @cors_allow_origins.nil?
425
495
  unless @cors_allow_origins.include?('*') or include_cors_allow_origin
426
- send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
496
+ send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
427
497
  return
428
498
  end
429
499
  end
@@ -433,14 +503,14 @@ module Fluent::Plugin
433
503
  # Decode payload according to the "Content-Encoding" header.
434
504
  # For now, we only support 'gzip' and 'deflate'.
435
505
  begin
436
- if @content_encoding == 'gzip'
506
+ if @content_encoding == 'gzip'.freeze
437
507
  @body = Zlib::GzipReader.new(StringIO.new(@body)).read
438
- elsif @content_encoding == 'deflate'
508
+ elsif @content_encoding == 'deflate'.freeze
439
509
  @body = Zlib::Inflate.inflate(@body)
440
510
  end
441
511
  rescue
442
512
  @log.warn 'fails to decode payload', error: $!.to_s
443
- send_response_and_close("400 Bad Request", {}, "")
513
+ send_response_and_close(RES_400_STATUS, {}, "")
444
514
  return
445
515
  end
446
516
 
@@ -466,8 +536,9 @@ module Fluent::Plugin
466
536
  params.merge!(@env)
467
537
  @env.clear
468
538
 
469
- code, header, body = *@callback.call(path_info, params)
539
+ code, header, body = @callback.call(path_info, params)
470
540
  body = body.to_s
541
+ header = header.dup if header.frozen?
471
542
 
472
543
  unless @cors_allow_origins.nil?
473
544
  if @cors_allow_origins.include?('*')
@@ -478,7 +549,7 @@ module Fluent::Plugin
478
549
  end
479
550
 
480
551
  if @keep_alive
481
- header['Connection'] = 'Keep-Alive'
552
+ header['Connection'] = 'Keep-Alive'.freeze
482
553
  send_response(code, header, body)
483
554
  else
484
555
  send_response_and_close(code, header, body)
@@ -504,13 +575,13 @@ module Fluent::Plugin
504
575
 
505
576
  def send_response(code, header, body)
506
577
  header['Content-Length'] ||= body.bytesize
507
- header['Content-Type'] ||= 'text/plain'
578
+ header['Content-Type'] ||= 'text/plain'.freeze
508
579
 
509
580
  data = %[HTTP/1.1 #{code}\r\n]
510
581
  header.each_pair {|k,v|
511
582
  data << "#{k}: #{v}\r\n"
512
583
  }
513
- data << "\r\n"
584
+ data << "\r\n".freeze
514
585
  @io.write(data)
515
586
 
516
587
  @io.write(body)
@@ -521,7 +592,7 @@ module Fluent::Plugin
521
592
  header.each_pair {|k,v|
522
593
  data << "#{k}: #{v}\r\n"
523
594
  }
524
- data << "\r\n"
595
+ data << "\r\n".freeze
525
596
  @io.write(data)
526
597
  end
527
598