fluentd 1.10.3-x64-mingw32 → 1.11.3-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +102 -0
  3. data/CONTRIBUTING.md +1 -1
  4. data/example/copy_roundrobin.conf +3 -3
  5. data/example/counter.conf +1 -1
  6. data/example/filter_stdout.conf +2 -2
  7. data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
  8. data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
  9. data/example/logevents.conf +5 -5
  10. data/example/multi_filters.conf +1 -1
  11. data/example/out_exec_filter.conf +2 -2
  12. data/example/out_forward.conf +1 -1
  13. data/example/out_forward_buf_file.conf +1 -1
  14. data/example/out_forward_client.conf +5 -5
  15. data/example/out_forward_heartbeat_none.conf +1 -1
  16. data/example/out_forward_sd.conf +1 -1
  17. data/example/out_forward_shared_key.conf +2 -2
  18. data/example/out_forward_tls.conf +1 -1
  19. data/example/out_forward_users.conf +3 -3
  20. data/example/out_null.conf +4 -4
  21. data/example/secondary_file.conf +1 -1
  22. data/fluentd.gemspec +5 -5
  23. data/lib/fluent/command/fluentd.rb +11 -0
  24. data/lib/fluent/log.rb +33 -3
  25. data/lib/fluent/match.rb +9 -0
  26. data/lib/fluent/plugin/buffer.rb +49 -40
  27. data/lib/fluent/plugin/buffer/chunk.rb +2 -1
  28. data/lib/fluent/plugin/in_dummy.rb +2 -123
  29. data/lib/fluent/plugin/in_exec.rb +4 -2
  30. data/lib/fluent/plugin/in_forward.rb +2 -2
  31. data/lib/fluent/plugin/in_gc_stat.rb +16 -0
  32. data/lib/fluent/plugin/in_http.rb +148 -77
  33. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  34. data/lib/fluent/plugin/in_sample.rb +141 -0
  35. data/lib/fluent/plugin/in_tail.rb +2 -2
  36. data/lib/fluent/plugin/in_unix.rb +77 -77
  37. data/lib/fluent/plugin/out_file.rb +1 -1
  38. data/lib/fluent/plugin/out_forward.rb +1 -1
  39. data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
  40. data/lib/fluent/plugin/out_http.rb +15 -2
  41. data/lib/fluent/plugin/output.rb +1 -0
  42. data/lib/fluent/plugin/parser_json.rb +5 -2
  43. data/lib/fluent/plugin/parser_syslog.rb +215 -54
  44. data/lib/fluent/plugin_helper/cert_option.rb +5 -8
  45. data/lib/fluent/plugin_helper/child_process.rb +3 -2
  46. data/lib/fluent/plugin_helper/socket.rb +1 -1
  47. data/lib/fluent/supervisor.rb +15 -6
  48. data/lib/fluent/system_config.rb +2 -1
  49. data/lib/fluent/test/filter_test.rb +2 -2
  50. data/lib/fluent/test/output_test.rb +3 -3
  51. data/lib/fluent/version.rb +1 -1
  52. data/test/plugin/out_forward/test_load_balancer.rb +46 -0
  53. data/test/plugin/test_buffer.rb +4 -0
  54. data/test/plugin/test_in_exec.rb +18 -0
  55. data/test/plugin/test_in_gc_stat.rb +24 -1
  56. data/test/plugin/test_in_http.rb +57 -0
  57. data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
  58. data/test/plugin/test_in_tail.rb +3 -0
  59. data/test/plugin/test_in_unix.rb +128 -73
  60. data/test/plugin/test_out_http.rb +38 -0
  61. data/test/plugin/test_parser_syslog.rb +66 -29
  62. data/test/plugin_helper/data/cert/empty.pem +0 -0
  63. data/test/plugin_helper/test_cert_option.rb +7 -0
  64. data/test/plugin_helper/test_child_process.rb +15 -0
  65. data/test/plugin_helper/test_http_server_helper.rb +5 -0
  66. data/test/plugin_helper/test_server.rb +34 -0
  67. data/test/plugin_helper/test_socket.rb +8 -0
  68. data/test/test_log.rb +44 -0
  69. data/test/test_match.rb +11 -0
  70. data/test/test_static_config_analysis.rb +2 -2
  71. metadata +19 -16
@@ -33,6 +33,15 @@ module Fluent
33
33
 
34
34
  class GlobMatchPattern < MatchPattern
35
35
  def initialize(pat)
36
+ if pat.start_with?('/')
37
+ if pat.end_with?('/')
38
+ @regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
39
+ return
40
+ else
41
+ raise Fluent::ConfigError, "invalid match - regex"
42
+ end
43
+ end
44
+
36
45
  stack = []
37
46
  regex = ['']
38
47
  escape = false
@@ -143,17 +143,33 @@ module Fluent
143
143
  end
144
144
  end
145
145
 
146
+ # timekey should be unixtime as usual.
147
+ # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
148
+ # We should check object_id stability to use object_id as optimization for comparing operations.
149
+ # e.g.)
150
+ # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
151
+ # => 1596187800
152
+ # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
153
+ # => true
154
+ def self.enable_optimize?
155
+ a1 = 2**30 - 1
156
+ a2 = 2**30 - 1
157
+ b1 = 2**62 - 1
158
+ b2 = 2**62 - 1
159
+ (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
160
+ end
161
+
146
162
  # This is an optimization code. Current Struct's implementation is comparing all data.
147
163
  # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
148
164
  # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
149
165
  # This means that this class is one of the most called object in Fluentd.
150
166
  # See https://github.com/fluent/fluentd/pull/2560
151
- # But, this optimization has a side effect on Windows due to differing object_id.
167
+ # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
152
168
  # This difference causes flood of buffer files.
153
- # So, this optimization should be enabled on non-Windows platform.
169
+ # So, this optimization should be enabled on `enable_optimize?` as true platforms.
154
170
  def hash
155
171
  timekey.object_id
156
- end unless Fluent.windows?
172
+ end if enable_optimize?
157
173
  end
158
174
 
159
175
  # for tests
@@ -176,6 +192,7 @@ module Fluent
176
192
 
177
193
  @stage_size = @queue_size = 0
178
194
  @timekeys = Hash.new(0)
195
+ @enable_update_timekeys = false
179
196
  @mutex = Mutex.new
180
197
  end
181
198
 
@@ -191,24 +208,23 @@ module Fluent
191
208
  end
192
209
  end
193
210
 
211
+ def enable_update_timekeys
212
+ @enable_update_timekeys = true
213
+ end
214
+
194
215
  def start
195
216
  super
196
217
 
197
218
  @stage, @queue = resume
198
219
  @stage.each_pair do |metadata, chunk|
199
220
  @stage_size += chunk.bytesize
200
- if chunk.metadata && chunk.metadata.timekey
201
- add_timekey(metadata.timekey)
202
- end
203
221
  end
204
222
  @queue.each do |chunk|
205
223
  @queued_num[chunk.metadata] ||= 0
206
224
  @queued_num[chunk.metadata] += 1
207
225
  @queue_size += chunk.bytesize
208
- if chunk.metadata && chunk.metadata.timekey
209
- add_timekey(chunk.metadata.timekey)
210
- end
211
226
  end
227
+ update_timekeys
212
228
  log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
213
229
  end
214
230
 
@@ -257,12 +273,9 @@ module Fluent
257
273
  Metadata.new(timekey, tag, variables)
258
274
  end
259
275
 
276
+ # Keep this method for existing code
260
277
  def metadata(timekey: nil, tag: nil, variables: nil)
261
- meta = Metadata.new(timekey, tag, variables)
262
- if (t = meta.timekey)
263
- add_timekey(t)
264
- end
265
- meta
278
+ Metadata.new(timekey, tag, variables)
266
279
  end
267
280
 
268
281
  def timekeys
@@ -456,9 +469,23 @@ module Fluent
456
469
  end
457
470
  end
458
471
 
472
+ def update_timekeys
473
+ synchronize do
474
+ chunks = @stage.values
475
+ chunks.concat(@queue)
476
+ @timekeys = chunks.each_with_object({}) do |chunk, keys|
477
+ if chunk.metadata && chunk.metadata.timekey
478
+ t = chunk.metadata.timekey
479
+ keys[t] = keys.fetch(t, 0) + 1
480
+ end
481
+ end
482
+ end
483
+ end
484
+
459
485
  # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
460
486
  def enqueue_all(force_enqueue = false)
461
487
  log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
488
+ update_timekeys if @enable_update_timekeys
462
489
 
463
490
  if block_given?
464
491
  synchronize{ @stage.keys }.each do |metadata|
@@ -537,10 +564,6 @@ module Fluent
537
564
  log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
538
565
  end
539
566
 
540
- if metadata && metadata.timekey
541
- del_timekey(metadata.timekey)
542
- end
543
-
544
567
  nil
545
568
  end
546
569
 
@@ -604,7 +627,11 @@ module Fluent
604
627
 
605
628
  if chunk_size_over?(chunk)
606
629
  if format && empty_chunk
607
- log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
630
+ if chunk.bytesize > @chunk_limit_size
631
+ log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
632
+ else
633
+ log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
634
+ end
608
635
  end
609
636
  chunk.rollback
610
637
 
@@ -768,11 +795,11 @@ module Fluent
768
795
  'total_queued_size' => stage_size + queue_size,
769
796
  }
770
797
 
771
- if (m = timekeys.min)
798
+ tkeys = timekeys
799
+ if (m = tkeys.min)
772
800
  stats['oldest_timekey'] = m
773
801
  end
774
-
775
- if (m = timekeys.max)
802
+ if (m = tkeys.max)
776
803
  stats['newest_timekey'] = m
777
804
  end
778
805
 
@@ -789,24 +816,6 @@ module Fluent
789
816
  !@queue.empty?
790
817
  end
791
818
  end
792
-
793
- def add_timekey(t)
794
- @mutex.synchronize do
795
- @timekeys[t] += 1
796
- end
797
- nil
798
- end
799
-
800
- def del_timekey(t)
801
- @mutex.synchronize do
802
- if @timekeys[t] <= 1
803
- @timekeys.delete(t)
804
- else
805
- @timekeys[t] -= 1
806
- end
807
- end
808
- nil
809
- end
810
819
  end
811
820
  end
812
821
  end
@@ -206,8 +206,9 @@ module Fluent
206
206
  output_io = if chunk_io.is_a?(StringIO)
207
207
  StringIO.new
208
208
  else
209
- Tempfile.new('decompressed-data').binmode
209
+ Tempfile.new('decompressed-data')
210
210
  end
211
+ output_io.binmode if output_io.is_a?(Tempfile)
211
212
  decompress(input_io: chunk_io, output_io: output_io)
212
213
  output_io.seek(0, IO::SEEK_SET)
213
214
  yield output_io
@@ -14,126 +14,5 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'json'
18
-
19
- require 'fluent/plugin/input'
20
- require 'fluent/config/error'
21
-
22
- module Fluent::Plugin
23
- class DummyInput < Input
24
- Fluent::Plugin.register_input('dummy', self)
25
-
26
- helpers :thread, :storage
27
-
28
- BIN_NUM = 10
29
- DEFAULT_STORAGE_TYPE = 'local'
30
-
31
- desc "The value is the tag assigned to the generated events."
32
- config_param :tag, :string
33
- desc "The number of events in event stream of each emits."
34
- config_param :size, :integer, default: 1
35
- desc "It configures how many events to generate per second."
36
- config_param :rate, :integer, default: 1
37
- desc "If specified, each generated event has an auto-incremented key field."
38
- config_param :auto_increment_key, :string, default: nil
39
- desc "The boolean to suspend-and-resume incremental value after restart"
40
- config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
41
- desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
42
- config_param :dummy, default: [{"message"=>"dummy"}] do |val|
43
- begin
44
- parsed = JSON.parse(val)
45
- rescue JSON::ParserError => ex
46
- # Fluent::ConfigParseError, "got incomplete JSON" will be raised
47
- # at literal_parser.rb with --use-v1-config, but I had to
48
- # take care at here for the case of --use-v0-config.
49
- raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
50
- end
51
- dummy = parsed.is_a?(Array) ? parsed : [parsed]
52
- dummy.each_with_index do |e, i|
53
- raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
54
- end
55
- dummy
56
- end
57
-
58
- def initialize
59
- super
60
- @storage = nil
61
- end
62
-
63
- def configure(conf)
64
- super
65
- @dummy_index = 0
66
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
- @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
68
- end
69
-
70
- def multi_workers_ready?
71
- true
72
- end
73
-
74
- def start
75
- super
76
-
77
- @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
78
- @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
79
-
80
- if @auto_increment_key && !@storage.get(:auto_increment_value)
81
- @storage.put(:auto_increment_value, -1)
82
- end
83
-
84
- thread_create(:dummy_input, &method(:run))
85
- end
86
-
87
- def run
88
- batch_num = (@rate / BIN_NUM).to_i
89
- residual_num = (@rate % BIN_NUM)
90
- while thread_current_running?
91
- current_time = Time.now.to_i
92
- BIN_NUM.times do
93
- break unless (thread_current_running? && Time.now.to_i <= current_time)
94
- wait(0.1) { emit(batch_num) }
95
- end
96
- emit(residual_num) if thread_current_running?
97
- # wait for next second
98
- while thread_current_running? && Time.now.to_i <= current_time
99
- sleep 0.01
100
- end
101
- end
102
- end
103
-
104
- def emit(num)
105
- begin
106
- if @size > 1
107
- num.times do
108
- router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
109
- end
110
- else
111
- num.times { router.emit(@tag, Fluent::Engine.now, generate) }
112
- end
113
- rescue => _
114
- # ignore all errors not to stop emits by emit errors
115
- end
116
- end
117
-
118
- def generate
119
- d = @dummy[@dummy_index]
120
- unless d
121
- @dummy_index = 0
122
- d = @dummy[@dummy_index]
123
- end
124
- @dummy_index += 1
125
- if @auto_increment_key
126
- d = d.dup
127
- d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
128
- end
129
- d
130
- end
131
-
132
- def wait(time)
133
- start_time = Time.now
134
- yield
135
- sleep_time = time - (Time.now - start_time)
136
- sleep sleep_time if sleep_time > 0
137
- end
138
- end
139
- end
17
+ # Remove this file in fluentd v2
18
+ require_relative 'in_sample'
@@ -25,6 +25,8 @@ module Fluent::Plugin
25
25
 
26
26
  desc 'The command (program) to execute.'
27
27
  config_param :command, :string
28
+ desc 'Specify connect mode to executed process'
29
+ config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
28
30
 
29
31
  config_section :parse do
30
32
  config_set_default :@type, 'tsv'
@@ -72,9 +74,9 @@ module Fluent::Plugin
72
74
  super
73
75
 
74
76
  if @run_interval
75
- child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
77
+ child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
76
78
  else
77
- child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
79
+ child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
78
80
  end
79
81
  end
80
82
 
@@ -327,7 +327,7 @@ module Fluent::Plugin
327
327
  record = e[1]
328
328
  next if record.nil?
329
329
  time = e[0]
330
- time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
330
+ time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
331
331
  es.add(time, record)
332
332
  }
333
333
  es
@@ -347,7 +347,7 @@ module Fluent::Plugin
347
347
  return msg[3] # retry never succeeded so return ack and drop incoming event.
348
348
  end
349
349
  return if record.nil?
350
- time = Fluent::Engine.now if time.to_i == 0
350
+ time = Fluent::EventTime.now if time.to_i == 0
351
351
  if @enable_field_injection
352
352
  record[@source_address_key] = conn.remote_addr if @source_address_key
353
353
  record[@source_hostname_key] = conn.remote_host if @source_hostname_key
@@ -24,13 +24,22 @@ module Fluent::Plugin
24
24
 
25
25
  def initialize
26
26
  super
27
+ @key_map = nil
27
28
  end
28
29
 
29
30
  config_param :emit_interval, :time, default: 60
31
+ config_param :use_symbol_keys, :bool, default: true
30
32
  config_param :tag, :string
31
33
 
32
34
  def configure(conf)
33
35
  super
36
+
37
+ unless @use_symbol_keys
38
+ @key_map = {}
39
+ GC.stat.each_key { |key|
40
+ @key_map[key] = key.to_s
41
+ }
42
+ end
34
43
  end
35
44
 
36
45
  def multi_workers_ready?
@@ -50,6 +59,13 @@ module Fluent::Plugin
50
59
  def on_timer
51
60
  now = Fluent::EventTime.now
52
61
  record = GC.stat
62
+ unless @use_symbol_keys
63
+ new_record = {}
64
+ record.each_pair { |k, v|
65
+ new_record[@key_map[k]] = v
66
+ }
67
+ record = new_record
68
+ end
53
69
  router.emit(@tag, now, record)
54
70
  end
55
71
  end
@@ -27,10 +27,28 @@ require 'json'
27
27
  module Fluent::Plugin
28
28
  class InHttpParser < Parser
29
29
  Fluent::Plugin.register_parser('in_http', self)
30
+
31
+ config_set_default :time_key, 'time'
32
+
33
+ def configure(conf)
34
+ super
35
+
36
+ # if no time parser related parameters, use in_http's time convert rule
37
+ @time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')
38
+ time_parser_create
39
+ else
40
+ nil
41
+ end
42
+ end
43
+
30
44
  def parse(text)
31
45
  # this plugin is dummy implementation not to raise error
32
46
  yield nil, nil
33
47
  end
48
+
49
+ def get_time_parser
50
+ @time_parser
51
+ end
34
52
  end
35
53
 
36
54
  class HttpInput < Input
@@ -54,12 +72,14 @@ module Fluent::Plugin
54
72
  desc 'Add REMOTE_ADDR header to the record.'
55
73
  config_param :add_remote_addr, :bool, default: false
56
74
  config_param :blocking_timeout, :time, default: 0.5
57
- desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
75
+ desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
58
76
  config_param :cors_allow_origins, :array, default: nil
59
77
  desc 'Respond with empty gif image of 1x1 pixel.'
60
78
  config_param :respond_with_empty_img, :bool, default: false
61
79
  desc 'Respond status code with 204.'
62
80
  config_param :use_204_response, :bool, default: false
81
+ desc 'Dump error log or not'
82
+ config_param :dump_error_log, :bool, default: true
63
83
 
64
84
  config_section :parse do
65
85
  config_set_default :@type, 'in_http'
@@ -67,6 +87,24 @@ module Fluent::Plugin
67
87
 
68
88
  EVENT_RECORD_PARAMETER = '_event_record'
69
89
 
90
+ def initialize
91
+ super
92
+
93
+ @km = nil
94
+ @format_name = nil
95
+ @parser_time_key = nil
96
+
97
+ # default parsers
98
+ @parser_msgpack = nil
99
+ @parser_json = nil
100
+ @default_time_parser = nil
101
+ @default_keep_time_key = nil
102
+ @float_time_parser = nil
103
+
104
+ # <parse> configured parser
105
+ @custom_parser = nil
106
+ end
107
+
70
108
  def configure(conf)
71
109
  compat_parameters_convert(conf, :parser)
72
110
 
@@ -74,20 +112,22 @@ module Fluent::Plugin
74
112
 
75
113
  m = if @parser_configs.first['@type'] == 'in_http'
76
114
  @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
115
+ @parser_msgpack.time_key = nil
77
116
  @parser_msgpack.estimate_current_event = false
78
117
  @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
118
+ @parser_json.time_key = nil
79
119
  @parser_json.estimate_current_event = false
120
+
121
+ default_parser = parser_create(usage: '')
80
122
  @format_name = 'default'
81
- @parser_time_key = if parser_config = conf.elements('parse').first
82
- parser_config['time_key'] || 'time'
83
- else
84
- 'time'
85
- end
123
+ @parser_time_key = default_parser.time_key
124
+ @default_time_parser = default_parser.get_time_parser
125
+ @default_keep_time_key = default_parser.keep_time_key
86
126
  method(:parse_params_default)
87
127
  else
88
- @parser = parser_create
128
+ @custom_parser = parser_create
89
129
  @format_name = @parser_configs.first['@type']
90
- @parser_time_key = @parser.time_key
130
+ @parser_time_key = @custom_parser.time_key
91
131
  method(:parse_params_with_parser)
92
132
  end
93
133
  self.singleton_class.module_eval do
@@ -142,6 +182,13 @@ module Fluent::Plugin
142
182
  super
143
183
  end
144
184
 
185
+ RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze
186
+ RESPONSE_200 = ["200 OK".freeze, RES_TEXT_HEADER, "".freeze].freeze
187
+ RESPONSE_204 = ["204 No Content".freeze, {}.freeze].freeze
188
+ RESPONSE_IMG = ["200 OK".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze
189
+ RES_400_STATUS = "400 Bad Request".freeze
190
+ RES_500_STATUS = "500 Internal Server Error".freeze
191
+
145
192
  def on_request(path_info, params)
146
193
  begin
147
194
  path = path_info[1..-1] # remove /
@@ -152,83 +199,77 @@ module Fluent::Plugin
152
199
  if record.nil?
153
200
  log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
154
201
  if @respond_with_empty_img
155
- return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
202
+ return RESPONSE_IMG
156
203
  else
157
204
  if @use_204_response
158
- return ["204 No Content", {}]
205
+ return RESPONSE_204
159
206
  else
160
- return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
207
+ return RESPONSE_200
161
208
  end
162
209
  end
163
210
  end
164
211
 
165
- unless record.is_a?(Array)
166
- if @add_http_headers
167
- params.each_pair { |k,v|
168
- if k.start_with?("HTTP_")
169
- record[k] = v
170
- end
171
- }
172
- end
173
- if @add_remote_addr
174
- record['REMOTE_ADDR'] = params['REMOTE_ADDR']
175
- end
176
- end
177
- time = if param_time = params['time']
178
- param_time = param_time.to_f
179
- param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
180
- else
181
- record_time.nil? ? Fluent::Engine.now : record_time
182
- end
183
- rescue
184
- return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
185
- end
186
-
187
- # TODO server error
188
- begin
212
+ mes = nil
189
213
  # Support batched requests
190
214
  if record.is_a?(Array)
191
215
  mes = Fluent::MultiEventStream.new
192
216
  record.each do |single_record|
193
- if @add_http_headers
194
- params.each_pair { |k,v|
195
- if k.start_with?("HTTP_")
196
- single_record[k] = v
197
- end
198
- }
199
- end
200
- if @add_remote_addr
201
- single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
202
- end
203
-
204
- if defined? @parser
205
- single_time = @parser.parse_time(single_record)
206
- single_time, single_record = @parser.convert_values(single_time, single_record)
217
+ add_params_to_record(single_record, params)
218
+
219
+ if param_time = params['time']
220
+ param_time = param_time.to_f
221
+ single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
222
+ elsif @custom_parser
223
+ single_time = @custom_parser.parse_time(single_record)
224
+ single_time, single_record = @custom_parser.convert_values(single_time, single_record)
207
225
  else
208
- single_time = if t = single_record.delete(@parser_time_key)
209
- Fluent::EventTime.from_time(Time.at(t))
210
- else
211
- time
212
- end
226
+ single_time = convert_time_field(single_record)
213
227
  end
214
228
 
215
229
  mes.add(single_time, single_record)
216
230
  end
231
+ else
232
+ add_params_to_record(record, params)
233
+
234
+ time = if param_time = params['time']
235
+ param_time = param_time.to_f
236
+ param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
237
+ else
238
+ if record_time.nil?
239
+ convert_time_field(record)
240
+ else
241
+ record_time
242
+ end
243
+ end
244
+ end
245
+ rescue => e
246
+ if @dump_error_log
247
+ log.error "failed to process request", error: e
248
+ end
249
+ return [RES_400_STATUS, RES_TEXT_HEADER, "400 Bad Request\n#{e}\n"]
250
+ end
251
+
252
+ # TODO server error
253
+ begin
254
+ if mes
217
255
  router.emit_stream(tag, mes)
218
256
  else
219
257
  router.emit(tag, time, record)
220
258
  end
221
- rescue
222
- return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
259
+ rescue => e
260
+ if @dump_error_log
261
+ log.error "failed to emit data", error: e
262
+ end
263
+ return [RES_500_STATUS, RES_TEXT_HEADER, "500 Internal Server Error\n#{e}\n"]
223
264
  end
224
265
 
225
266
  if @respond_with_empty_img
226
- return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
267
+ return RESPONSE_IMG
227
268
  else
228
269
  if @use_204_response
229
- return ["204 No Content", {}]
270
+ return RESPONSE_204
230
271
  else
231
- return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
272
+ return RESPONSE_200
232
273
  end
233
274
  end
234
275
  end
@@ -267,7 +308,7 @@ module Fluent::Plugin
267
308
 
268
309
  def parse_params_with_parser(params)
269
310
  if content = params[EVENT_RECORD_PARAMETER]
270
- @parser.parse(content) { |time, record|
311
+ @custom_parser.parse(content) { |time, record|
271
312
  raise "Received event is not #{@format_name}: #{content}" if record.nil?
272
313
  return time, record
273
314
  }
@@ -276,6 +317,32 @@ module Fluent::Plugin
276
317
  end
277
318
  end
278
319
 
320
+ def add_params_to_record(record, params)
321
+ if @add_http_headers
322
+ params.each_pair { |k, v|
323
+ if k.start_with?("HTTP_".freeze)
324
+ record[k] = v
325
+ end
326
+ }
327
+ end
328
+
329
+ if @add_remote_addr
330
+ record['REMOTE_ADDR'] = params['REMOTE_ADDR']
331
+ end
332
+ end
333
+
334
+ def convert_time_field(record)
335
+ if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)
336
+ if @default_time_parser
337
+ @default_time_parser.parse(t)
338
+ else
339
+ Fluent::EventTime.from_time(Time.at(t))
340
+ end
341
+ else
342
+ Fluent::EventTime.now
343
+ end
344
+ end
345
+
279
346
  class Handler
280
347
  attr_reader :content_type
281
348
 
@@ -358,7 +425,7 @@ module Fluent::Plugin
358
425
  end
359
426
  }
360
427
  if expect
361
- if expect == '100-continue'
428
+ if expect == '100-continue'.freeze
362
429
  if !size || size < @body_size_limit
363
430
  send_response_nobody("100 Continue", {})
364
431
  else
@@ -380,17 +447,20 @@ module Fluent::Plugin
380
447
  @body << chunk
381
448
  end
382
449
 
450
+ RES_200_STATUS = "200 OK".freeze
451
+ RES_403_STATUS = "403 Forbidden".freeze
452
+
383
453
  # Web browsers can send an OPTIONS request before performing POST
384
454
  # to check if cross-origin requests are supported.
385
455
  def handle_options_request
386
456
  # Is CORS enabled in the first place?
387
457
  if @cors_allow_origins.nil?
388
- return send_response_and_close("403 Forbidden", {}, "")
458
+ return send_response_and_close(RES_403_STATUS, {}, "")
389
459
  end
390
460
 
391
461
  # in_http does not support HTTP methods except POST
392
462
  if @access_control_request_method != 'POST'
393
- return send_response_and_close("403 Forbidden", {}, "")
463
+ return send_response_and_close(RES_403_STATUS, {}, "")
394
464
  end
395
465
 
396
466
  header = {
@@ -401,29 +471,29 @@ module Fluent::Plugin
401
471
  # Check the origin and send back a CORS response
402
472
  if @cors_allow_origins.include?('*')
403
473
  header["Access-Control-Allow-Origin"] = "*"
404
- send_response_and_close("200 OK", header, "")
474
+ send_response_and_close(RES_200_STATUS, header, "")
405
475
  elsif include_cors_allow_origin
406
476
  header["Access-Control-Allow-Origin"] = @origin
407
- send_response_and_close("200 OK", header, "")
477
+ send_response_and_close(RES_200_STATUS, header, "")
408
478
  else
409
- send_response_and_close("403 Forbidden", {}, "")
479
+ send_response_and_close(RES_403_STATUS, {}, "")
410
480
  end
411
481
  end
412
482
 
413
483
  def on_message_complete
414
484
  return if closing?
415
485
 
416
- if @parser.http_method == 'OPTIONS'
486
+ if @parser.http_method == 'OPTIONS'.freeze
417
487
  return handle_options_request()
418
488
  end
419
489
 
420
490
  # CORS check
421
491
  # ==========
422
492
  # For every incoming request, we check if we have some CORS
423
- # restrictions and white listed origins through @cors_allow_origins.
493
+ # restrictions and allow listed origins through @cors_allow_origins.
424
494
  unless @cors_allow_origins.nil?
425
495
  unless @cors_allow_origins.include?('*') or include_cors_allow_origin
426
- send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
496
+ send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
427
497
  return
428
498
  end
429
499
  end
@@ -433,14 +503,14 @@ module Fluent::Plugin
433
503
  # Decode payload according to the "Content-Encoding" header.
434
504
  # For now, we only support 'gzip' and 'deflate'.
435
505
  begin
436
- if @content_encoding == 'gzip'
506
+ if @content_encoding == 'gzip'.freeze
437
507
  @body = Zlib::GzipReader.new(StringIO.new(@body)).read
438
- elsif @content_encoding == 'deflate'
508
+ elsif @content_encoding == 'deflate'.freeze
439
509
  @body = Zlib::Inflate.inflate(@body)
440
510
  end
441
511
  rescue
442
512
  @log.warn 'fails to decode payload', error: $!.to_s
443
- send_response_and_close("400 Bad Request", {}, "")
513
+ send_response_and_close(RES_400_STATUS, {}, "")
444
514
  return
445
515
  end
446
516
 
@@ -466,8 +536,9 @@ module Fluent::Plugin
466
536
  params.merge!(@env)
467
537
  @env.clear
468
538
 
469
- code, header, body = *@callback.call(path_info, params)
539
+ code, header, body = @callback.call(path_info, params)
470
540
  body = body.to_s
541
+ header = header.dup if header.frozen?
471
542
 
472
543
  unless @cors_allow_origins.nil?
473
544
  if @cors_allow_origins.include?('*')
@@ -478,7 +549,7 @@ module Fluent::Plugin
478
549
  end
479
550
 
480
551
  if @keep_alive
481
- header['Connection'] = 'Keep-Alive'
552
+ header['Connection'] = 'Keep-Alive'.freeze
482
553
  send_response(code, header, body)
483
554
  else
484
555
  send_response_and_close(code, header, body)
@@ -504,13 +575,13 @@ module Fluent::Plugin
504
575
 
505
576
  def send_response(code, header, body)
506
577
  header['Content-Length'] ||= body.bytesize
507
- header['Content-Type'] ||= 'text/plain'
578
+ header['Content-Type'] ||= 'text/plain'.freeze
508
579
 
509
580
  data = %[HTTP/1.1 #{code}\r\n]
510
581
  header.each_pair {|k,v|
511
582
  data << "#{k}: #{v}\r\n"
512
583
  }
513
- data << "\r\n"
584
+ data << "\r\n".freeze
514
585
  @io.write(data)
515
586
 
516
587
  @io.write(body)
@@ -521,7 +592,7 @@ module Fluent::Plugin
521
592
  header.each_pair {|k,v|
522
593
  data << "#{k}: #{v}\r\n"
523
594
  }
524
- data << "\r\n"
595
+ data << "\r\n".freeze
525
596
  @io.write(data)
526
597
  end
527
598