fluentd 1.10.3-x64-mingw32 → 1.11.3-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +102 -0
- data/CONTRIBUTING.md +1 -1
- data/example/copy_roundrobin.conf +3 -3
- data/example/counter.conf +1 -1
- data/example/filter_stdout.conf +2 -2
- data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
- data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
- data/example/logevents.conf +5 -5
- data/example/multi_filters.conf +1 -1
- data/example/out_exec_filter.conf +2 -2
- data/example/out_forward.conf +1 -1
- data/example/out_forward_buf_file.conf +1 -1
- data/example/out_forward_client.conf +5 -5
- data/example/out_forward_heartbeat_none.conf +1 -1
- data/example/out_forward_sd.conf +1 -1
- data/example/out_forward_shared_key.conf +2 -2
- data/example/out_forward_tls.conf +1 -1
- data/example/out_forward_users.conf +3 -3
- data/example/out_null.conf +4 -4
- data/example/secondary_file.conf +1 -1
- data/fluentd.gemspec +5 -5
- data/lib/fluent/command/fluentd.rb +11 -0
- data/lib/fluent/log.rb +33 -3
- data/lib/fluent/match.rb +9 -0
- data/lib/fluent/plugin/buffer.rb +49 -40
- data/lib/fluent/plugin/buffer/chunk.rb +2 -1
- data/lib/fluent/plugin/in_dummy.rb +2 -123
- data/lib/fluent/plugin/in_exec.rb +4 -2
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_gc_stat.rb +16 -0
- data/lib/fluent/plugin/in_http.rb +148 -77
- data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_tail.rb +2 -2
- data/lib/fluent/plugin/in_unix.rb +77 -77
- data/lib/fluent/plugin/out_file.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +1 -1
- data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
- data/lib/fluent/plugin/out_http.rb +15 -2
- data/lib/fluent/plugin/output.rb +1 -0
- data/lib/fluent/plugin/parser_json.rb +5 -2
- data/lib/fluent/plugin/parser_syslog.rb +215 -54
- data/lib/fluent/plugin_helper/cert_option.rb +5 -8
- data/lib/fluent/plugin_helper/child_process.rb +3 -2
- data/lib/fluent/plugin_helper/socket.rb +1 -1
- data/lib/fluent/supervisor.rb +15 -6
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/test/filter_test.rb +2 -2
- data/lib/fluent/test/output_test.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/out_forward/test_load_balancer.rb +46 -0
- data/test/plugin/test_buffer.rb +4 -0
- data/test/plugin/test_in_exec.rb +18 -0
- data/test/plugin/test_in_gc_stat.rb +24 -1
- data/test/plugin/test_in_http.rb +57 -0
- data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
- data/test/plugin/test_in_tail.rb +3 -0
- data/test/plugin/test_in_unix.rb +128 -73
- data/test/plugin/test_out_http.rb +38 -0
- data/test/plugin/test_parser_syslog.rb +66 -29
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/test_cert_option.rb +7 -0
- data/test/plugin_helper/test_child_process.rb +15 -0
- data/test/plugin_helper/test_http_server_helper.rb +5 -0
- data/test/plugin_helper/test_server.rb +34 -0
- data/test/plugin_helper/test_socket.rb +8 -0
- data/test/test_log.rb +44 -0
- data/test/test_match.rb +11 -0
- data/test/test_static_config_analysis.rb +2 -2
- metadata +19 -16
data/lib/fluent/match.rb
CHANGED
@@ -33,6 +33,15 @@ module Fluent
|
|
33
33
|
|
34
34
|
class GlobMatchPattern < MatchPattern
|
35
35
|
def initialize(pat)
|
36
|
+
if pat.start_with?('/')
|
37
|
+
if pat.end_with?('/')
|
38
|
+
@regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
|
39
|
+
return
|
40
|
+
else
|
41
|
+
raise Fluent::ConfigError, "invalid match - regex"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
36
45
|
stack = []
|
37
46
|
regex = ['']
|
38
47
|
escape = false
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -143,17 +143,33 @@ module Fluent
|
|
143
143
|
end
|
144
144
|
end
|
145
145
|
|
146
|
+
# timekey should be unixtime as usual.
|
147
|
+
# So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
|
148
|
+
# We should check object_id stability to use object_id as optimization for comparing operations.
|
149
|
+
# e.g.)
|
150
|
+
# irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
|
151
|
+
# => 1596187800
|
152
|
+
# irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
|
153
|
+
# => true
|
154
|
+
def self.enable_optimize?
|
155
|
+
a1 = 2**30 - 1
|
156
|
+
a2 = 2**30 - 1
|
157
|
+
b1 = 2**62 - 1
|
158
|
+
b2 = 2**62 - 1
|
159
|
+
(a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
|
160
|
+
end
|
161
|
+
|
146
162
|
# This is an optimization code. Current Struct's implementation is comparing all data.
|
147
163
|
# https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
|
148
164
|
# Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
|
149
165
|
# This means that this class is one of the most called object in Fluentd.
|
150
166
|
# See https://github.com/fluent/fluentd/pull/2560
|
151
|
-
# But, this optimization has a side effect on Windows due to differing object_id.
|
167
|
+
# But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
|
152
168
|
# This difference causes flood of buffer files.
|
153
|
-
# So, this optimization should be enabled on
|
169
|
+
# So, this optimization should be enabled on `enable_optimize?` as true platforms.
|
154
170
|
def hash
|
155
171
|
timekey.object_id
|
156
|
-
end
|
172
|
+
end if enable_optimize?
|
157
173
|
end
|
158
174
|
|
159
175
|
# for tests
|
@@ -176,6 +192,7 @@ module Fluent
|
|
176
192
|
|
177
193
|
@stage_size = @queue_size = 0
|
178
194
|
@timekeys = Hash.new(0)
|
195
|
+
@enable_update_timekeys = false
|
179
196
|
@mutex = Mutex.new
|
180
197
|
end
|
181
198
|
|
@@ -191,24 +208,23 @@ module Fluent
|
|
191
208
|
end
|
192
209
|
end
|
193
210
|
|
211
|
+
def enable_update_timekeys
|
212
|
+
@enable_update_timekeys = true
|
213
|
+
end
|
214
|
+
|
194
215
|
def start
|
195
216
|
super
|
196
217
|
|
197
218
|
@stage, @queue = resume
|
198
219
|
@stage.each_pair do |metadata, chunk|
|
199
220
|
@stage_size += chunk.bytesize
|
200
|
-
if chunk.metadata && chunk.metadata.timekey
|
201
|
-
add_timekey(metadata.timekey)
|
202
|
-
end
|
203
221
|
end
|
204
222
|
@queue.each do |chunk|
|
205
223
|
@queued_num[chunk.metadata] ||= 0
|
206
224
|
@queued_num[chunk.metadata] += 1
|
207
225
|
@queue_size += chunk.bytesize
|
208
|
-
if chunk.metadata && chunk.metadata.timekey
|
209
|
-
add_timekey(chunk.metadata.timekey)
|
210
|
-
end
|
211
226
|
end
|
227
|
+
update_timekeys
|
212
228
|
log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
|
213
229
|
end
|
214
230
|
|
@@ -257,12 +273,9 @@ module Fluent
|
|
257
273
|
Metadata.new(timekey, tag, variables)
|
258
274
|
end
|
259
275
|
|
276
|
+
# Keep this method for existing code
|
260
277
|
def metadata(timekey: nil, tag: nil, variables: nil)
|
261
|
-
|
262
|
-
if (t = meta.timekey)
|
263
|
-
add_timekey(t)
|
264
|
-
end
|
265
|
-
meta
|
278
|
+
Metadata.new(timekey, tag, variables)
|
266
279
|
end
|
267
280
|
|
268
281
|
def timekeys
|
@@ -456,9 +469,23 @@ module Fluent
|
|
456
469
|
end
|
457
470
|
end
|
458
471
|
|
472
|
+
def update_timekeys
|
473
|
+
synchronize do
|
474
|
+
chunks = @stage.values
|
475
|
+
chunks.concat(@queue)
|
476
|
+
@timekeys = chunks.each_with_object({}) do |chunk, keys|
|
477
|
+
if chunk.metadata && chunk.metadata.timekey
|
478
|
+
t = chunk.metadata.timekey
|
479
|
+
keys[t] = keys.fetch(t, 0) + 1
|
480
|
+
end
|
481
|
+
end
|
482
|
+
end
|
483
|
+
end
|
484
|
+
|
459
485
|
# At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
|
460
486
|
def enqueue_all(force_enqueue = false)
|
461
487
|
log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
|
488
|
+
update_timekeys if @enable_update_timekeys
|
462
489
|
|
463
490
|
if block_given?
|
464
491
|
synchronize{ @stage.keys }.each do |metadata|
|
@@ -537,10 +564,6 @@ module Fluent
|
|
537
564
|
log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
|
538
565
|
end
|
539
566
|
|
540
|
-
if metadata && metadata.timekey
|
541
|
-
del_timekey(metadata.timekey)
|
542
|
-
end
|
543
|
-
|
544
567
|
nil
|
545
568
|
end
|
546
569
|
|
@@ -604,7 +627,11 @@ module Fluent
|
|
604
627
|
|
605
628
|
if chunk_size_over?(chunk)
|
606
629
|
if format && empty_chunk
|
607
|
-
|
630
|
+
if chunk.bytesize > @chunk_limit_size
|
631
|
+
log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
|
632
|
+
else
|
633
|
+
log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
|
634
|
+
end
|
608
635
|
end
|
609
636
|
chunk.rollback
|
610
637
|
|
@@ -768,11 +795,11 @@ module Fluent
|
|
768
795
|
'total_queued_size' => stage_size + queue_size,
|
769
796
|
}
|
770
797
|
|
771
|
-
|
798
|
+
tkeys = timekeys
|
799
|
+
if (m = tkeys.min)
|
772
800
|
stats['oldest_timekey'] = m
|
773
801
|
end
|
774
|
-
|
775
|
-
if (m = timekeys.max)
|
802
|
+
if (m = tkeys.max)
|
776
803
|
stats['newest_timekey'] = m
|
777
804
|
end
|
778
805
|
|
@@ -789,24 +816,6 @@ module Fluent
|
|
789
816
|
!@queue.empty?
|
790
817
|
end
|
791
818
|
end
|
792
|
-
|
793
|
-
def add_timekey(t)
|
794
|
-
@mutex.synchronize do
|
795
|
-
@timekeys[t] += 1
|
796
|
-
end
|
797
|
-
nil
|
798
|
-
end
|
799
|
-
|
800
|
-
def del_timekey(t)
|
801
|
-
@mutex.synchronize do
|
802
|
-
if @timekeys[t] <= 1
|
803
|
-
@timekeys.delete(t)
|
804
|
-
else
|
805
|
-
@timekeys[t] -= 1
|
806
|
-
end
|
807
|
-
end
|
808
|
-
nil
|
809
|
-
end
|
810
819
|
end
|
811
820
|
end
|
812
821
|
end
|
@@ -206,8 +206,9 @@ module Fluent
|
|
206
206
|
output_io = if chunk_io.is_a?(StringIO)
|
207
207
|
StringIO.new
|
208
208
|
else
|
209
|
-
Tempfile.new('decompressed-data')
|
209
|
+
Tempfile.new('decompressed-data')
|
210
210
|
end
|
211
|
+
output_io.binmode if output_io.is_a?(Tempfile)
|
211
212
|
decompress(input_io: chunk_io, output_io: output_io)
|
212
213
|
output_io.seek(0, IO::SEEK_SET)
|
213
214
|
yield output_io
|
@@ -14,126 +14,5 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
require 'fluent/plugin/input'
|
20
|
-
require 'fluent/config/error'
|
21
|
-
|
22
|
-
module Fluent::Plugin
|
23
|
-
class DummyInput < Input
|
24
|
-
Fluent::Plugin.register_input('dummy', self)
|
25
|
-
|
26
|
-
helpers :thread, :storage
|
27
|
-
|
28
|
-
BIN_NUM = 10
|
29
|
-
DEFAULT_STORAGE_TYPE = 'local'
|
30
|
-
|
31
|
-
desc "The value is the tag assigned to the generated events."
|
32
|
-
config_param :tag, :string
|
33
|
-
desc "The number of events in event stream of each emits."
|
34
|
-
config_param :size, :integer, default: 1
|
35
|
-
desc "It configures how many events to generate per second."
|
36
|
-
config_param :rate, :integer, default: 1
|
37
|
-
desc "If specified, each generated event has an auto-incremented key field."
|
38
|
-
config_param :auto_increment_key, :string, default: nil
|
39
|
-
desc "The boolean to suspend-and-resume incremental value after restart"
|
40
|
-
config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
|
41
|
-
desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
|
42
|
-
config_param :dummy, default: [{"message"=>"dummy"}] do |val|
|
43
|
-
begin
|
44
|
-
parsed = JSON.parse(val)
|
45
|
-
rescue JSON::ParserError => ex
|
46
|
-
# Fluent::ConfigParseError, "got incomplete JSON" will be raised
|
47
|
-
# at literal_parser.rb with --use-v1-config, but I had to
|
48
|
-
# take care at here for the case of --use-v0-config.
|
49
|
-
raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
|
50
|
-
end
|
51
|
-
dummy = parsed.is_a?(Array) ? parsed : [parsed]
|
52
|
-
dummy.each_with_index do |e, i|
|
53
|
-
raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
|
54
|
-
end
|
55
|
-
dummy
|
56
|
-
end
|
57
|
-
|
58
|
-
def initialize
|
59
|
-
super
|
60
|
-
@storage = nil
|
61
|
-
end
|
62
|
-
|
63
|
-
def configure(conf)
|
64
|
-
super
|
65
|
-
@dummy_index = 0
|
66
|
-
config = conf.elements.select{|e| e.name == 'storage' }.first
|
67
|
-
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
68
|
-
end
|
69
|
-
|
70
|
-
def multi_workers_ready?
|
71
|
-
true
|
72
|
-
end
|
73
|
-
|
74
|
-
def start
|
75
|
-
super
|
76
|
-
|
77
|
-
@storage.put(:increment_value, 0) unless @storage.get(:increment_value)
|
78
|
-
@storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
|
79
|
-
|
80
|
-
if @auto_increment_key && !@storage.get(:auto_increment_value)
|
81
|
-
@storage.put(:auto_increment_value, -1)
|
82
|
-
end
|
83
|
-
|
84
|
-
thread_create(:dummy_input, &method(:run))
|
85
|
-
end
|
86
|
-
|
87
|
-
def run
|
88
|
-
batch_num = (@rate / BIN_NUM).to_i
|
89
|
-
residual_num = (@rate % BIN_NUM)
|
90
|
-
while thread_current_running?
|
91
|
-
current_time = Time.now.to_i
|
92
|
-
BIN_NUM.times do
|
93
|
-
break unless (thread_current_running? && Time.now.to_i <= current_time)
|
94
|
-
wait(0.1) { emit(batch_num) }
|
95
|
-
end
|
96
|
-
emit(residual_num) if thread_current_running?
|
97
|
-
# wait for next second
|
98
|
-
while thread_current_running? && Time.now.to_i <= current_time
|
99
|
-
sleep 0.01
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def emit(num)
|
105
|
-
begin
|
106
|
-
if @size > 1
|
107
|
-
num.times do
|
108
|
-
router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
|
109
|
-
end
|
110
|
-
else
|
111
|
-
num.times { router.emit(@tag, Fluent::Engine.now, generate) }
|
112
|
-
end
|
113
|
-
rescue => _
|
114
|
-
# ignore all errors not to stop emits by emit errors
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def generate
|
119
|
-
d = @dummy[@dummy_index]
|
120
|
-
unless d
|
121
|
-
@dummy_index = 0
|
122
|
-
d = @dummy[@dummy_index]
|
123
|
-
end
|
124
|
-
@dummy_index += 1
|
125
|
-
if @auto_increment_key
|
126
|
-
d = d.dup
|
127
|
-
d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
|
128
|
-
end
|
129
|
-
d
|
130
|
-
end
|
131
|
-
|
132
|
-
def wait(time)
|
133
|
-
start_time = Time.now
|
134
|
-
yield
|
135
|
-
sleep_time = time - (Time.now - start_time)
|
136
|
-
sleep sleep_time if sleep_time > 0
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
17
|
+
# Remove this file in fluentd v2
|
18
|
+
require_relative 'in_sample'
|
@@ -25,6 +25,8 @@ module Fluent::Plugin
|
|
25
25
|
|
26
26
|
desc 'The command (program) to execute.'
|
27
27
|
config_param :command, :string
|
28
|
+
desc 'Specify connect mode to executed process'
|
29
|
+
config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
|
28
30
|
|
29
31
|
config_section :parse do
|
30
32
|
config_set_default :@type, 'tsv'
|
@@ -72,9 +74,9 @@ module Fluent::Plugin
|
|
72
74
|
super
|
73
75
|
|
74
76
|
if @run_interval
|
75
|
-
child_process_execute(:exec_input, @command, interval: @run_interval, mode: [
|
77
|
+
child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
|
76
78
|
else
|
77
|
-
child_process_execute(:exec_input, @command, immediate: true, mode: [
|
79
|
+
child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
|
78
80
|
end
|
79
81
|
end
|
80
82
|
|
@@ -327,7 +327,7 @@ module Fluent::Plugin
|
|
327
327
|
record = e[1]
|
328
328
|
next if record.nil?
|
329
329
|
time = e[0]
|
330
|
-
time = Fluent::
|
330
|
+
time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
|
331
331
|
es.add(time, record)
|
332
332
|
}
|
333
333
|
es
|
@@ -347,7 +347,7 @@ module Fluent::Plugin
|
|
347
347
|
return msg[3] # retry never succeeded so return ack and drop incoming event.
|
348
348
|
end
|
349
349
|
return if record.nil?
|
350
|
-
time = Fluent::
|
350
|
+
time = Fluent::EventTime.now if time.to_i == 0
|
351
351
|
if @enable_field_injection
|
352
352
|
record[@source_address_key] = conn.remote_addr if @source_address_key
|
353
353
|
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
|
@@ -24,13 +24,22 @@ module Fluent::Plugin
|
|
24
24
|
|
25
25
|
def initialize
|
26
26
|
super
|
27
|
+
@key_map = nil
|
27
28
|
end
|
28
29
|
|
29
30
|
config_param :emit_interval, :time, default: 60
|
31
|
+
config_param :use_symbol_keys, :bool, default: true
|
30
32
|
config_param :tag, :string
|
31
33
|
|
32
34
|
def configure(conf)
|
33
35
|
super
|
36
|
+
|
37
|
+
unless @use_symbol_keys
|
38
|
+
@key_map = {}
|
39
|
+
GC.stat.each_key { |key|
|
40
|
+
@key_map[key] = key.to_s
|
41
|
+
}
|
42
|
+
end
|
34
43
|
end
|
35
44
|
|
36
45
|
def multi_workers_ready?
|
@@ -50,6 +59,13 @@ module Fluent::Plugin
|
|
50
59
|
def on_timer
|
51
60
|
now = Fluent::EventTime.now
|
52
61
|
record = GC.stat
|
62
|
+
unless @use_symbol_keys
|
63
|
+
new_record = {}
|
64
|
+
record.each_pair { |k, v|
|
65
|
+
new_record[@key_map[k]] = v
|
66
|
+
}
|
67
|
+
record = new_record
|
68
|
+
end
|
53
69
|
router.emit(@tag, now, record)
|
54
70
|
end
|
55
71
|
end
|
@@ -27,10 +27,28 @@ require 'json'
|
|
27
27
|
module Fluent::Plugin
|
28
28
|
class InHttpParser < Parser
|
29
29
|
Fluent::Plugin.register_parser('in_http', self)
|
30
|
+
|
31
|
+
config_set_default :time_key, 'time'
|
32
|
+
|
33
|
+
def configure(conf)
|
34
|
+
super
|
35
|
+
|
36
|
+
# if no time parser related parameters, use in_http's time convert rule
|
37
|
+
@time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')
|
38
|
+
time_parser_create
|
39
|
+
else
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
30
44
|
def parse(text)
|
31
45
|
# this plugin is dummy implementation not to raise error
|
32
46
|
yield nil, nil
|
33
47
|
end
|
48
|
+
|
49
|
+
def get_time_parser
|
50
|
+
@time_parser
|
51
|
+
end
|
34
52
|
end
|
35
53
|
|
36
54
|
class HttpInput < Input
|
@@ -54,12 +72,14 @@ module Fluent::Plugin
|
|
54
72
|
desc 'Add REMOTE_ADDR header to the record.'
|
55
73
|
config_param :add_remote_addr, :bool, default: false
|
56
74
|
config_param :blocking_timeout, :time, default: 0.5
|
57
|
-
desc 'Set a
|
75
|
+
desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
|
58
76
|
config_param :cors_allow_origins, :array, default: nil
|
59
77
|
desc 'Respond with empty gif image of 1x1 pixel.'
|
60
78
|
config_param :respond_with_empty_img, :bool, default: false
|
61
79
|
desc 'Respond status code with 204.'
|
62
80
|
config_param :use_204_response, :bool, default: false
|
81
|
+
desc 'Dump error log or not'
|
82
|
+
config_param :dump_error_log, :bool, default: true
|
63
83
|
|
64
84
|
config_section :parse do
|
65
85
|
config_set_default :@type, 'in_http'
|
@@ -67,6 +87,24 @@ module Fluent::Plugin
|
|
67
87
|
|
68
88
|
EVENT_RECORD_PARAMETER = '_event_record'
|
69
89
|
|
90
|
+
def initialize
|
91
|
+
super
|
92
|
+
|
93
|
+
@km = nil
|
94
|
+
@format_name = nil
|
95
|
+
@parser_time_key = nil
|
96
|
+
|
97
|
+
# default parsers
|
98
|
+
@parser_msgpack = nil
|
99
|
+
@parser_json = nil
|
100
|
+
@default_time_parser = nil
|
101
|
+
@default_keep_time_key = nil
|
102
|
+
@float_time_parser = nil
|
103
|
+
|
104
|
+
# <parse> configured parser
|
105
|
+
@custom_parser = nil
|
106
|
+
end
|
107
|
+
|
70
108
|
def configure(conf)
|
71
109
|
compat_parameters_convert(conf, :parser)
|
72
110
|
|
@@ -74,20 +112,22 @@ module Fluent::Plugin
|
|
74
112
|
|
75
113
|
m = if @parser_configs.first['@type'] == 'in_http'
|
76
114
|
@parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
|
115
|
+
@parser_msgpack.time_key = nil
|
77
116
|
@parser_msgpack.estimate_current_event = false
|
78
117
|
@parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
|
118
|
+
@parser_json.time_key = nil
|
79
119
|
@parser_json.estimate_current_event = false
|
120
|
+
|
121
|
+
default_parser = parser_create(usage: '')
|
80
122
|
@format_name = 'default'
|
81
|
-
@parser_time_key =
|
82
|
-
|
83
|
-
|
84
|
-
'time'
|
85
|
-
end
|
123
|
+
@parser_time_key = default_parser.time_key
|
124
|
+
@default_time_parser = default_parser.get_time_parser
|
125
|
+
@default_keep_time_key = default_parser.keep_time_key
|
86
126
|
method(:parse_params_default)
|
87
127
|
else
|
88
|
-
@
|
128
|
+
@custom_parser = parser_create
|
89
129
|
@format_name = @parser_configs.first['@type']
|
90
|
-
@parser_time_key = @
|
130
|
+
@parser_time_key = @custom_parser.time_key
|
91
131
|
method(:parse_params_with_parser)
|
92
132
|
end
|
93
133
|
self.singleton_class.module_eval do
|
@@ -142,6 +182,13 @@ module Fluent::Plugin
|
|
142
182
|
super
|
143
183
|
end
|
144
184
|
|
185
|
+
RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze
|
186
|
+
RESPONSE_200 = ["200 OK".freeze, RES_TEXT_HEADER, "".freeze].freeze
|
187
|
+
RESPONSE_204 = ["204 No Content".freeze, {}.freeze].freeze
|
188
|
+
RESPONSE_IMG = ["200 OK".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze
|
189
|
+
RES_400_STATUS = "400 Bad Request".freeze
|
190
|
+
RES_500_STATUS = "500 Internal Server Error".freeze
|
191
|
+
|
145
192
|
def on_request(path_info, params)
|
146
193
|
begin
|
147
194
|
path = path_info[1..-1] # remove /
|
@@ -152,83 +199,77 @@ module Fluent::Plugin
|
|
152
199
|
if record.nil?
|
153
200
|
log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
|
154
201
|
if @respond_with_empty_img
|
155
|
-
return
|
202
|
+
return RESPONSE_IMG
|
156
203
|
else
|
157
204
|
if @use_204_response
|
158
|
-
return
|
205
|
+
return RESPONSE_204
|
159
206
|
else
|
160
|
-
return
|
207
|
+
return RESPONSE_200
|
161
208
|
end
|
162
209
|
end
|
163
210
|
end
|
164
211
|
|
165
|
-
|
166
|
-
if @add_http_headers
|
167
|
-
params.each_pair { |k,v|
|
168
|
-
if k.start_with?("HTTP_")
|
169
|
-
record[k] = v
|
170
|
-
end
|
171
|
-
}
|
172
|
-
end
|
173
|
-
if @add_remote_addr
|
174
|
-
record['REMOTE_ADDR'] = params['REMOTE_ADDR']
|
175
|
-
end
|
176
|
-
end
|
177
|
-
time = if param_time = params['time']
|
178
|
-
param_time = param_time.to_f
|
179
|
-
param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
|
180
|
-
else
|
181
|
-
record_time.nil? ? Fluent::Engine.now : record_time
|
182
|
-
end
|
183
|
-
rescue
|
184
|
-
return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
|
185
|
-
end
|
186
|
-
|
187
|
-
# TODO server error
|
188
|
-
begin
|
212
|
+
mes = nil
|
189
213
|
# Support batched requests
|
190
214
|
if record.is_a?(Array)
|
191
215
|
mes = Fluent::MultiEventStream.new
|
192
216
|
record.each do |single_record|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
|
202
|
-
end
|
203
|
-
|
204
|
-
if defined? @parser
|
205
|
-
single_time = @parser.parse_time(single_record)
|
206
|
-
single_time, single_record = @parser.convert_values(single_time, single_record)
|
217
|
+
add_params_to_record(single_record, params)
|
218
|
+
|
219
|
+
if param_time = params['time']
|
220
|
+
param_time = param_time.to_f
|
221
|
+
single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
|
222
|
+
elsif @custom_parser
|
223
|
+
single_time = @custom_parser.parse_time(single_record)
|
224
|
+
single_time, single_record = @custom_parser.convert_values(single_time, single_record)
|
207
225
|
else
|
208
|
-
single_time =
|
209
|
-
Fluent::EventTime.from_time(Time.at(t))
|
210
|
-
else
|
211
|
-
time
|
212
|
-
end
|
226
|
+
single_time = convert_time_field(single_record)
|
213
227
|
end
|
214
228
|
|
215
229
|
mes.add(single_time, single_record)
|
216
230
|
end
|
231
|
+
else
|
232
|
+
add_params_to_record(record, params)
|
233
|
+
|
234
|
+
time = if param_time = params['time']
|
235
|
+
param_time = param_time.to_f
|
236
|
+
param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
|
237
|
+
else
|
238
|
+
if record_time.nil?
|
239
|
+
convert_time_field(record)
|
240
|
+
else
|
241
|
+
record_time
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
245
|
+
rescue => e
|
246
|
+
if @dump_error_log
|
247
|
+
log.error "failed to process request", error: e
|
248
|
+
end
|
249
|
+
return [RES_400_STATUS, RES_TEXT_HEADER, "400 Bad Request\n#{e}\n"]
|
250
|
+
end
|
251
|
+
|
252
|
+
# TODO server error
|
253
|
+
begin
|
254
|
+
if mes
|
217
255
|
router.emit_stream(tag, mes)
|
218
256
|
else
|
219
257
|
router.emit(tag, time, record)
|
220
258
|
end
|
221
|
-
rescue
|
222
|
-
|
259
|
+
rescue => e
|
260
|
+
if @dump_error_log
|
261
|
+
log.error "failed to emit data", error: e
|
262
|
+
end
|
263
|
+
return [RES_500_STATUS, RES_TEXT_HEADER, "500 Internal Server Error\n#{e}\n"]
|
223
264
|
end
|
224
265
|
|
225
266
|
if @respond_with_empty_img
|
226
|
-
return
|
267
|
+
return RESPONSE_IMG
|
227
268
|
else
|
228
269
|
if @use_204_response
|
229
|
-
return
|
270
|
+
return RESPONSE_204
|
230
271
|
else
|
231
|
-
return
|
272
|
+
return RESPONSE_200
|
232
273
|
end
|
233
274
|
end
|
234
275
|
end
|
@@ -267,7 +308,7 @@ module Fluent::Plugin
|
|
267
308
|
|
268
309
|
def parse_params_with_parser(params)
|
269
310
|
if content = params[EVENT_RECORD_PARAMETER]
|
270
|
-
@
|
311
|
+
@custom_parser.parse(content) { |time, record|
|
271
312
|
raise "Received event is not #{@format_name}: #{content}" if record.nil?
|
272
313
|
return time, record
|
273
314
|
}
|
@@ -276,6 +317,32 @@ module Fluent::Plugin
|
|
276
317
|
end
|
277
318
|
end
|
278
319
|
|
320
|
+
def add_params_to_record(record, params)
|
321
|
+
if @add_http_headers
|
322
|
+
params.each_pair { |k, v|
|
323
|
+
if k.start_with?("HTTP_".freeze)
|
324
|
+
record[k] = v
|
325
|
+
end
|
326
|
+
}
|
327
|
+
end
|
328
|
+
|
329
|
+
if @add_remote_addr
|
330
|
+
record['REMOTE_ADDR'] = params['REMOTE_ADDR']
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
def convert_time_field(record)
|
335
|
+
if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)
|
336
|
+
if @default_time_parser
|
337
|
+
@default_time_parser.parse(t)
|
338
|
+
else
|
339
|
+
Fluent::EventTime.from_time(Time.at(t))
|
340
|
+
end
|
341
|
+
else
|
342
|
+
Fluent::EventTime.now
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
279
346
|
class Handler
|
280
347
|
attr_reader :content_type
|
281
348
|
|
@@ -358,7 +425,7 @@ module Fluent::Plugin
|
|
358
425
|
end
|
359
426
|
}
|
360
427
|
if expect
|
361
|
-
if expect == '100-continue'
|
428
|
+
if expect == '100-continue'.freeze
|
362
429
|
if !size || size < @body_size_limit
|
363
430
|
send_response_nobody("100 Continue", {})
|
364
431
|
else
|
@@ -380,17 +447,20 @@ module Fluent::Plugin
|
|
380
447
|
@body << chunk
|
381
448
|
end
|
382
449
|
|
450
|
+
RES_200_STATUS = "200 OK".freeze
|
451
|
+
RES_403_STATUS = "403 Forbidden".freeze
|
452
|
+
|
383
453
|
# Web browsers can send an OPTIONS request before performing POST
|
384
454
|
# to check if cross-origin requests are supported.
|
385
455
|
def handle_options_request
|
386
456
|
# Is CORS enabled in the first place?
|
387
457
|
if @cors_allow_origins.nil?
|
388
|
-
return send_response_and_close(
|
458
|
+
return send_response_and_close(RES_403_STATUS, {}, "")
|
389
459
|
end
|
390
460
|
|
391
461
|
# in_http does not support HTTP methods except POST
|
392
462
|
if @access_control_request_method != 'POST'
|
393
|
-
return send_response_and_close(
|
463
|
+
return send_response_and_close(RES_403_STATUS, {}, "")
|
394
464
|
end
|
395
465
|
|
396
466
|
header = {
|
@@ -401,29 +471,29 @@ module Fluent::Plugin
|
|
401
471
|
# Check the origin and send back a CORS response
|
402
472
|
if @cors_allow_origins.include?('*')
|
403
473
|
header["Access-Control-Allow-Origin"] = "*"
|
404
|
-
send_response_and_close(
|
474
|
+
send_response_and_close(RES_200_STATUS, header, "")
|
405
475
|
elsif include_cors_allow_origin
|
406
476
|
header["Access-Control-Allow-Origin"] = @origin
|
407
|
-
send_response_and_close(
|
477
|
+
send_response_and_close(RES_200_STATUS, header, "")
|
408
478
|
else
|
409
|
-
send_response_and_close(
|
479
|
+
send_response_and_close(RES_403_STATUS, {}, "")
|
410
480
|
end
|
411
481
|
end
|
412
482
|
|
413
483
|
def on_message_complete
|
414
484
|
return if closing?
|
415
485
|
|
416
|
-
if @parser.http_method == 'OPTIONS'
|
486
|
+
if @parser.http_method == 'OPTIONS'.freeze
|
417
487
|
return handle_options_request()
|
418
488
|
end
|
419
489
|
|
420
490
|
# CORS check
|
421
491
|
# ==========
|
422
492
|
# For every incoming request, we check if we have some CORS
|
423
|
-
# restrictions and
|
493
|
+
# restrictions and allow listed origins through @cors_allow_origins.
|
424
494
|
unless @cors_allow_origins.nil?
|
425
495
|
unless @cors_allow_origins.include?('*') or include_cors_allow_origin
|
426
|
-
send_response_and_close(
|
496
|
+
send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
|
427
497
|
return
|
428
498
|
end
|
429
499
|
end
|
@@ -433,14 +503,14 @@ module Fluent::Plugin
|
|
433
503
|
# Decode payload according to the "Content-Encoding" header.
|
434
504
|
# For now, we only support 'gzip' and 'deflate'.
|
435
505
|
begin
|
436
|
-
if @content_encoding == 'gzip'
|
506
|
+
if @content_encoding == 'gzip'.freeze
|
437
507
|
@body = Zlib::GzipReader.new(StringIO.new(@body)).read
|
438
|
-
elsif @content_encoding == 'deflate'
|
508
|
+
elsif @content_encoding == 'deflate'.freeze
|
439
509
|
@body = Zlib::Inflate.inflate(@body)
|
440
510
|
end
|
441
511
|
rescue
|
442
512
|
@log.warn 'fails to decode payload', error: $!.to_s
|
443
|
-
send_response_and_close(
|
513
|
+
send_response_and_close(RES_400_STATUS, {}, "")
|
444
514
|
return
|
445
515
|
end
|
446
516
|
|
@@ -466,8 +536,9 @@ module Fluent::Plugin
|
|
466
536
|
params.merge!(@env)
|
467
537
|
@env.clear
|
468
538
|
|
469
|
-
code, header, body =
|
539
|
+
code, header, body = @callback.call(path_info, params)
|
470
540
|
body = body.to_s
|
541
|
+
header = header.dup if header.frozen?
|
471
542
|
|
472
543
|
unless @cors_allow_origins.nil?
|
473
544
|
if @cors_allow_origins.include?('*')
|
@@ -478,7 +549,7 @@ module Fluent::Plugin
|
|
478
549
|
end
|
479
550
|
|
480
551
|
if @keep_alive
|
481
|
-
header['Connection'] = 'Keep-Alive'
|
552
|
+
header['Connection'] = 'Keep-Alive'.freeze
|
482
553
|
send_response(code, header, body)
|
483
554
|
else
|
484
555
|
send_response_and_close(code, header, body)
|
@@ -504,13 +575,13 @@ module Fluent::Plugin
|
|
504
575
|
|
505
576
|
def send_response(code, header, body)
|
506
577
|
header['Content-Length'] ||= body.bytesize
|
507
|
-
header['Content-Type'] ||= 'text/plain'
|
578
|
+
header['Content-Type'] ||= 'text/plain'.freeze
|
508
579
|
|
509
580
|
data = %[HTTP/1.1 #{code}\r\n]
|
510
581
|
header.each_pair {|k,v|
|
511
582
|
data << "#{k}: #{v}\r\n"
|
512
583
|
}
|
513
|
-
data << "\r\n"
|
584
|
+
data << "\r\n".freeze
|
514
585
|
@io.write(data)
|
515
586
|
|
516
587
|
@io.write(body)
|
@@ -521,7 +592,7 @@ module Fluent::Plugin
|
|
521
592
|
header.each_pair {|k,v|
|
522
593
|
data << "#{k}: #{v}\r\n"
|
523
594
|
}
|
524
|
-
data << "\r\n"
|
595
|
+
data << "\r\n".freeze
|
525
596
|
@io.write(data)
|
526
597
|
end
|
527
598
|
|