fluentd 1.6.3-x64-mingw32 → 1.7.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +35 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
  4. data/CHANGELOG.md +58 -0
  5. data/README.md +5 -1
  6. data/fluentd.gemspec +1 -1
  7. data/lib/fluent/clock.rb +4 -0
  8. data/lib/fluent/compat/output.rb +3 -3
  9. data/lib/fluent/compat/socket_util.rb +1 -1
  10. data/lib/fluent/config/element.rb +3 -3
  11. data/lib/fluent/config/literal_parser.rb +1 -1
  12. data/lib/fluent/config/section.rb +4 -1
  13. data/lib/fluent/error.rb +4 -0
  14. data/lib/fluent/event.rb +28 -24
  15. data/lib/fluent/event_router.rb +2 -1
  16. data/lib/fluent/log.rb +1 -1
  17. data/lib/fluent/msgpack_factory.rb +8 -0
  18. data/lib/fluent/plugin/bare_output.rb +4 -4
  19. data/lib/fluent/plugin/buf_file_single.rb +211 -0
  20. data/lib/fluent/plugin/buffer.rb +62 -63
  21. data/lib/fluent/plugin/buffer/chunk.rb +21 -3
  22. data/lib/fluent/plugin/buffer/file_chunk.rb +37 -12
  23. data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
  24. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
  25. data/lib/fluent/plugin/compressable.rb +10 -6
  26. data/lib/fluent/plugin/filter_grep.rb +2 -2
  27. data/lib/fluent/plugin/formatter_csv.rb +10 -6
  28. data/lib/fluent/plugin/in_syslog.rb +10 -3
  29. data/lib/fluent/plugin/in_tail.rb +7 -2
  30. data/lib/fluent/plugin/in_tcp.rb +34 -7
  31. data/lib/fluent/plugin/multi_output.rb +4 -4
  32. data/lib/fluent/plugin/out_exec_filter.rb +1 -0
  33. data/lib/fluent/plugin/out_file.rb +13 -3
  34. data/lib/fluent/plugin/out_forward.rb +126 -588
  35. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  36. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  37. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  38. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  39. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
  40. data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
  41. data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
  42. data/lib/fluent/plugin/out_http.rb +231 -0
  43. data/lib/fluent/plugin/output.rb +29 -35
  44. data/lib/fluent/plugin/parser.rb +77 -0
  45. data/lib/fluent/plugin/parser_csv.rb +75 -0
  46. data/lib/fluent/plugin_helper/server.rb +1 -1
  47. data/lib/fluent/plugin_helper/thread.rb +1 -0
  48. data/lib/fluent/root_agent.rb +1 -1
  49. data/lib/fluent/time.rb +4 -2
  50. data/lib/fluent/timezone.rb +21 -7
  51. data/lib/fluent/version.rb +1 -1
  52. data/test/command/test_fluentd.rb +1 -1
  53. data/test/command/test_plugin_generator.rb +18 -2
  54. data/test/config/test_configurable.rb +78 -40
  55. data/test/counter/test_store.rb +1 -1
  56. data/test/helper.rb +1 -0
  57. data/test/helpers/process_extenstion.rb +33 -0
  58. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  59. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  60. data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
  61. data/test/plugin/out_forward/test_load_balancer.rb +60 -0
  62. data/test/plugin/out_forward/test_socket_cache.rb +139 -0
  63. data/test/plugin/test_buf_file.rb +118 -2
  64. data/test/plugin/test_buf_file_single.rb +734 -0
  65. data/test/plugin/test_buffer.rb +4 -48
  66. data/test/plugin/test_buffer_file_chunk.rb +19 -1
  67. data/test/plugin/test_buffer_file_single_chunk.rb +620 -0
  68. data/test/plugin/test_formatter_csv.rb +16 -0
  69. data/test/plugin/test_in_syslog.rb +56 -6
  70. data/test/plugin/test_in_tail.rb +1 -1
  71. data/test/plugin/test_in_tcp.rb +25 -0
  72. data/test/plugin/test_out_forward.rb +75 -201
  73. data/test/plugin/test_out_http.rb +352 -0
  74. data/test/plugin/test_output_as_buffered.rb +27 -24
  75. data/test/plugin/test_parser.rb +40 -0
  76. data/test/plugin/test_parser_csv.rb +83 -0
  77. data/test/plugin_helper/test_record_accessor.rb +1 -1
  78. data/test/test_time_formatter.rb +140 -121
  79. metadata +33 -4
@@ -24,6 +24,8 @@ module Fluent
24
24
  class FileChunk < Chunk
25
25
  class FileChunkError < StandardError; end
26
26
 
27
+ BUFFER_HEADER = "\xc1\x00".force_encoding(Encoding::ASCII_8BIT).freeze
28
+
27
29
  ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log
28
30
  ### buffer chunk path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log
29
31
  ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta
@@ -74,7 +76,8 @@ module Fluent
74
76
  @size += @adding_size
75
77
  @bytesize += @adding_bytes
76
78
  @adding_bytes = @adding_size = 0
77
- @modified_at = Time.now
79
+ @modified_at = Fluent::Clock.real_now
80
+ @modified_at_object = nil
78
81
 
79
82
  true
80
83
  end
@@ -214,14 +217,19 @@ module Fluent
214
217
  end
215
218
 
216
219
  def restore_metadata(bindata)
217
- data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
220
+ data = restore_metadata_with_new_format(bindata)
221
+
222
+ unless data
223
+ # old type of restore
224
+ data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
225
+ end
218
226
 
219
- now = Time.now
227
+ now = Fluent::Clock.real_now
220
228
 
221
229
  @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
222
230
  @size = data[:s] || 0
223
- @created_at = Time.at(data.fetch(:c, now.to_i))
224
- @modified_at = Time.at(data.fetch(:m, now.to_i))
231
+ @created_at = data.fetch(:c, now.to_i)
232
+ @modified_at = data.fetch(:m, now.to_i)
225
233
 
226
234
  @metadata.timekey = data[:timekey]
227
235
  @metadata.tag = data[:tag]
@@ -231,8 +239,8 @@ module Fluent
231
239
  def restore_metadata_partially(chunk)
232
240
  @unique_id = self.class.unique_id_from_path(chunk.path) || @unique_id
233
241
  @size = 0
234
- @created_at = chunk.ctime # birthtime isn't supported on Windows (and Travis?)
235
- @modified_at = chunk.mtime
242
+ @created_at = chunk.ctime.to_i # birthtime isn't supported on Windows (and Travis?)
243
+ @modified_at = chunk.mtime.to_i
236
244
 
237
245
  @metadata.timekey = nil
238
246
  @metadata.tag = nil
@@ -243,13 +251,13 @@ module Fluent
243
251
  data = @metadata.to_h.merge({
244
252
  id: @unique_id,
245
253
  s: (update ? @size + @adding_size : @size),
246
- c: @created_at.to_i,
247
- m: (update ? Time.now : @modified_at).to_i,
254
+ c: @created_at,
255
+ m: (update ? Fluent::Clock.real_now : @modified_at),
248
256
  })
249
- bin = msgpack_packer.pack(data).to_s
257
+ bin = Fluent::MessagePackFactory.thread_local_msgpack_packer.pack(data).full_pack
258
+ size = [bin.bytesize].pack('N')
250
259
  @meta.seek(0, IO::SEEK_SET)
251
- @meta.write(bin)
252
- @meta.truncate(bin.bytesize)
260
+ @meta.write(BUFFER_HEADER + size + bin)
253
261
  end
254
262
 
255
263
  def file_rename(file, old_path, new_path, callback=nil)
@@ -377,6 +385,23 @@ module Fluent
377
385
  end
378
386
  @state = :queued
379
387
  end
388
+
389
+ private
390
+
391
+ def restore_metadata_with_new_format(chunk)
392
+ if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data size(4)
393
+ return nil
394
+ end
395
+
396
+ if chunk.slice(0, 2) == BUFFER_HEADER
397
+ size = chunk.slice(2, 4).unpack('N').first
398
+ if size
399
+ return msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
400
+ end
401
+ end
402
+
403
+ nil
404
+ end
380
405
  end
381
406
  end
382
407
  end
@@ -0,0 +1,314 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'uri'
18
+ require 'fluent/plugin/buffer/chunk'
19
+ require 'fluent/unique_id'
20
+ require 'fluent/msgpack_factory'
21
+
22
+ module Fluent
23
+ module Plugin
24
+ class Buffer
25
+ class FileSingleChunk < Chunk
26
+ class FileChunkError < StandardError; end
27
+
28
+ ## buffer path user specified : /path/to/directory
29
+ ## buffer chunk path : /path/to/directory/fsb.key.b513b61c9791029c2513b61c9791029c2.buf
30
+ ## state: b/q - 'b'(on stage), 'q'(enqueued)
31
+
32
+ include SystemConfig::Mixin
33
+ include MessagePackFactory::Mixin
34
+
35
+ PATH_EXT = 'buf'
36
+ PATH_SUFFIX = ".#{PATH_EXT}"
37
+ PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n # //n switch means explicit 'ASCII-8BIT' pattern
38
+ FILE_PERMISSION = 0644
39
+
40
+ attr_reader :path, :permission
41
+
42
+ def initialize(metadata, path, mode, key, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
43
+ super(metadata, compress: compress)
44
+ @key = key
45
+ @permission = perm.is_a?(String) ? perm.to_i(8) : perm
46
+ @bytesize = @size = @adding_bytes = @adding_size = 0
47
+
48
+ case mode
49
+ when :create then create_new_chunk(path, metadata, @permission)
50
+ when :staged then load_existing_staged_chunk(path)
51
+ when :queued then load_existing_enqueued_chunk(path)
52
+ else
53
+ raise ArgumentError, "Invalid file chunk mode: #{mode}"
54
+ end
55
+ end
56
+
57
+ def concat(bulk, bulk_size)
58
+ raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
59
+
60
+ bulk.force_encoding(Encoding::ASCII_8BIT)
61
+ @chunk.write(bulk)
62
+ @adding_bytes += bulk.bytesize
63
+ @adding_size += bulk_size
64
+ true
65
+ end
66
+
67
+ def commit
68
+ @commit_position = @chunk.pos
69
+ @size += @adding_size
70
+ @bytesize += @adding_bytes
71
+ @adding_bytes = @adding_size = 0
72
+ @modified_at = Fluent::Clock.real_now
73
+
74
+ true
75
+ end
76
+
77
+ def rollback
78
+ if @chunk.pos != @commit_position
79
+ @chunk.seek(@commit_position, IO::SEEK_SET)
80
+ @chunk.truncate(@commit_position)
81
+ end
82
+ @adding_bytes = @adding_size = 0
83
+ true
84
+ end
85
+
86
+ def bytesize
87
+ @bytesize + @adding_bytes
88
+ end
89
+
90
+ def size
91
+ @size + @adding_size
92
+ end
93
+
94
+ def empty?
95
+ @bytesize.zero?
96
+ end
97
+
98
+ def enqueued!
99
+ return unless self.staged?
100
+
101
+ new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)
102
+
103
+ begin
104
+ file_rename(@chunk, @path, new_chunk_path, ->(new_io) { @chunk = new_io })
105
+ rescue => e
106
+ begin
107
+ file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)
108
+ rescue => re
109
+ # In this point, restore buffer state is hard because previous `file_rename` failed by resource problem.
110
+ # Retry is one possible approach but it may cause livelock under limited resources or high load environment.
111
+ # So we ignore such errors for now and log better message instead.
112
+ # "Too many open files" should be fixed by proper buffer configuration and system setting.
113
+ raise "can't enqueue buffer file and failed to restore. This may causes inconsistent state: path = #{@path}, error = '#{e}', retry error = '#{re}'"
114
+ else
115
+ raise "can't enqueue buffer file: path = #{@path}, error = '#{e}'"
116
+ end
117
+ end
118
+
119
+ @path = new_chunk_path
120
+
121
+ super
122
+ end
123
+
124
+ def close
125
+ super
126
+ size = @chunk.size
127
+ @chunk.close
128
+ if size == 0
129
+ File.unlink(@path)
130
+ end
131
+ end
132
+
133
+ def purge
134
+ super
135
+ @chunk.close
136
+ @bytesize = @size = @adding_bytes = @adding_size = 0
137
+ File.unlink(@path)
138
+ end
139
+
140
+ def read(**kwargs)
141
+ @chunk.seek(0, IO::SEEK_SET)
142
+ @chunk.read
143
+ end
144
+
145
+ def open(**kwargs, &block)
146
+ @chunk.seek(0, IO::SEEK_SET)
147
+ val = yield @chunk
148
+ @chunk.seek(0, IO::SEEK_END) if self.staged?
149
+ val
150
+ end
151
+
152
+ def self.assume_chunk_state(path)
153
+ return :unknown unless path.end_with?(PATH_SUFFIX)
154
+
155
+ if PATH_REGEXP =~ path
156
+ $1 == 'b' ? :staged : :queued
157
+ else
158
+ # files which matches to glob of buffer file pattern
159
+ # it includes files which are created by out_file
160
+ :unknown
161
+ end
162
+ end
163
+
164
+ def self.unique_id_and_key_from_path(path)
165
+ base = File.basename(path)
166
+ res = PATH_REGEXP =~ base
167
+ return nil unless res
168
+
169
+ key = base[4..res - 1] # remove 'fsb.' and '.'
170
+ hex_id = $2 # remove '.' and '.buf'
171
+ unique_id = hex_id.scan(/../).map {|x| x.to_i(16) }.pack('C*')
172
+ [unique_id, key]
173
+ end
174
+
175
+ def self.generate_stage_chunk_path(path, key, unique_id)
176
+ pos = path.index('.*.')
177
+ raise "BUG: buffer chunk path on stage MUST have '.*.'" unless pos
178
+
179
+ prefix = path[0...pos]
180
+ suffix = path[(pos + 3)..-1]
181
+
182
+ chunk_id = Fluent::UniqueId.hex(unique_id)
183
+ "#{prefix}.#{key}.b#{chunk_id}.#{suffix}"
184
+ end
185
+
186
+ def self.generate_queued_chunk_path(path, unique_id)
187
+ chunk_id = Fluent::UniqueId.hex(unique_id)
188
+ staged_path = ".b#{chunk_id}."
189
+ if path.index(staged_path)
190
+ path.sub(staged_path, ".q#{chunk_id}.")
191
+ else # for unexpected cases (ex: users rename files while opened by fluentd)
192
+ path + ".q#{chunk_id}.chunk"
193
+ end
194
+ end
195
+
196
+ def restore_metadata
197
+ if res = self.class.unique_id_and_key_from_path(@path)
198
+ @unique_id = res.first
199
+ key = decode_key(res.last)
200
+ if @key
201
+ @metadata.variables = {@key => key}
202
+ else
203
+ @metadata.tag = key
204
+ end
205
+ else
206
+ raise FileChunkError, "Invalid chunk found. unique_id and key not exist: #{@path}"
207
+ end
208
+ @size = 0
209
+
210
+ stat = File.stat(@path)
211
+ @created_at = stat.ctime.to_i
212
+ @modified_at = stat.mtime.to_i
213
+ end
214
+
215
+ def restore_size(chunk_format)
216
+ count = 0
217
+ File.open(@path, 'rb') { |f|
218
+ if chunk_format == :msgpack
219
+ msgpack_unpacker(f).each { |d| count += 1 }
220
+ else
221
+ f.each_line { |l| count += 1 }
222
+ end
223
+ }
224
+ @size = count
225
+ end
226
+
227
+ def file_rename(file, old_path, new_path, callback = nil)
228
+ pos = file.pos
229
+ if Fluent.windows?
230
+ file.close
231
+ File.rename(old_path, new_path)
232
+ file = File.open(new_path, 'rb', @permission)
233
+ else
234
+ File.rename(old_path, new_path)
235
+ file.reopen(new_path, 'rb')
236
+ end
237
+ file.set_encoding(Encoding::ASCII_8BIT)
238
+ file.sync = true
239
+ file.binmode
240
+ file.pos = pos
241
+ callback.call(file) if callback
242
+ end
243
+
244
+ URI_PARSER = URI::Parser.new
245
+ ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
246
+
247
+ def encode_key(metadata)
248
+ k = @key ? metadata.variables[@key] : metadata.tag
249
+ k ||= ''
250
+ URI_PARSER.escape(k, ESCAPE_REGEXP)
251
+ end
252
+
253
+ def decode_key(key)
254
+ URI_PARSER.unescape(key)
255
+ end
256
+
257
+ def create_new_chunk(path, metadata, perm)
258
+ @path = self.class.generate_stage_chunk_path(path, encode_key(metadata), @unique_id)
259
+ begin
260
+ @chunk = File.open(@path, 'wb+', perm)
261
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
262
+ @chunk.sync = true
263
+ @chunk.binmode
264
+ rescue => e
265
+ # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
266
+ # If other cases are possible, we will change erorr handling with proper classes.
267
+ raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
268
+ end
269
+
270
+ @state = :unstaged
271
+ @bytesize = 0
272
+ @commit_position = @chunk.pos # must be 0
273
+ @adding_bytes = 0
274
+ @adding_size = 0
275
+ end
276
+
277
+ def load_existing_staged_chunk(path)
278
+ @path = path
279
+ raise FileChunkError, "staged file chunk is empty" if File.size(@path).zero?
280
+
281
+ @chunk = File.open(@path, 'rb+')
282
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
283
+ @chunk.sync = true
284
+ @chunk.binmode
285
+ @chunk.seek(0, IO::SEEK_END)
286
+
287
+ restore_metadata
288
+
289
+ @state = :staged
290
+ @bytesize = @chunk.size
291
+ @commit_position = @chunk.pos
292
+ @adding_bytes = 0
293
+ @adding_size = 0
294
+ end
295
+
296
+ def load_existing_enqueued_chunk(path)
297
+ @path = path
298
+ raise FileChunkError, "enqueued file chunk is empty" if File.size(@path).zero?
299
+
300
+ @chunk = File.open(@path, 'rb')
301
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
302
+ @chunk.binmode
303
+ @chunk.seek(0, IO::SEEK_SET)
304
+
305
+ restore_metadata
306
+
307
+ @state = :queued
308
+ @bytesize = @chunk.size
309
+ @commit_position = @chunk.size
310
+ end
311
+ end
312
+ end
313
+ end
314
+ end
@@ -43,7 +43,8 @@ module Fluent
43
43
  @chunk_bytes += @adding_bytes
44
44
 
45
45
  @adding_bytes = @adding_size = 0
46
- @modified_at = Time.now
46
+ @modified_at = Fluent::Clock.real_now
47
+ @modified_at_object = nil
47
48
  true
48
49
  end
49
50
 
@@ -64,9 +64,11 @@ module Fluent
64
64
  unused = gz.unused
65
65
  gz.finish
66
66
 
67
- break if unused.nil?
68
- adjust = unused.length
69
- io.pos -= adjust
67
+ unless unused.nil?
68
+ adjust = unused.length
69
+ io.pos -= adjust
70
+ end
71
+ break if io.eof?
70
72
  end
71
73
 
72
74
  out
@@ -80,9 +82,11 @@ module Fluent
80
82
  unused = gz.unused
81
83
  gz.finish
82
84
 
83
- break if unused.nil?
84
- adjust = unused.length
85
- input.pos -= adjust
85
+ unless unused.nil?
86
+ adjust = unused.length
87
+ input.pos -= adjust
88
+ end
89
+ break if input.eof?
86
90
  end
87
91
 
88
92
  output
@@ -110,7 +110,7 @@ module Fluent::Plugin
110
110
  end
111
111
 
112
112
  if @regexps.size > 1
113
- log.info "Top level multiple <regexp> is intepreted as 'and' condition"
113
+ log.info "Top level multiple <regexp> is interpreted as 'and' condition"
114
114
  end
115
115
  @regexps.each do |e|
116
116
  raise Fluent::ConfigError, "Duplicate key: #{e.key}" if regexp_and_conditions.key?(e.key)
@@ -118,7 +118,7 @@ module Fluent::Plugin
118
118
  end
119
119
 
120
120
  if @excludes.size > 1
121
- log.info "Top level multiple <exclude> is intepreted as 'or' condition"
121
+ log.info "Top level multiple <exclude> is interpreted as 'or' condition"
122
122
  end
123
123
  @excludes.each do |e|
124
124
  raise Fluent::ConfigError, "Duplicate key: #{e.key}" if exclude_or_conditions.key?(e.key)