fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -156,6 +156,18 @@ module Fluent
156
156
  @_state.terminate
157
157
  end
158
158
 
159
+ def called_in_test?
160
+ caller_locations.each do |location|
161
+ # Thread::Backtrace::Location#path returns base filename or absolute path.
162
+ # #absolute_path returns absolute_path always.
163
+ # https://bugs.ruby-lang.org/issues/12159
164
+ if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
165
+ return true
166
+ end
167
+ end
168
+ false
169
+ end
170
+
159
171
  def inspect
160
172
  # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
161
173
  # Original commit comment says that:
@@ -66,7 +66,7 @@ module Fluent
66
66
  end
67
67
 
68
68
  type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
69
- if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
69
+ if @@buffer_paths.has_key?(@path) && !called_in_test?
70
70
  type_using_this_path = @@buffer_paths[@path]
71
71
  raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
72
72
  end
@@ -114,18 +114,6 @@ module Fluent
114
114
  @multi_workers_available
115
115
  end
116
116
 
117
- def buffer_path_for_test?
118
- caller_locations.each do |location|
119
- # Thread::Backtrace::Location#path returns base filename or absolute path.
120
- # #absolute_path returns absolute_path always.
121
- # https://bugs.ruby-lang.org/issues/12159
122
- if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
123
- return true
124
- end
125
- end
126
- false
127
- end
128
-
129
117
  def start
130
118
  FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
131
119
 
@@ -153,7 +141,13 @@ module Fluent
153
141
  next
154
142
  end
155
143
 
156
- chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
144
+ begin
145
+ chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
146
+ rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
147
+ handle_broken_files(path, mode, e)
148
+ next
149
+ end
150
+
157
151
  case chunk.state
158
152
  when :staged
159
153
  stage[chunk.metadata] = chunk
@@ -170,10 +164,20 @@ module Fluent
170
164
  def generate_chunk(metadata)
171
165
  # FileChunk generates real path with unique_id
172
166
  if @file_permission
173
- Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
167
+ chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
174
168
  else
175
- Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
169
+ chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
176
170
  end
171
+
172
+ log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
173
+
174
+ return chunk
175
+ end
176
+
177
+ def handle_broken_files(path, mode, e)
178
+ log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
179
+ # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
180
+ File.unlink(path, path + '.meta') rescue nil
177
181
  end
178
182
  end
179
183
  end
@@ -55,6 +55,9 @@ module Fluent
55
55
  # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
56
56
  config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
57
57
 
58
+ desc 'The max number of queued chunks.'
59
+ config_param :queued_chunks_limit_size, :integer, default: nil
60
+
58
61
  desc 'Compress buffered data.'
59
62
  config_param :compress, :enum, list: [:text, :gzip], default: :text
60
63
 
@@ -62,6 +65,74 @@ module Fluent
62
65
  def empty?
63
66
  timekey.nil? && tag.nil? && variables.nil?
64
67
  end
68
+
69
+ def cmp_variables(v1, v2)
70
+ if v1.nil? && v2.nil?
71
+ return 0
72
+ elsif v1.nil? # v2 is non-nil
73
+ return -1
74
+ elsif v2.nil? # v1 is non-nil
75
+ return 1
76
+ end
77
+ # both of v1 and v2 are non-nil
78
+ v1_sorted_keys = v1.keys.sort
79
+ v2_sorted_keys = v2.keys.sort
80
+ if v1_sorted_keys != v2_sorted_keys
81
+ if v1_sorted_keys.size == v2_sorted_keys.size
82
+ v1_sorted_keys <=> v2_sorted_keys
83
+ else
84
+ v1_sorted_keys.size <=> v2_sorted_keys.size
85
+ end
86
+ else
87
+ v1_sorted_keys.each do |k|
88
+ a = v1[k]
89
+ b = v2[k]
90
+ if a && b && a != b
91
+ return a <=> b
92
+ elsif a && b || (!a && !b) # same value (including both are nil)
93
+ next
94
+ elsif a # b is nil
95
+ return 1
96
+ else # a is nil (but b is non-nil)
97
+ return -1
98
+ end
99
+ end
100
+
101
+ 0
102
+ end
103
+ end
104
+
105
+ def <=>(o)
106
+ timekey2 = o.timekey
107
+ tag2 = o.tag
108
+ variables2 = o.variables
109
+ if (!!timekey ^ !!timekey2) || (!!tag ^ !!tag2) || (!!variables ^ !!variables2)
110
+ # One has value in a field, but another doesn't have value in same field
111
+ # This case occurs very rarely
112
+ if timekey == timekey2 # including the case of nil == nil
113
+ if tag == tag2
114
+ cmp_variables(variables, variables2)
115
+ elsif tag.nil?
116
+ -1
117
+ elsif tag2.nil?
118
+ 1
119
+ else
120
+ tag <=> tag2
121
+ end
122
+ elsif timekey.nil?
123
+ -1
124
+ elsif timekey2.nil?
125
+ 1
126
+ else
127
+ timekey <=> timekey2
128
+ end
129
+ else
130
+ # objects have values in same field pairs (comparison with non-nil and nil doesn't occur here)
131
+ (timekey <=> timekey2 || 0).nonzero? || # if `a <=> b` is nil, then both are nil
132
+ (tag <=> tag2 || 0).nonzero? ||
133
+ cmp_variables(variables, variables2)
134
+ end
135
+ end
65
136
  end
66
137
 
67
138
  # for tests
@@ -80,6 +151,7 @@ module Fluent
80
151
  @queue = [] #=> Array (chunks) : already flushed (not written)
81
152
  @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged)
82
153
  @queued_num = {} # metadata => int (number of queued chunks)
154
+ @dequeued_num = {} # metadata => int (number of dequeued chunks)
83
155
 
84
156
  @stage_size = @queue_size = 0
85
157
  @metadata_list = [] # keys of @stage
@@ -172,7 +244,8 @@ module Fluent
172
244
  end
173
245
 
174
246
  def add_metadata(metadata)
175
- log.trace "adding metadata", instance: self.object_id, metadata: metadata
247
+ log.on_trace { log.trace "adding metadata", instance: self.object_id, metadata: metadata }
248
+
176
249
  synchronize do
177
250
  if i = @metadata_list.index(metadata)
178
251
  @metadata_list[i]
@@ -195,7 +268,7 @@ module Fluent
195
268
  return if metadata_and_data.size < 1
196
269
  raise BufferOverflowError, "buffer space has too many data" unless storable?
197
270
 
198
- log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size
271
+ log.on_trace { log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size }
199
272
 
200
273
  staged_bytesize = 0
201
274
  operated_chunks = []
@@ -203,7 +276,9 @@ module Fluent
203
276
  chunks_to_enqueue = []
204
277
 
205
278
  begin
206
- metadata_and_data.each do |metadata, data|
279
+ # sort metadata to get lock of chunks in same order with other threads
280
+ metadata_and_data.keys.sort.each do |metadata|
281
+ data = metadata_and_data[metadata]
207
282
  write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
208
283
  chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
209
284
  operated_chunks << chunk
@@ -296,6 +371,10 @@ module Fluent
296
371
  end
297
372
  end
298
373
 
374
+ def queue_full?
375
+ synchronize { @queue.size } >= @queued_chunks_limit_size
376
+ end
377
+
299
378
  def queued_records
300
379
  synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
301
380
  end
@@ -312,60 +391,70 @@ module Fluent
312
391
  end
313
392
 
314
393
  def enqueue_chunk(metadata)
315
- log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata
316
- synchronize do
317
- chunk = @stage.delete(metadata)
318
- return nil unless chunk
394
+ log.on_trace { log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata }
319
395
 
320
- chunk.synchronize do
396
+ chunk = synchronize do
397
+ @stage.delete(metadata)
398
+ end
399
+ return nil unless chunk
400
+
401
+ chunk.synchronize do
402
+ synchronize do
321
403
  if chunk.empty?
322
404
  chunk.close
323
405
  else
324
406
  @queue << chunk
325
407
  @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
326
- chunk.enqueued! if chunk.respond_to?(:enqueued!)
408
+ chunk.enqueued!
327
409
  end
410
+ bytesize = chunk.bytesize
411
+ @stage_size -= bytesize
412
+ @queue_size += bytesize
328
413
  end
329
- bytesize = chunk.bytesize
330
- @stage_size -= bytesize
331
- @queue_size += bytesize
332
414
  end
333
415
  nil
334
416
  end
335
417
 
336
418
  def enqueue_unstaged_chunk(chunk)
337
- log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
419
+ log.on_trace { log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata }
420
+
338
421
  synchronize do
339
422
  chunk.synchronize do
340
423
  metadata = chunk.metadata
341
424
  @queue << chunk
342
425
  @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
343
- chunk.enqueued! if chunk.respond_to?(:enqueued!)
426
+ chunk.enqueued!
344
427
  end
345
428
  @queue_size += chunk.bytesize
346
429
  end
347
430
  end
348
431
 
349
- def enqueue_all
350
- log.trace "enqueueing all chunks in buffer", instance: self.object_id
351
- synchronize do
352
- if block_given?
353
- @stage.keys.each do |metadata|
354
- chunk = @stage[metadata]
355
- v = yield metadata, chunk
356
- enqueue_chunk(metadata) if v
357
- end
358
- else
359
- @stage.keys.each do |metadata|
360
- enqueue_chunk(metadata)
361
- end
432
+ # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
433
+ def enqueue_all(force_enqueue = false)
434
+ log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
435
+
436
+ if block_given?
437
+ synchronize{ @stage.keys }.each do |metadata|
438
+ return if !force_enqueue && queue_full?
439
+ # NOTE: The following line might cause data race depending on Ruby implementations except CRuby
440
+ # cf. https://github.com/fluent/fluentd/pull/1721#discussion_r146170251
441
+ chunk = @stage[metadata]
442
+ next unless chunk
443
+ v = yield metadata, chunk
444
+ enqueue_chunk(metadata) if v
445
+ end
446
+ else
447
+ synchronize{ @stage.keys }.each do |metadata|
448
+ return if !force_enqueue && queue_full?
449
+ enqueue_chunk(metadata)
362
450
  end
363
451
  end
364
452
  end
365
453
 
366
454
  def dequeue_chunk
367
455
  return nil if @queue.empty?
368
- log.trace "dequeueing a chunk", instance: self.object_id
456
+ log.on_trace { log.trace "dequeueing a chunk", instance: self.object_id }
457
+
369
458
  synchronize do
370
459
  chunk = @queue.shift
371
460
 
@@ -374,19 +463,23 @@ module Fluent
374
463
 
375
464
  @dequeued[chunk.unique_id] = chunk
376
465
  @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
466
+ @dequeued_num[chunk.metadata] ||= 0
467
+ @dequeued_num[chunk.metadata] += 1
377
468
  log.trace "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
378
469
  chunk
379
470
  end
380
471
  end
381
472
 
382
473
  def takeback_chunk(chunk_id)
383
- log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
474
+ log.on_trace { log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id) }
475
+
384
476
  synchronize do
385
477
  chunk = @dequeued.delete(chunk_id)
386
478
  return false unless chunk # already purged by other thread
387
479
  @queue.unshift(chunk)
388
480
  log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
389
481
  @queued_num[chunk.metadata] += 1 # BUG if nil
482
+ @dequeued_num[chunk.metadata] -= 1
390
483
  end
391
484
  true
392
485
  end
@@ -397,7 +490,8 @@ module Fluent
397
490
  return nil unless chunk # purged by other threads
398
491
 
399
492
  metadata = chunk.metadata
400
- log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
493
+ log.on_trace { log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
494
+
401
495
  begin
402
496
  bytesize = chunk.bytesize
403
497
  chunk.purge
@@ -407,8 +501,11 @@ module Fluent
407
501
  log.error_backtrace
408
502
  end
409
503
 
410
- if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
504
+ @dequeued_num[chunk.metadata] -= 1
505
+ if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?
411
506
  @metadata_list.delete(metadata)
507
+ @queued_num.delete(metadata)
508
+ @dequeued_num.delete(metadata)
412
509
  end
413
510
  log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
414
511
  end
@@ -416,7 +513,8 @@ module Fluent
416
513
  end
417
514
 
418
515
  def clear_queue!
419
- log.trace "clearing queue", instance: self.object_id
516
+ log.on_trace { log.trace "clearing queue", instance: self.object_id }
517
+
420
518
  synchronize do
421
519
  until @queue.empty?
422
520
  begin
@@ -22,6 +22,8 @@ module Fluent
22
22
  module Plugin
23
23
  class Buffer
24
24
  class FileChunk < Chunk
25
+ class FileChunkError < StandardError; end
26
+
25
27
  ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log
26
28
  ### buffer chunk path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log
27
29
  ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta
@@ -244,9 +246,10 @@ module Fluent
244
246
  c: @created_at.to_i,
245
247
  m: (update ? Time.now : @modified_at).to_i,
246
248
  })
249
+ bin = msgpack_packer.pack(data).to_s
247
250
  @meta.seek(0, IO::SEEK_SET)
248
- @meta.truncate(0)
249
- @meta.write(msgpack_packer.pack(data))
251
+ @meta.write(bin)
252
+ @meta.truncate(bin.bytesize)
250
253
  end
251
254
 
252
255
  def file_rename(file, old_path, new_path, callback=nil)
@@ -284,6 +287,7 @@ module Fluent
284
287
  @meta.set_encoding(Encoding::ASCII_8BIT)
285
288
  @meta.sync = true
286
289
  @meta.binmode
290
+ write_metadata(update: false)
287
291
  rescue => e
288
292
  # This case is easier than enqueued!. Just removing pre-create buffer file
289
293
  @chunk.close rescue nil
@@ -307,6 +311,8 @@ module Fluent
307
311
  # staging buffer chunk without metadata is classic buffer chunk file
308
312
  # and it should be enqueued immediately
309
313
  if File.exist?(@meta_path)
314
+ raise FileChunkError, "staged file chunk is empty" if File.size(@path).zero?
315
+
310
316
  @chunk = File.open(@path, 'rb+')
311
317
  @chunk.set_encoding(Encoding::ASCII_8BIT)
312
318
  @chunk.sync = true
@@ -317,7 +323,13 @@ module Fluent
317
323
  @meta.set_encoding(Encoding::ASCII_8BIT)
318
324
  @meta.sync = true
319
325
  @meta.binmode
320
- restore_metadata(@meta.read)
326
+ begin
327
+ restore_metadata(@meta.read)
328
+ rescue => e
329
+ @chunk.close
330
+ @meta.close
331
+ raise FileChunkError, "staged meta file is broken. #{e.message}"
332
+ end
321
333
  @meta.seek(0, IO::SEEK_SET)
322
334
 
323
335
  @state = :staged
@@ -343,6 +355,8 @@ module Fluent
343
355
 
344
356
  def load_existing_enqueued_chunk(path)
345
357
  @path = path
358
+ raise FileChunkError, "enqueued file chunk is empty" if File.size(@path).zero?
359
+
346
360
  @chunk = File.open(@path, 'rb')
347
361
  @chunk.set_encoding(Encoding::ASCII_8BIT)
348
362
  @chunk.binmode
@@ -352,7 +366,12 @@ module Fluent
352
366
 
353
367
  @meta_path = @path + '.meta'
354
368
  if File.readable?(@meta_path)
355
- restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })
369
+ begin
370
+ restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })
371
+ rescue => e
372
+ @chunk.close
373
+ raise FileChunkError, "enqueued meta file is broken. #{e.message}"
374
+ end
356
375
  else
357
376
  restore_metadata_partially(@chunk)
358
377
  end
@@ -60,7 +60,7 @@ module Fluent
60
60
  out = ''
61
61
  loop do
62
62
  gz = Zlib::GzipReader.new(io)
63
- out += gz.read
63
+ out << gz.read
64
64
  unused = gz.unused
65
65
  gz.finish
66
66