fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -71,7 +71,7 @@ module Fluent::Plugin
71
71
  end
72
72
 
73
73
  def write(chunk)
74
- path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
74
+ path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
75
75
  path = generate_path(path_without_suffix)
76
76
  FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
77
77
 
@@ -106,7 +106,7 @@ module Fluent::Plugin
106
106
  raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
107
107
  end
108
108
 
109
- vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) }
109
+ vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
110
110
 
111
111
  if ph = vars.find { |v| !@chunk_keys.include?(v) }
112
112
  raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
@@ -24,7 +24,6 @@ module Fluent::Plugin
24
24
 
25
25
  DEFAULT_LINE_FORMAT_TYPE = 'stdout'
26
26
  DEFAULT_FORMAT_TYPE = 'json'
27
- TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
28
27
 
29
28
  config_section :buffer do
30
29
  config_set_default :chunk_keys, ['tag']
@@ -93,7 +93,7 @@ module Fluent
93
93
  def initialize
94
94
  super
95
95
  $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
96
- $log.warn "see 'forward' section in http://docs.fluentd.org/ for the high-availability configuration."
96
+ $log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
97
97
  end
98
98
 
99
99
  config_param :port, :integer, default: LISTEN_PORT
@@ -14,7 +14,9 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/error'
17
18
  require 'fluent/plugin/base'
19
+ require 'fluent/plugin_helper/record_accessor'
18
20
  require 'fluent/log'
19
21
  require 'fluent/plugin_id'
20
22
  require 'fluent/plugin_helper'
@@ -36,8 +38,9 @@ module Fluent
36
38
  helpers_internal :thread, :retry_state
37
39
 
38
40
  CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
39
- CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
40
- CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
41
+ CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
42
+ CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
43
+ CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
41
44
 
42
45
  CHUNKING_FIELD_WARN_NUM = 4
43
46
 
@@ -89,10 +92,11 @@ module Fluent
89
92
  # k: times
90
93
  # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
91
94
  config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
92
- config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponencial backoff for retries.'
93
- config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponencial backoff between retries while failing.'
95
+ config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
96
+ config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
94
97
 
95
98
  config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
99
+ config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
96
100
  end
97
101
 
98
102
  config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
@@ -150,7 +154,7 @@ module Fluent
150
154
  end
151
155
 
152
156
  # Internal states
153
- FlushThreadState = Struct.new(:thread, :next_clock)
157
+ FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)
154
158
  DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
155
159
  def expired?
156
160
  time + timeout < Time.now
@@ -161,7 +165,7 @@ module Fluent
161
165
  attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
162
166
 
163
167
  # for tests
164
- attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag
168
+ attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
165
169
  attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
166
170
  # output_enqueue_thread_waiting: for test of output.rb itself
167
171
  attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
@@ -203,7 +207,7 @@ module Fluent
203
207
  @output_flush_threads = nil
204
208
 
205
209
  @simple_chunking = nil
206
- @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
210
+ @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
207
211
  @flush_mode = nil
208
212
  @timekey_zone = nil
209
213
 
@@ -222,9 +226,9 @@ module Fluent
222
226
  end
223
227
  self.context_router = primary.context_router
224
228
 
225
- (class << self; self; end).module_eval do
229
+ singleton_class.module_eval do
226
230
  define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
227
- define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) }
231
+ define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) }
228
232
  end
229
233
  end
230
234
 
@@ -276,14 +280,35 @@ module Fluent
276
280
  @chunk_keys = @buffer_config.chunk_keys.dup
277
281
  @chunk_key_time = !!@chunk_keys.delete('time')
278
282
  @chunk_key_tag = !!@chunk_keys.delete('tag')
279
- if @chunk_keys.any?{ |key| key !~ CHUNK_KEY_PATTERN }
283
+ if @chunk_keys.any? { |key|
284
+ begin
285
+ k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)
286
+ if k.is_a?(String)
287
+ k !~ CHUNK_KEY_PATTERN
288
+ else
289
+ if key.start_with?('$[')
290
+ raise Fluent::ConfigError, "in chunk_keys: bracket notation is not allowed"
291
+ else
292
+ false
293
+ end
294
+ end
295
+ rescue => e
296
+ raise Fluent::ConfigError, "in chunk_keys: #{e.message}"
297
+ end
298
+ }
280
299
  raise Fluent::ConfigError, "chunk_keys specification includes invalid char"
300
+ else
301
+ @chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]
281
302
  end
282
303
 
283
304
  if @chunk_key_time
284
305
  raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
285
306
  Fluent::Timezone.validate!(@buffer_config.timekey_zone)
286
307
  @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
308
+ @timekey = @buffer_config.timekey
309
+ @timekey_use_utc = @buffer_config.timekey_use_utc
310
+ @offset = Fluent::Timezone.utc_offset(@timekey_zone)
311
+ @calculate_offset = @offset.respond_to?(:call) ? @offset : nil
287
312
  @output_time_formatter_cache = {}
288
313
  end
289
314
 
@@ -329,6 +354,10 @@ module Fluent
329
354
  log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
330
355
  end
331
356
  end
357
+
358
+ if @buffer.queued_chunks_limit_size.nil?
359
+ @buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count
360
+ end
332
361
  end
333
362
 
334
363
  if @secondary_config
@@ -343,6 +372,9 @@ module Fluent
343
372
  end
344
373
  secondary_conf = conf.elements(name: 'secondary').first
345
374
  @secondary = Plugin.new_output(secondary_type)
375
+ unless @secondary.respond_to?(:acts_as_secondary)
376
+ raise Fluent::ConfigError, "Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output"
377
+ end
346
378
  @secondary.acts_as_secondary(self)
347
379
  @secondary.configure(secondary_conf)
348
380
  if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
@@ -370,7 +402,7 @@ module Fluent
370
402
 
371
403
  if @buffering
372
404
  m = method(:emit_buffered)
373
- (class << self; self; end).module_eval do
405
+ singleton_class.module_eval do
374
406
  define_method(:emit_events, m)
375
407
  end
376
408
 
@@ -384,7 +416,7 @@ module Fluent
384
416
  @delayed_commit_timeout = @buffer_config.delayed_commit_timeout
385
417
  else # !@buffering
386
418
  m = method(:emit_sync)
387
- (class << self; self; end).module_eval do
419
+ singleton_class.module_eval do
388
420
  define_method(:emit_events, m)
389
421
  end
390
422
  end
@@ -415,7 +447,7 @@ module Fluent
415
447
 
416
448
  @buffer_config.flush_thread_count.times do |i|
417
449
  thread_title = "flush_thread_#{i}".to_sym
418
- thread_state = FlushThreadState.new(nil, nil)
450
+ thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
419
451
  thread = thread_create(thread_title) do
420
452
  flush_thread_run(thread_state)
421
453
  end
@@ -481,9 +513,14 @@ module Fluent
481
513
  @output_flush_threads_running = false
482
514
  if @output_flush_threads && !@output_flush_threads.empty?
483
515
  @output_flush_threads.each do |state|
484
- state.thread.run if state.thread.alive? # to wakeup thread and make it to stop by itself
485
- end
486
- @output_flush_threads.each do |state|
516
+ # to wakeup thread and make it to stop by itself
517
+ state.mutex.synchronize {
518
+ if state.thread && state.thread.status
519
+ state.next_clock = 0
520
+ state.cond_var.signal
521
+ end
522
+ }
523
+ Thread.pass
487
524
  state.thread.join
488
525
  end
489
526
  end
@@ -653,7 +690,7 @@ module Fluent
653
690
  str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
654
691
  if ph == "tag"
655
692
  parts << -1
656
- elsif ph =~ /^tag\[(\d+)\]$/
693
+ elsif ph =~ /^tag\[(-?\d+)\]$/
657
694
  parts << $1.to_i
658
695
  end
659
696
  end
@@ -661,13 +698,27 @@ module Fluent
661
698
  end
662
699
 
663
700
  def get_placeholders_keys(str)
664
- str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| s == "tag"}.sort
701
+ str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
665
702
  end
666
703
 
667
704
  # TODO: optimize this code
668
- def extract_placeholders(str, metadata)
705
+ def extract_placeholders(str, chunk)
706
+ metadata = if chunk.is_a?(Fluent::Plugin::Buffer::Chunk)
707
+ chunk_passed = true
708
+ chunk.metadata
709
+ else
710
+ chunk_passed = false
711
+ # For existing plugins. Old plugin passes Chunk.metadata instead of Chunk
712
+ chunk
713
+ end
669
714
  if metadata.empty?
670
- str
715
+ str.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
716
+ if chunk_passed
717
+ dump_unique_id_hex(chunk.unique_id)
718
+ else
719
+ log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
720
+ end
721
+ }
671
722
  else
672
723
  rvalue = str.dup
673
724
  # strftime formatting
@@ -675,15 +726,17 @@ module Fluent
675
726
  @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
676
727
  rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
677
728
  end
678
- # ${tag}, ${tag[0]}, ${tag[1]}, ...
729
+ # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
679
730
  if @chunk_key_tag
680
731
  if str.include?('${tag}')
681
732
  rvalue = rvalue.gsub('${tag}', metadata.tag)
682
733
  end
683
734
  if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
684
735
  hash = {}
685
- metadata.tag.split('.').each_with_index do |part, i|
736
+ tag_parts = metadata.tag.split('.')
737
+ tag_parts.each_with_index do |part, i|
686
738
  hash["${tag[#{i}]}"] = part
739
+ hash["${tag[#{i-tag_parts.size}]}"] = part
687
740
  end
688
741
  rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
689
742
  end
@@ -700,9 +753,15 @@ module Fluent
700
753
  rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
701
754
  end
702
755
  if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
703
- log.warn "chunk key placeholder '#{$1}' not replaced. templace:#{str}"
756
+ log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
704
757
  end
705
- rvalue
758
+ rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
759
+ if chunk_passed
760
+ dump_unique_id_hex(chunk.unique_id)
761
+ else
762
+ log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
763
+ end
764
+ }
706
765
  end
707
766
  end
708
767
 
@@ -761,28 +820,42 @@ module Fluent
761
820
  if !@chunk_key_time && !@chunk_key_tag
762
821
  @buffer.metadata()
763
822
  elsif @chunk_key_time && @chunk_key_tag
764
- time_int = time.to_i
765
- timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
823
+ timekey = calculate_timekey(time)
766
824
  @buffer.metadata(timekey: timekey, tag: tag)
767
825
  elsif @chunk_key_time
768
- time_int = time.to_i
769
- timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
826
+ timekey = calculate_timekey(time)
770
827
  @buffer.metadata(timekey: timekey)
771
828
  else
772
829
  @buffer.metadata(tag: tag)
773
830
  end
774
831
  else
775
832
  timekey = if @chunk_key_time
776
- time_int = time.to_i
777
- (time_int - (time_int % @buffer_config.timekey)).to_i
833
+ calculate_timekey(time)
778
834
  else
779
835
  nil
780
836
  end
781
- pairs = Hash[@chunk_keys.map{|k| [k.to_sym, record[k]]}]
837
+ pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
782
838
  @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
783
839
  end
784
840
  end
785
841
 
842
+ def calculate_timekey(time)
843
+ time_int = time.to_i
844
+ if @timekey_use_utc
845
+ (time_int - (time_int % @timekey)).to_i
846
+ else
847
+ offset = @calculate_offset ? @calculate_offset.call(time) : @offset
848
+ (time_int - ((time_int + offset)% @timekey)).to_i
849
+ end
850
+ end
851
+
852
+ def chunk_for_test(tag, time, record)
853
+ require 'fluent/plugin/buffer/memory_chunk'
854
+
855
+ m = metadata_for_test(tag, time, record)
856
+ Fluent::Plugin::Buffer::MemoryChunk.new(m)
857
+ end
858
+
786
859
  def metadata_for_test(tag, time, record)
787
860
  raise "BUG: #metadata_for_test is available only when no actual metadata exists" unless @buffer.metadata_list.empty?
788
861
  m = metadata(tag, time, record)
@@ -825,7 +898,7 @@ module Fluent
825
898
  begin
826
899
  oldest = @buffer.dequeue_chunk
827
900
  if oldest
828
- log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: oldest.unique_id
901
+ log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
829
902
  @buffer.purge_chunk(oldest.unique_id)
830
903
  else
831
904
  log.error "no queued chunks to be dropped for drop_oldest_chunk"
@@ -925,7 +998,8 @@ module Fluent
925
998
  end
926
999
 
927
1000
  def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
928
- log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed
1001
+ log.on_trace { log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }
1002
+
929
1003
  if delayed
930
1004
  @dequeued_chunks_mutex.synchronize do
931
1005
  @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
@@ -945,7 +1019,9 @@ module Fluent
945
1019
  end
946
1020
  end
947
1021
 
948
- def rollback_write(chunk_id)
1022
+ # update_retry parameter is for preventing busy loop by async write
1023
+ # We will remove this parameter by re-design retry_state management between threads.
1024
+ def rollback_write(chunk_id, update_retry: true)
949
1025
  # This API is to rollback chunks explicitly from plugins.
950
1026
  # 3rd party plugins can depend it on automatic rollback of #try_rollback_write
951
1027
  @dequeued_chunks_mutex.synchronize do
@@ -956,8 +1032,10 @@ module Fluent
956
1032
  # in many cases, false can be just ignored
957
1033
  if @buffer.takeback_chunk(chunk_id)
958
1034
  @counters_monitor.synchronize{ @rollback_count += 1 }
959
- primary = @as_secondary ? @primary_instance : self
960
- primary.update_retry_state(chunk_id, @as_secondary)
1035
+ if update_retry
1036
+ primary = @as_secondary ? @primary_instance : self
1037
+ primary.update_retry_state(chunk_id, @as_secondary)
1038
+ end
961
1039
  true
962
1040
  else
963
1041
  false
@@ -1003,11 +1081,13 @@ module Fluent
1003
1081
  end
1004
1082
  end
1005
1083
 
1084
+ UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError]
1085
+
1006
1086
  def try_flush
1007
1087
  chunk = @buffer.dequeue_chunk
1008
1088
  return unless chunk
1009
1089
 
1010
- log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
1090
+ log.on_trace { log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id) }
1011
1091
 
1012
1092
  output = self
1013
1093
  using_secondary = false
@@ -1047,6 +1127,43 @@ module Fluent
1047
1127
  commit_write(chunk_id, delayed: false, secondary: using_secondary)
1048
1128
  log.trace "done to commit a chunk", chunk: dump_chunk_id
1049
1129
  end
1130
+ rescue *UNRECOVERABLE_ERRORS => e
1131
+ if @secondary
1132
+ if using_secondary
1133
+ log.warn "got unrecoverable error in secondary.", error: e
1134
+ log.warn_backtrace
1135
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1136
+ else
1137
+ if (self.class == @secondary.class)
1138
+ log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
1139
+ log.warn_backtrace
1140
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1141
+ else
1142
+ # Call secondary output directly without retry update.
1143
+ # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
1144
+ if @secondary.delayed_commit
1145
+ log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
1146
+ log.warn_backtrace
1147
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1148
+ else
1149
+ log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
1150
+ log.warn_backtrace
1151
+ begin
1152
+ @secondary.write(chunk)
1153
+ commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
1154
+ rescue => e
1155
+ log.warn "got an error in secondary for unrecoverable error", error: e
1156
+ log.warn_backtrace
1157
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1158
+ end
1159
+ end
1160
+ end
1161
+ end
1162
+ else
1163
+ log.warn "got unrecoverable error in primary and no secondary", error: e
1164
+ log.warn_backtrace
1165
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1166
+ end
1050
1167
  rescue => e
1051
1168
  log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
1052
1169
  if output.delayed_commit
@@ -1062,6 +1179,25 @@ module Fluent
1062
1179
  end
1063
1180
  end
1064
1181
 
1182
+ def backup_chunk(chunk, using_secondary, delayed_commit)
1183
+ if @buffer_config.disable_chunk_backup
1184
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
1185
+ else
1186
+ unique_id = dump_unique_id_hex(chunk.unique_id)
1187
+ safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
1188
+ backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
1189
+ backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
1190
+ backup_dir = File.dirname(backup_file)
1191
+
1192
+ log.warn "bad chunk is moved to #{backup_file}"
1193
+ FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
1194
+ File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
1195
+ chunk.write_to(f)
1196
+ }
1197
+ end
1198
+ commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
1199
+ end
1200
+
1065
1201
  def check_slow_flush(start)
1066
1202
  elapsed_time = Fluent::Clock.now - start
1067
1203
  if elapsed_time > @slow_flush_log_threshold
@@ -1136,17 +1272,20 @@ module Fluent
1136
1272
  # Without locks: it is rough but enough to select "next" writer selection
1137
1273
  @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
1138
1274
  state = @output_flush_threads[@output_flush_thread_current_position]
1139
- state.next_clock = 0
1140
- if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
1141
- state.thread.run
1142
- else
1143
- log.warn "thread is already dead"
1144
- end
1275
+ state.mutex.synchronize {
1276
+ if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
1277
+ state.next_clock = 0
1278
+ state.cond_var.signal
1279
+ else
1280
+ log.warn "thread is already dead"
1281
+ end
1282
+ }
1283
+ Thread.pass
1145
1284
  end
1146
1285
 
1147
1286
  def force_flush
1148
1287
  if @buffering
1149
- @buffer.enqueue_all
1288
+ @buffer.enqueue_all(true)
1150
1289
  submit_flush_all
1151
1290
  end
1152
1291
  end
@@ -1178,8 +1317,13 @@ module Fluent
1178
1317
  # only for tests of output plugin
1179
1318
  def flush_thread_wakeup
1180
1319
  @output_flush_threads.each do |state|
1181
- state.next_clock = 0
1182
- state.thread.run
1320
+ state.mutex.synchronize {
1321
+ if state.thread && state.thread.status
1322
+ state.next_clock = 0
1323
+ state.cond_var.signal
1324
+ end
1325
+ }
1326
+ Thread.pass
1183
1327
  end
1184
1328
  end
1185
1329
 
@@ -1258,30 +1402,48 @@ module Fluent
1258
1402
  end
1259
1403
  log.debug "flush_thread actually running"
1260
1404
 
1405
+ state.mutex.lock
1261
1406
  begin
1262
1407
  # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
1263
1408
  while @output_flush_threads_running
1264
1409
  current_clock = Fluent::Clock.now
1265
- interval = state.next_clock - current_clock
1410
+ next_retry_time = nil
1266
1411
 
1267
- if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
1268
- try_flush
1412
+ @retry_mutex.synchronize do
1413
+ next_retry_time = @retry ? @retry.next_time : nil
1414
+ end
1269
1415
 
1270
- # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
1271
- interval = next_flush_time.to_f - Time.now.to_f
1272
- # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
1273
- # because @retry still exists (#commit_write is not called yet in #try_flush)
1274
- # @retry should be cleared if delayed commit is enabled? Or any other solution?
1275
- state.next_clock = Fluent::Clock.now + interval
1416
+ if state.next_clock > current_clock
1417
+ interval = state.next_clock - current_clock
1418
+ elsif next_retry_time && next_retry_time > Time.now
1419
+ interval = next_retry_time.to_f - Time.now.to_f
1420
+ else
1421
+ state.mutex.unlock
1422
+ begin
1423
+ try_flush
1424
+ # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
1425
+ interval = next_flush_time.to_f - Time.now.to_f
1426
+ # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
1427
+ # because @retry still exists (#commit_write is not called yet in #try_flush)
1428
+ # @retry should be cleared if delayed commit is enabled? Or any other solution?
1429
+ state.next_clock = Fluent::Clock.now + interval
1430
+ ensure
1431
+ state.mutex.lock
1432
+ end
1276
1433
  end
1277
1434
 
1278
1435
  if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
1279
1436
  unless @output_flush_interrupted
1280
- try_rollback_write
1437
+ state.mutex.unlock
1438
+ begin
1439
+ try_rollback_write
1440
+ ensure
1441
+ state.mutex.lock
1442
+ end
1281
1443
  end
1282
1444
  end
1283
1445
 
1284
- sleep interval if interval > 0
1446
+ state.cond_var.wait(state.mutex, interval) if interval > 0
1285
1447
  end
1286
1448
  rescue => e
1287
1449
  # normal errors are rescued by output plugins in #try_flush
@@ -1289,6 +1451,8 @@ module Fluent
1289
1451
  log.error "error on output thread", error: e
1290
1452
  log.error_backtrace
1291
1453
  raise
1454
+ ensure
1455
+ state.mutex.unlock
1292
1456
  end
1293
1457
  end
1294
1458
  end