fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -30,11 +30,17 @@ module Fluent::Plugin
30
30
 
31
31
  helpers :timer, :thread
32
32
 
33
+ desc 'The address to bind to.'
33
34
  config_param :bind, :string, default: '0.0.0.0'
35
+ desc 'The port to listen to.'
34
36
  config_param :port, :integer, default: 24220
37
+ desc 'The tag with which internal metrics are emitted.'
35
38
  config_param :tag, :string, default: nil
39
+ desc 'Determine the rate to emit internal metrics as events.'
36
40
  config_param :emit_interval, :time, default: 60
41
+ desc 'Determine whether to include the config information.'
37
42
  config_param :include_config, :bool, default: true
43
+ desc 'Determine whether to include the retry information.'
38
44
  config_param :include_retry, :bool, default: true
39
45
 
40
46
  class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
@@ -279,7 +285,7 @@ module Fluent::Plugin
279
285
  MONITOR_INFO = {
280
286
  'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
281
287
  'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
282
- 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size },
288
+ 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
283
289
  'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
284
290
  }
285
291
 
@@ -75,6 +75,8 @@ module Fluent::Plugin
75
75
  config_param :tag, :string
76
76
  desc 'The transport protocol used to receive logs.(udp, tcp)'
77
77
  config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
78
+ desc 'The message frame type.(traditional, octet_count)'
79
+ config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
78
80
 
79
81
  desc 'If true, add source host to event record.'
80
82
  config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
@@ -83,6 +85,8 @@ module Fluent::Plugin
83
85
 
84
86
  desc 'The field name of hostname of sender.'
85
87
  config_param :source_hostname_key, :string, default: nil
88
+ desc 'Try to resolve hostname from IP addresses or not.'
89
+ config_param :resolve_hostname, :bool, default: nil
86
90
  desc 'The field name of source address of sender.'
87
91
  config_param :source_address_key, :string, default: nil
88
92
  desc 'The field name of the priority.'
@@ -116,7 +120,13 @@ module Fluent::Plugin
116
120
  end
117
121
  @source_address_key = @source_host_key
118
122
  end
119
- @resolve_name = !!@source_hostname_key
123
+ if @source_hostname_key
124
+ if @resolve_hostname.nil?
125
+ @resolve_hostname = true
126
+ elsif !@resolve_hostname # user specifies "false" in config
127
+ raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
128
+ end
129
+ end
120
130
 
121
131
  @_event_loop_run_timeout = @blocking_timeout
122
132
  end
@@ -138,24 +148,35 @@ module Fluent::Plugin
138
148
  end
139
149
 
140
150
  def start_udp_server
141
- server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_name) do |data, sock|
151
+ server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_hostname) do |data, sock|
142
152
  message_handler(data.chomp, sock)
143
153
  end
144
154
  end
145
155
 
146
156
  def start_tcp_server
147
- # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
148
- delimiter = "\n"
157
+ octet_count_frame = @frame_type == :octet_count
158
+
159
+ # syslog family adds "\n" to each message when transport is TCP and traditional frame
160
+ delimiter = octet_count_frame ? " " : "\n"
149
161
  delimiter_size = delimiter.size
150
- server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_name) do |conn|
151
- buffer = ""
162
+ server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
152
163
  conn.data do |data|
164
+ buffer = conn.buffer
153
165
  buffer << data
154
166
  pos = 0
155
- while idx = buffer.index(delimiter, pos)
156
- msg = buffer[pos...idx]
157
- pos = idx + delimiter_size
158
- message_handler(msg, conn)
167
+ if octet_count_frame
168
+ while idx = buffer.index(delimiter, pos)
169
+ num = Integer(buffer[pos..idx])
170
+ pos = idx + num
171
+ msg = buffer[idx + 1...pos]
172
+ message_handler(msg, conn)
173
+ end
174
+ else
175
+ while idx = buffer.index(delimiter, pos)
176
+ msg = buffer[pos...idx]
177
+ pos = idx + delimiter_size
178
+ message_handler(msg, conn)
179
+ end
159
180
  end
160
181
  buffer.slice!(0, pos) if pos > 0
161
182
  end
@@ -34,6 +34,16 @@ module Fluent::Plugin
34
34
 
35
35
  helpers :timer, :event_loop, :parser, :compat_parameters
36
36
 
37
+ class WatcherSetupError < StandardError
38
+ def initialize(msg)
39
+ @message = msg
40
+ end
41
+
42
+ def to_s
43
+ @message
44
+ end
45
+ end
46
+
37
47
  FILE_PERMISSION = 0644
38
48
 
39
49
  def initialize
@@ -70,6 +80,8 @@ module Fluent::Plugin
70
80
  config_param :emit_unmatched_lines, :bool, default: false
71
81
  desc 'Enable the additional watch timer.'
72
82
  config_param :enable_watch_timer, :bool, default: true
83
+ desc 'Enable the stat watcher based on inotify.'
84
+ config_param :enable_stat_watcher, :bool, default: true
73
85
  desc 'The encoding after conversion of the input.'
74
86
  config_param :encoding, :string, default: nil
75
87
  desc 'The encoding of the input.'
@@ -87,6 +99,8 @@ module Fluent::Plugin
87
99
 
88
100
  attr_reader :paths
89
101
 
102
+ @@pos_file_paths = {}
103
+
90
104
  def configure(conf)
91
105
  compat_parameters_convert(conf, :parser)
92
106
  parser_config = conf.elements('parse').first
@@ -103,13 +117,23 @@ module Fluent::Plugin
103
117
 
104
118
  super
105
119
 
120
+ if !@enable_watch_timer && !@enable_stat_watcher
121
+ raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
122
+ end
123
+
106
124
  @paths = @path.split(',').map {|path| path.strip }
107
125
  if @paths.empty?
108
126
  raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
109
127
  end
110
128
 
111
129
  # TODO: Use plugin_root_dir and storage plugin to store positions if available
112
- unless @pos_file
130
+ if @pos_file
131
+ if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
132
+ plugin_id_using_this_path = @@pos_file_paths[@pos_file]
133
+ raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
134
+ end
135
+ @@pos_file_paths[@pos_file] = self.plugin_id
136
+ else
113
137
  $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
114
138
  $log.warn "this parameter is highly recommended to save the position to resume tailing."
115
139
  end
@@ -130,6 +154,7 @@ module Fluent::Plugin
130
154
  def configure_tag
131
155
  if @tag.index('*')
132
156
  @tag_prefix, @tag_suffix = @tag.split('*')
157
+ @tag_prefix ||= ''
133
158
  @tag_suffix ||= ''
134
159
  else
135
160
  @tag_prefix = nil
@@ -187,25 +212,28 @@ module Fluent::Plugin
187
212
  date = Time.now
188
213
  paths = []
189
214
 
190
- excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
191
215
  @paths.each { |path|
192
216
  path = date.strftime(path)
193
217
  if path.include?('*')
194
218
  paths += Dir.glob(path).select { |p|
195
- is_file = !File.directory?(p)
196
- if File.readable?(p) && is_file
197
- if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
198
- false
219
+ begin
220
+ is_file = !File.directory?(p)
221
+ if File.readable?(p) && is_file
222
+ if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
223
+ false
224
+ else
225
+ true
226
+ end
199
227
  else
200
- true
201
- end
202
- else
203
- if is_file
204
- unless @ignore_list.include?(path)
205
- log.warn "#{p} unreadable. It is excluded and would be examined next time."
206
- @ignore_list << path if @ignore_repeated_permission_error
228
+ if is_file
229
+ unless @ignore_list.include?(path)
230
+ log.warn "#{p} unreadable. It is excluded and would be examined next time."
231
+ @ignore_list << path if @ignore_repeated_permission_error
232
+ end
207
233
  end
234
+ false
208
235
  end
236
+ rescue Errno::ENOENT
209
237
  false
210
238
  end
211
239
  }
@@ -214,6 +242,7 @@ module Fluent::Plugin
214
242
  paths << path
215
243
  end
216
244
  }
245
+ excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
217
246
  paths - excluded
218
247
  end
219
248
 
@@ -235,12 +264,21 @@ module Fluent::Plugin
235
264
 
236
265
  def setup_watcher(path, pe)
237
266
  line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
238
- tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
267
+ tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
239
268
  tw.attach do |watcher|
240
- watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
241
- event_loop_attach(watcher.stat_trigger)
269
+ event_loop_attach(watcher.timer_trigger) if watcher.timer_trigger
270
+ event_loop_attach(watcher.stat_trigger) if watcher.stat_trigger
242
271
  end
243
272
  tw
273
+ rescue => e
274
+ if tw
275
+ tw.detach { |watcher|
276
+ event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
277
+ event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
278
+ }
279
+ tw.close
280
+ end
281
+ raise e
244
282
  end
245
283
 
246
284
  def start_watchers(paths)
@@ -257,7 +295,13 @@ module Fluent::Plugin
257
295
  end
258
296
  end
259
297
 
260
- @tails[path] = setup_watcher(path, pe)
298
+ begin
299
+ tw = setup_watcher(path, pe)
300
+ rescue WatcherSetupError => e
301
+ log.warn "Skip #{path} because unexpected setup error happens: #{e}"
302
+ next
303
+ end
304
+ @tails[path] = tw
261
305
  }
262
306
  end
263
307
 
@@ -302,7 +346,10 @@ module Fluent::Plugin
302
346
  # so adding close_io argument to avoid this problem.
303
347
  # At shutdown, IOHandler's io will be released automatically after detached the event loop
304
348
  def detach_watcher(tw, close_io = true)
305
- tw.detach
349
+ tw.detach { |watcher|
350
+ event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
351
+ event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
352
+ }
306
353
  tw.close if close_io
307
354
  flush_buffer(tw)
308
355
  if tw.unwatched && @pf
@@ -311,6 +358,8 @@ module Fluent::Plugin
311
358
  end
312
359
 
313
360
  def detach_watcher_after_rotate_wait(tw)
361
+ # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
362
+ # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
314
363
  timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
315
364
  detach_watcher(tw)
316
365
  end
@@ -426,18 +475,19 @@ module Fluent::Plugin
426
475
  end
427
476
 
428
477
  class TailWatcher
429
- def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
478
+ def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
430
479
  @path = path
431
480
  @rotate_wait = rotate_wait
432
481
  @pe = pe || MemoryPositionEntry.new
433
482
  @read_from_head = read_from_head
434
483
  @enable_watch_timer = enable_watch_timer
484
+ @enable_stat_watcher = enable_stat_watcher
435
485
  @read_lines_limit = read_lines_limit
436
486
  @receive_lines = receive_lines
437
487
  @update_watcher = update_watcher
438
488
 
439
- @stat_trigger = StatWatcher.new(self, &method(:on_notify))
440
- @timer_trigger = nil
489
+ @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
490
+ @timer_trigger = @enable_watch_timer ? TimerTrigger.new(1, log, &method(:on_notify)) : nil
441
491
 
442
492
  @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
443
493
  @io_handler = nil
@@ -452,7 +502,7 @@ module Fluent::Plugin
452
502
  attr_reader :path
453
503
  attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
454
504
  attr_reader :from_encoding, :encoding
455
- attr_reader :stat_trigger, :enable_watch_timer
505
+ attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
456
506
  attr_accessor :timer_trigger
457
507
  attr_accessor :line_buffer, :line_buffer_timer_flusher
458
508
  attr_accessor :unwatched # This is used for removing position entry from PositionFile
@@ -471,8 +521,7 @@ module Fluent::Plugin
471
521
  end
472
522
 
473
523
  def detach
474
- @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
475
- @stat_trigger.detach if @stat_trigger.attached?
524
+ yield self
476
525
  @io_handler.on_notify if @io_handler
477
526
  end
478
527
 
@@ -509,7 +558,10 @@ module Fluent::Plugin
509
558
  # assuming following situation:
510
559
  # a) file was once renamed and backed, or
511
560
  # b) symlink or hardlink to the same file is recreated
512
- # in either case, seek to the saved position
561
+ # in either case of a and b, seek to the saved position
562
+ # c) file was once renamed, truncated and then backed
563
+ # in this case, consider it truncated
564
+ @pe.update(inode, 0) if fsize < @pe.read_pos
513
565
  elsif last_inode != 0
514
566
  # this is FilePositionEntry and fluentd once started.
515
567
  # read data from the head of the rotated file.
@@ -539,6 +591,8 @@ module Fluent::Plugin
539
591
  @pe.update(inode, 0)
540
592
  else # file is rotated and new file found
541
593
  watcher_needs_update = true
594
+ # Handle the old log file before renewing TailWatcher [fluentd#1055]
595
+ @io_handler.on_notify
542
596
  end
543
597
  else # file is rotated and new file not found
544
598
  # Clear RotateHandler to avoid duplicated file watch in same path.
@@ -566,6 +620,21 @@ module Fluent::Plugin
566
620
  pe # This pe will be updated in on_rotate after TailWatcher is initialized
567
621
  end
568
622
 
623
+ class TimerTrigger < Coolio::TimerWatcher
624
+ def initialize(interval, log, &callback)
625
+ @callback = callback
626
+ @log = log
627
+ super(interval, true)
628
+ end
629
+
630
+ def on_timer
631
+ @callback.call
632
+ rescue => e
633
+ @log.error e.to_s
634
+ @log.error_backtrace
635
+ end
636
+ end
637
+
569
638
  class StatWatcher < Coolio::StatWatcher
570
639
  def initialize(watcher, &callback)
571
640
  @watcher = watcher
@@ -582,7 +651,6 @@ module Fluent::Plugin
582
651
  end
583
652
  end
584
653
 
585
-
586
654
  class FIFO
587
655
  def initialize(from_encoding, encoding)
588
656
  @from_encoding = from_encoding
@@ -637,10 +705,15 @@ module Fluent::Plugin
637
705
  @iobuf = ''.force_encoding('ASCII-8BIT')
638
706
  @lines = []
639
707
  @io = nil
708
+ @notify_mutex = Mutex.new
640
709
  @watcher.log.info "following tail of #{@watcher.path}"
641
710
  end
642
711
 
643
712
  def on_notify
713
+ @notify_mutex.synchronize { handle_notify }
714
+ end
715
+
716
+ def handle_notify
644
717
  with_io do |io|
645
718
  begin
646
719
  read_more = false
@@ -689,6 +762,9 @@ module Fluent::Plugin
689
762
  io = Fluent::FileWrapper.open(@watcher.path)
690
763
  io.seek(@watcher.pe.read_pos + @fifo.bytesize)
691
764
  io
765
+ rescue RangeError
766
+ io.close if io
767
+ raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
692
768
  rescue Errno::ENOENT
693
769
  nil
694
770
  end
@@ -706,6 +782,9 @@ module Fluent::Plugin
706
782
  @io ||= open
707
783
  yield @io
708
784
  end
785
+ rescue WatcherSetupError => e
786
+ close
787
+ raise e
709
788
  rescue
710
789
  @watcher.log.error $!.to_s
711
790
  @watcher.log.error_backtrace
@@ -790,8 +869,9 @@ module Fluent::Plugin
790
869
  class PositionFile
791
870
  UNWATCHED_POSITION = 0xffffffffffffffff
792
871
 
793
- def initialize(file, map, last_pos)
872
+ def initialize(file, file_mutex, map, last_pos)
794
873
  @file = file
874
+ @file_mutex = file_mutex
795
875
  @map = map
796
876
  @last_pos = last_pos
797
877
  end
@@ -801,29 +881,34 @@ module Fluent::Plugin
801
881
  return m
802
882
  end
803
883
 
804
- @file.pos = @last_pos
805
- @file.write path
806
- @file.write "\t"
807
- seek = @file.pos
808
- @file.write "0000000000000000\t0000000000000000\n"
809
- @last_pos = @file.pos
810
-
811
- @map[path] = FilePositionEntry.new(@file, seek)
884
+ @file_mutex.synchronize {
885
+ @file.pos = @last_pos
886
+ @file.write "#{path}\t0000000000000000\t0000000000000000\n"
887
+ seek = @last_pos + path.bytesize + 1
888
+ @last_pos = @file.pos
889
+ @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
890
+ }
812
891
  end
813
892
 
814
893
  def self.parse(file)
815
894
  compact(file)
816
895
 
896
+ file_mutex = Mutex.new
817
897
  map = {}
818
898
  file.pos = 0
819
899
  file.each_line {|line|
820
900
  m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
821
- next unless m
901
+ unless m
902
+ $log.warn "Unparsable line in pos_file: #{line}"
903
+ next
904
+ end
822
905
  path = m[1]
906
+ pos = m[2].to_i(16)
907
+ ino = m[3].to_i(16)
823
908
  seek = file.pos - line.bytesize + path.bytesize + 1
824
- map[path] = FilePositionEntry.new(file, seek)
909
+ map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
825
910
  }
826
- new(file, map, file.pos)
911
+ new(file, file_mutex, map, file.pos)
827
912
  end
828
913
 
829
914
  # Clean up unwatched file entries
@@ -831,7 +916,10 @@ module Fluent::Plugin
831
916
  file.pos = 0
832
917
  existent_entries = file.each_line.map { |line|
833
918
  m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
834
- next unless m
919
+ unless m
920
+ $log.warn "Unparsable line in pos_file: #{line}"
921
+ next
922
+ end
835
923
  path = m[1]
836
924
  pos = m[2].to_i(16)
837
925
  ino = m[3].to_i(16)
@@ -854,36 +942,37 @@ module Fluent::Plugin
854
942
  LN_OFFSET = 33
855
943
  SIZE = 34
856
944
 
857
- def initialize(file, seek)
945
+ def initialize(file, file_mutex, seek, pos, inode)
858
946
  @file = file
947
+ @file_mutex = file_mutex
859
948
  @seek = seek
860
- @pos = nil
949
+ @pos = pos
950
+ @inode = inode
861
951
  end
862
952
 
863
953
  def update(ino, pos)
864
- @file.pos = @seek
865
- @file.write "%016x\t%016x" % [pos, ino]
954
+ @file_mutex.synchronize {
955
+ @file.pos = @seek
956
+ @file.write "%016x\t%016x" % [pos, ino]
957
+ }
866
958
  @pos = pos
959
+ @inode = ino
867
960
  end
868
961
 
869
962
  def update_pos(pos)
870
- @file.pos = @seek
871
- @file.write "%016x" % pos
963
+ @file_mutex.synchronize {
964
+ @file.pos = @seek
965
+ @file.write "%016x" % pos
966
+ }
872
967
  @pos = pos
873
968
  end
874
969
 
875
970
  def read_inode
876
- @file.pos = @seek + INO_OFFSET
877
- raw = @file.read(16)
878
- raw ? raw.to_i(16) : 0
971
+ @inode
879
972
  end
880
973
 
881
974
  def read_pos
882
- @pos ||= begin
883
- @file.pos = @seek
884
- raw = @file.read(16)
885
- raw ? raw.to_i(16) : 0
886
- end
975
+ @pos
887
976
  end
888
977
  end
889
978