fluentd 1.11.1-x64-mingw32 → 1.12.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
  3. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. data/.github/workflows/stale-actions.yml +22 -0
  5. data/.travis.yml +22 -2
  6. data/CHANGELOG.md +111 -0
  7. data/README.md +1 -1
  8. data/appveyor.yml +3 -0
  9. data/bin/fluent-cap-ctl +7 -0
  10. data/bin/fluent-ctl +7 -0
  11. data/example/copy_roundrobin.conf +3 -3
  12. data/example/counter.conf +1 -1
  13. data/example/filter_stdout.conf +2 -2
  14. data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
  15. data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
  16. data/example/logevents.conf +5 -5
  17. data/example/multi_filters.conf +1 -1
  18. data/example/out_exec_filter.conf +2 -2
  19. data/example/out_forward.conf +1 -1
  20. data/example/out_forward_buf_file.conf +1 -1
  21. data/example/out_forward_client.conf +5 -5
  22. data/example/out_forward_heartbeat_none.conf +1 -1
  23. data/example/out_forward_sd.conf +1 -1
  24. data/example/out_forward_shared_key.conf +2 -2
  25. data/example/out_forward_tls.conf +1 -1
  26. data/example/out_forward_users.conf +3 -3
  27. data/example/out_null.conf +4 -4
  28. data/example/secondary_file.conf +1 -1
  29. data/fluentd.gemspec +7 -6
  30. data/lib/fluent/capability.rb +87 -0
  31. data/lib/fluent/command/cap_ctl.rb +174 -0
  32. data/lib/fluent/command/ctl.rb +177 -0
  33. data/lib/fluent/command/plugin_config_formatter.rb +2 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/log.rb +33 -3
  36. data/lib/fluent/match.rb +9 -0
  37. data/lib/fluent/plugin.rb +5 -0
  38. data/lib/fluent/plugin/buffer.rb +32 -42
  39. data/lib/fluent/plugin/buffer/chunk.rb +2 -1
  40. data/lib/fluent/plugin/formatter.rb +24 -0
  41. data/lib/fluent/plugin/formatter_csv.rb +1 -1
  42. data/lib/fluent/plugin/formatter_hash.rb +3 -1
  43. data/lib/fluent/plugin/formatter_json.rb +3 -1
  44. data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
  45. data/lib/fluent/plugin/formatter_out_file.rb +6 -4
  46. data/lib/fluent/plugin/formatter_single_value.rb +4 -2
  47. data/lib/fluent/plugin/formatter_tsv.rb +4 -2
  48. data/lib/fluent/plugin/in_dummy.rb +2 -123
  49. data/lib/fluent/plugin/in_exec.rb +4 -2
  50. data/lib/fluent/plugin/in_http.rb +25 -4
  51. data/lib/fluent/plugin/in_sample.rb +141 -0
  52. data/lib/fluent/plugin/in_tail.rb +109 -41
  53. data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
  54. data/lib/fluent/plugin/in_tcp.rb +1 -0
  55. data/lib/fluent/plugin/out_http.rb +20 -2
  56. data/lib/fluent/plugin/output.rb +15 -6
  57. data/lib/fluent/plugin/parser_json.rb +5 -2
  58. data/lib/fluent/plugin_helper/cert_option.rb +5 -8
  59. data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
  60. data/lib/fluent/plugin_helper/inject.rb +4 -1
  61. data/lib/fluent/plugin_helper/retry_state.rb +4 -0
  62. data/lib/fluent/plugin_helper/socket.rb +1 -1
  63. data/lib/fluent/supervisor.rb +151 -48
  64. data/lib/fluent/system_config.rb +2 -1
  65. data/lib/fluent/time.rb +1 -0
  66. data/lib/fluent/version.rb +1 -1
  67. data/lib/fluent/winsvc.rb +22 -4
  68. data/test/command/test_binlog_reader.rb +22 -6
  69. data/test/command/test_cap_ctl.rb +100 -0
  70. data/test/command/test_ctl.rb +57 -0
  71. data/test/command/test_plugin_config_formatter.rb +57 -2
  72. data/test/plugin/in_tail/test_position_file.rb +45 -25
  73. data/test/plugin/test_buffer.rb +4 -0
  74. data/test/plugin/test_filter_stdout.rb +6 -1
  75. data/test/plugin/test_formatter_hash.rb +6 -3
  76. data/test/plugin/test_formatter_json.rb +14 -4
  77. data/test/plugin/test_formatter_ltsv.rb +13 -5
  78. data/test/plugin/test_formatter_out_file.rb +35 -14
  79. data/test/plugin/test_formatter_single_value.rb +12 -6
  80. data/test/plugin/test_formatter_tsv.rb +12 -4
  81. data/test/plugin/test_in_exec.rb +18 -0
  82. data/test/plugin/test_in_http.rb +25 -0
  83. data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
  84. data/test/plugin/test_in_tail.rb +433 -30
  85. data/test/plugin/test_out_file.rb +23 -18
  86. data/test/plugin/test_output.rb +12 -0
  87. data/test/plugin/test_parser_syslog.rb +2 -2
  88. data/test/plugin_helper/data/cert/empty.pem +0 -0
  89. data/test/plugin_helper/test_cert_option.rb +7 -0
  90. data/test/plugin_helper/test_compat_parameters.rb +7 -2
  91. data/test/plugin_helper/test_http_server_helper.rb +5 -0
  92. data/test/plugin_helper/test_inject.rb +42 -0
  93. data/test/plugin_helper/test_server.rb +34 -0
  94. data/test/plugin_helper/test_socket.rb +8 -0
  95. data/test/test_capability.rb +74 -0
  96. data/test/test_formatter.rb +34 -10
  97. data/test/test_log.rb +44 -0
  98. data/test/test_match.rb +11 -0
  99. data/test/test_output.rb +6 -1
  100. data/test/test_static_config_analysis.rb +2 -2
  101. data/test/test_supervisor.rb +119 -1
  102. metadata +50 -18
@@ -14,126 +14,5 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'json'
18
-
19
- require 'fluent/plugin/input'
20
- require 'fluent/config/error'
21
-
22
- module Fluent::Plugin
23
- class DummyInput < Input
24
- Fluent::Plugin.register_input('dummy', self)
25
-
26
- helpers :thread, :storage
27
-
28
- BIN_NUM = 10
29
- DEFAULT_STORAGE_TYPE = 'local'
30
-
31
- desc "The value is the tag assigned to the generated events."
32
- config_param :tag, :string
33
- desc "The number of events in event stream of each emits."
34
- config_param :size, :integer, default: 1
35
- desc "It configures how many events to generate per second."
36
- config_param :rate, :integer, default: 1
37
- desc "If specified, each generated event has an auto-incremented key field."
38
- config_param :auto_increment_key, :string, default: nil
39
- desc "The boolean to suspend-and-resume incremental value after restart"
40
- config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
41
- desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
42
- config_param :dummy, default: [{"message"=>"dummy"}] do |val|
43
- begin
44
- parsed = JSON.parse(val)
45
- rescue JSON::ParserError => ex
46
- # Fluent::ConfigParseError, "got incomplete JSON" will be raised
47
- # at literal_parser.rb with --use-v1-config, but I had to
48
- # take care at here for the case of --use-v0-config.
49
- raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
50
- end
51
- dummy = parsed.is_a?(Array) ? parsed : [parsed]
52
- dummy.each_with_index do |e, i|
53
- raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
54
- end
55
- dummy
56
- end
57
-
58
- def initialize
59
- super
60
- @storage = nil
61
- end
62
-
63
- def configure(conf)
64
- super
65
- @dummy_index = 0
66
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
- @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
68
- end
69
-
70
- def multi_workers_ready?
71
- true
72
- end
73
-
74
- def start
75
- super
76
-
77
- @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
78
- @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
79
-
80
- if @auto_increment_key && !@storage.get(:auto_increment_value)
81
- @storage.put(:auto_increment_value, -1)
82
- end
83
-
84
- thread_create(:dummy_input, &method(:run))
85
- end
86
-
87
- def run
88
- batch_num = (@rate / BIN_NUM).to_i
89
- residual_num = (@rate % BIN_NUM)
90
- while thread_current_running?
91
- current_time = Time.now.to_i
92
- BIN_NUM.times do
93
- break unless (thread_current_running? && Time.now.to_i <= current_time)
94
- wait(0.1) { emit(batch_num) }
95
- end
96
- emit(residual_num) if thread_current_running?
97
- # wait for next second
98
- while thread_current_running? && Time.now.to_i <= current_time
99
- sleep 0.01
100
- end
101
- end
102
- end
103
-
104
- def emit(num)
105
- begin
106
- if @size > 1
107
- num.times do
108
- router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
109
- end
110
- else
111
- num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
112
- end
113
- rescue => _
114
- # ignore all errors not to stop emits by emit errors
115
- end
116
- end
117
-
118
- def generate
119
- d = @dummy[@dummy_index]
120
- unless d
121
- @dummy_index = 0
122
- d = @dummy[@dummy_index]
123
- end
124
- @dummy_index += 1
125
- if @auto_increment_key
126
- d = d.dup
127
- d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
128
- end
129
- d
130
- end
131
-
132
- def wait(time)
133
- start_time = Time.now
134
- yield
135
- sleep_time = time - (Time.now - start_time)
136
- sleep sleep_time if sleep_time > 0
137
- end
138
- end
139
- end
17
+ # Remove this file in fluentd v2
18
+ require_relative 'in_sample'
@@ -25,6 +25,8 @@ module Fluent::Plugin
25
25
 
26
26
  desc 'The command (program) to execute.'
27
27
  config_param :command, :string
28
+ desc 'Specify connect mode to executed process'
29
+ config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
28
30
 
29
31
  config_section :parse do
30
32
  config_set_default :@type, 'tsv'
@@ -72,9 +74,9 @@ module Fluent::Plugin
72
74
  super
73
75
 
74
76
  if @run_interval
75
- child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
77
+ child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
76
78
  else
77
- child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
79
+ child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
78
80
  end
79
81
  end
80
82
 
@@ -72,7 +72,7 @@ module Fluent::Plugin
72
72
  desc 'Add REMOTE_ADDR header to the record.'
73
73
  config_param :add_remote_addr, :bool, default: false
74
74
  config_param :blocking_timeout, :time, default: 0.5
75
- desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
75
+ desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
76
76
  config_param :cors_allow_origins, :array, default: nil
77
77
  desc 'Respond with empty gif image of 1x1 pixel.'
78
78
  config_param :respond_with_empty_img, :bool, default: false
@@ -80,6 +80,8 @@ module Fluent::Plugin
80
80
  config_param :use_204_response, :bool, default: false
81
81
  desc 'Dump error log or not'
82
82
  config_param :dump_error_log, :bool, default: true
83
+ desc 'Add QUERY_ prefix query params to record'
84
+ config_param :add_query_params, :bool, default: false
83
85
 
84
86
  config_section :parse do
85
87
  config_set_default :@type, 'in_http'
@@ -277,7 +279,7 @@ module Fluent::Plugin
277
279
  private
278
280
 
279
281
  def on_server_connect(conn)
280
- handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
282
+ handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
281
283
 
282
284
  conn.on(:data) do |data|
283
285
  handler.on_read(data)
@@ -326,6 +328,14 @@ module Fluent::Plugin
326
328
  }
327
329
  end
328
330
 
331
+ if @add_query_params
332
+ params.each_pair { |k, v|
333
+ if k.start_with?("QUERY_".freeze)
334
+ record[k] = v
335
+ end
336
+ }
337
+ end
338
+
329
339
  if @add_remote_addr
330
340
  record['REMOTE_ADDR'] = params['REMOTE_ADDR']
331
341
  end
@@ -346,7 +356,7 @@ module Fluent::Plugin
346
356
  class Handler
347
357
  attr_reader :content_type
348
358
 
349
- def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
359
+ def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
350
360
  @io = io
351
361
  @km = km
352
362
  @callback = callback
@@ -356,6 +366,7 @@ module Fluent::Plugin
356
366
  @log = log
357
367
  @cors_allow_origins = cors_allow_origins
358
368
  @idle = 0
369
+ @add_query_params = add_query_params
359
370
  @km.add(self)
360
371
 
361
372
  @remote_port, @remote_addr = io.remote_port, io.remote_addr
@@ -490,7 +501,7 @@ module Fluent::Plugin
490
501
  # CORS check
491
502
  # ==========
492
503
  # For every incoming request, we check if we have some CORS
493
- # restrictions and white listed origins through @cors_allow_origins.
504
+ # restrictions and allow listed origins through @cors_allow_origins.
494
505
  unless @cors_allow_origins.nil?
495
506
  unless @cors_allow_origins.include?('*') or include_cors_allow_origin
496
507
  send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
@@ -533,7 +544,17 @@ module Fluent::Plugin
533
544
  end
534
545
  path_info = uri.path
535
546
 
547
+ if (@add_query_params)
548
+
549
+ query_params = WEBrick::HTTPUtils.parse_query(uri.query)
550
+
551
+ query_params.each_pair {|k,v|
552
+ params["QUERY_#{k.gsub('-','_').upcase}"] = v
553
+ }
554
+ end
555
+
536
556
  params.merge!(@env)
557
+
537
558
  @env.clear
538
559
 
539
560
  code, header, body = @callback.call(path_info, params)
@@ -0,0 +1,141 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'json'
18
+
19
+ require 'fluent/plugin/input'
20
+ require 'fluent/config/error'
21
+
22
+ module Fluent::Plugin
23
+ class SampleInput < Input
24
+ Fluent::Plugin.register_input('sample', self)
25
+ Fluent::Plugin.register_input('dummy', self)
26
+
27
+ helpers :thread, :storage
28
+
29
+ BIN_NUM = 10
30
+ DEFAULT_STORAGE_TYPE = 'local'
31
+
32
+ desc "The value is the tag assigned to the generated events."
33
+ config_param :tag, :string
34
+ desc "The number of events in event stream of each emits."
35
+ config_param :size, :integer, default: 1
36
+ desc "It configures how many events to generate per second."
37
+ config_param :rate, :integer, default: 1
38
+ desc "If specified, each generated event has an auto-incremented key field."
39
+ config_param :auto_increment_key, :string, default: nil
40
+ desc "The boolean to suspend-and-resume incremental value after restart"
41
+ config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
+ config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
+ begin
45
+ parsed = JSON.parse(val)
46
+ rescue JSON::ParserError => ex
47
+ # Fluent::ConfigParseError, "got incomplete JSON" will be raised
48
+ # at literal_parser.rb with --use-v1-config, but I had to
49
+ # take care at here for the case of --use-v0-config.
50
+ raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
51
+ end
52
+ sample = parsed.is_a?(Array) ? parsed : [parsed]
53
+ sample.each_with_index do |e, i|
54
+ raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
55
+ end
56
+ sample
57
+ end
58
+
59
+ def initialize
60
+ super
61
+ @storage = nil
62
+ end
63
+
64
+ def configure(conf)
65
+ super
66
+ @sample_index = 0
67
+ config = conf.elements.select{|e| e.name == 'storage' }.first
68
+ @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
+ end
70
+
71
+ def multi_workers_ready?
72
+ true
73
+ end
74
+
75
+ def start
76
+ super
77
+
78
+ @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
79
+ # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
80
+ @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
81
+
82
+ if @auto_increment_key && !@storage.get(:auto_increment_value)
83
+ @storage.put(:auto_increment_value, -1)
84
+ end
85
+
86
+ thread_create(:sample_input, &method(:run))
87
+ end
88
+
89
+ def run
90
+ batch_num = (@rate / BIN_NUM).to_i
91
+ residual_num = (@rate % BIN_NUM)
92
+ while thread_current_running?
93
+ current_time = Time.now.to_i
94
+ BIN_NUM.times do
95
+ break unless (thread_current_running? && Time.now.to_i <= current_time)
96
+ wait(0.1) { emit(batch_num) }
97
+ end
98
+ emit(residual_num) if thread_current_running?
99
+ # wait for next second
100
+ while thread_current_running? && Time.now.to_i <= current_time
101
+ sleep 0.01
102
+ end
103
+ end
104
+ end
105
+
106
+ def emit(num)
107
+ begin
108
+ if @size > 1
109
+ num.times do
110
+ router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
111
+ end
112
+ else
113
+ num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
114
+ end
115
+ rescue => _
116
+ # ignore all errors not to stop emits by emit errors
117
+ end
118
+ end
119
+
120
+ def generate
121
+ d = @sample[@sample_index]
122
+ unless d
123
+ @sample_index = 0
124
+ d = @sample[@sample_index]
125
+ end
126
+ @sample_index += 1
127
+ if @auto_increment_key
128
+ d = d.dup
129
+ d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
+ end
131
+ d
132
+ end
133
+
134
+ def wait(time)
135
+ start_time = Time.now
136
+ yield
137
+ sleep_time = time - (Time.now - start_time)
138
+ sleep sleep_time if sleep_time > 0
139
+ end
140
+ end
141
+ end
@@ -22,6 +22,7 @@ require 'fluent/event'
22
22
  require 'fluent/plugin/buffer'
23
23
  require 'fluent/plugin/parser_multiline'
24
24
  require 'fluent/variable_store'
25
+ require 'fluent/capability'
25
26
  require 'fluent/plugin/in_tail/position_file'
26
27
 
27
28
  if Fluent.windows?
@@ -104,6 +105,8 @@ module Fluent::Plugin
104
105
  config_param :ignore_repeated_permission_error, :bool, default: false
105
106
  desc 'Format path with the specified timezone'
106
107
  config_param :path_timezone, :string, default: nil
108
+ desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
109
+ config_param :follow_inodes, :bool, default: false
107
110
 
108
111
  config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
109
112
  config_argument :usage, :string, default: 'in_tail_parser'
@@ -154,6 +157,9 @@ module Fluent::Plugin
154
157
  end
155
158
  @variable_store[@pos_file] = self.plugin_id
156
159
  else
160
+ if @follow_inodes
161
+ raise Fluent::ConfigError, "Can't follow inodes without pos_file configuration parameter"
162
+ end
157
163
  $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
158
164
  $log.warn "this parameter is highly recommended to save the position to resume tailing."
159
165
  end
@@ -171,6 +177,7 @@ module Fluent::Plugin
171
177
  @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
172
178
  # parser is already created by parser helper
173
179
  @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
180
+ @capability = Fluent::Capability.new(:current_process)
174
181
  end
175
182
 
176
183
  def configure_tag
@@ -214,7 +221,7 @@ module Fluent::Plugin
214
221
  FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)
215
222
  @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
216
223
  @pf_file.sync = true
217
- @pf = PositionFile.load(@pf_file, logger: log)
224
+ @pf = PositionFile.load(@pf_file, @follow_inodes, expand_paths, logger: log)
218
225
 
219
226
  if @pos_file_compaction_interval
220
227
  timer_execute(:in_tail_refresh_compact_pos_file, @pos_file_compaction_interval) do
@@ -238,7 +245,7 @@ module Fluent::Plugin
238
245
 
239
246
  def shutdown
240
247
  # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
241
- stop_watchers(@tails.keys, immediate: true, remove_watcher: false)
248
+ stop_watchers(existence_path, immediate: true, remove_watcher: false)
242
249
  @pf_file.close if @pf_file
243
250
 
244
251
  super
@@ -250,6 +257,11 @@ module Fluent::Plugin
250
257
  close_watcher_handles
251
258
  end
252
259
 
260
+ def have_read_capability?
261
+ @capability.have_capability?(:effective, :dac_read_search) ||
262
+ @capability.have_capability?(:effective, :dac_override)
263
+ end
264
+
253
265
  def expand_paths
254
266
  date = Fluent::EventTime.now
255
267
  paths = []
@@ -263,7 +275,7 @@ module Fluent::Plugin
263
275
  paths += Dir.glob(path).select { |p|
264
276
  begin
265
277
  is_file = !File.directory?(p)
266
- if File.readable?(p) && is_file
278
+ if (File.readable?(p) || have_read_capability?) && is_file
267
279
  if @limit_recently_modified && File.mtime(p) < (date.to_time - @limit_recently_modified)
268
280
  false
269
281
  else
@@ -296,7 +308,31 @@ module Fluent::Plugin
296
308
  end
297
309
  path.include?('*') ? Dir.glob(path) : path
298
310
  }.flatten.uniq
299
- paths.uniq - excluded
311
+ # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
312
+ hash = {}
313
+ (paths - excluded).select { |path|
314
+ FileTest.exist?(path)
315
+ }.each { |path|
316
+ target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
317
+ if @follow_inodes
318
+ hash[target_info.ino] = target_info
319
+ else
320
+ hash[target_info.path] = target_info
321
+ end
322
+ }
323
+ hash
324
+ end
325
+
326
+ def existence_path
327
+ hash = {}
328
+ @tails.each_key {|target_info|
329
+ if @follow_inodes
330
+ hash[target_info.ino] = target_info
331
+ else
332
+ hash[target_info.path] = target_info
333
+ end
334
+ }
335
+ hash
300
336
  end
301
337
 
302
338
  # in_tail with '*' path doesn't check rotation file equality at refresh phase.
@@ -305,21 +341,21 @@ module Fluent::Plugin
305
341
  # In such case, you should separate log directory and specify two paths in path parameter.
306
342
  # e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file
307
343
  def refresh_watchers
308
- target_paths = expand_paths
309
- existence_paths = @tails.keys
344
+ target_paths_hash = expand_paths
345
+ existence_paths_hash = existence_path
310
346
 
311
347
  log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" }
312
348
 
313
- unwatched = existence_paths - target_paths
314
- added = target_paths - existence_paths
349
+ unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
350
+ added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
315
351
 
316
- stop_watchers(unwatched, immediate: false, unwatched: true) unless unwatched.empty?
317
- start_watchers(added) unless added.empty?
352
+ stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
353
+ start_watchers(added_hash) unless added_hash.empty?
318
354
  end
319
355
 
320
- def setup_watcher(path, pe)
356
+ def setup_watcher(target_info, pe)
321
357
  line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
322
- tw = TailWatcher.new(path, pe, log, @read_from_head, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
358
+ tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
323
359
 
324
360
  if @enable_watch_timer
325
361
  tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -350,47 +386,52 @@ module Fluent::Plugin
350
386
  raise e
351
387
  end
352
388
 
353
- def start_watchers(paths)
354
- paths.each { |path|
389
+ def start_watchers(targets_info)
390
+ targets_info.each_value { |target_info|
355
391
  pe = nil
356
392
  if @pf
357
- pe = @pf[path]
393
+ pe = @pf[target_info]
358
394
  if @read_from_head && pe.read_inode.zero?
359
395
  begin
360
- pe.update(Fluent::FileWrapper.stat(path).ino, 0)
396
+ pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
361
397
  rescue Errno::ENOENT
362
- $log.warn "#{path} not found. Continuing without tailing it."
398
+ $log.warn "#{target_info.path} not found. Continuing without tailing it."
363
399
  end
364
400
  end
365
401
  end
366
402
 
367
403
  begin
368
- tw = setup_watcher(path, pe)
404
+ tw = setup_watcher(target_info, pe)
369
405
  rescue WatcherSetupError => e
370
- log.warn "Skip #{path} because unexpected setup error happens: #{e}"
406
+ log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
371
407
  next
372
408
  end
373
- @tails[path] = tw
409
+ target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
410
+ @tails[target_info] = tw
374
411
  }
375
412
  end
376
413
 
377
- def stop_watchers(paths, immediate: false, unwatched: false, remove_watcher: true)
378
- paths.each { |path|
379
- tw = remove_watcher ? @tails.delete(path) : @tails[path]
414
+ def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
415
+ targets_info.each_value { |target_info|
416
+ if remove_watcher
417
+ tw = @tails.delete(target_info)
418
+ else
419
+ tw = @tails[target_info]
420
+ end
380
421
  if tw
381
422
  tw.unwatched = unwatched
382
423
  if immediate
383
- detach_watcher(tw, false)
424
+ detach_watcher(tw, target_info.ino, false)
384
425
  else
385
- detach_watcher_after_rotate_wait(tw)
426
+ detach_watcher_after_rotate_wait(tw, target_info.ino)
386
427
  end
387
428
  end
388
429
  }
389
430
  end
390
431
 
391
432
  def close_watcher_handles
392
- @tails.keys.each do |path|
393
- tw = @tails.delete(path)
433
+ @tails.keys.each do |target_info|
434
+ tw = @tails.delete(target_info)
394
435
  if tw
395
436
  tw.close
396
437
  end
@@ -398,25 +439,39 @@ module Fluent::Plugin
398
439
  end
399
440
 
400
441
  # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
401
- def update_watcher(path, pe)
402
- log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
442
+ def update_watcher(target_info, pe)
443
+ log.info("detected rotation of #{target_info.path}; waiting #{@rotate_wait} seconds")
403
444
 
404
445
  if @pf
405
- unless pe.read_inode == @pf[path].read_inode
446
+ pe_inode = pe.read_inode
447
+ target_info_from_position_entry = TargetInfo.new(target_info.path, pe_inode)
448
+ unless pe_inode == @pf[target_info_from_position_entry].read_inode
406
449
  log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
407
450
  return
408
451
  end
409
452
  end
410
- rotated_tw = @tails[path]
411
- @tails[path] = setup_watcher(path, pe)
412
- detach_watcher_after_rotate_wait(rotated_tw) if rotated_tw
453
+
454
+ rotated_target_info = TargetInfo.new(target_info.path, pe.read_inode)
455
+ rotated_tw = @tails[rotated_target_info]
456
+ new_target_info = target_info.dup
457
+
458
+ if @follow_inodes
459
+ new_position_entry = @pf[target_info]
460
+
461
+ if new_position_entry.read_inode == 0
462
+ @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
463
+ end
464
+ else
465
+ @tails[new_target_info] = setup_watcher(new_target_info, pe)
466
+ end
467
+ detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
413
468
  end
414
469
 
415
470
  # TailWatcher#close is called by another thread at shutdown phase.
416
471
  # It causes 'can't modify string; temporarily locked' error in IOHandler
417
472
  # so adding close_io argument to avoid this problem.
418
473
  # At shutdown, IOHandler's io will be released automatically after detached the event loop
419
- def detach_watcher(tw, close_io = true)
474
+ def detach_watcher(tw, ino, close_io = true)
420
475
  tw.watchers.each do |watcher|
421
476
  event_loop_detach(watcher)
422
477
  end
@@ -425,15 +480,16 @@ module Fluent::Plugin
425
480
  tw.close if close_io
426
481
 
427
482
  if tw.unwatched && @pf
428
- @pf.unwatch(tw.path)
483
+ target_info = TargetInfo.new(tw.path, ino)
484
+ @pf.unwatch(target_info)
429
485
  end
430
486
  end
431
487
 
432
- def detach_watcher_after_rotate_wait(tw)
488
+ def detach_watcher_after_rotate_wait(tw, ino)
433
489
  # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
434
490
  # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
435
491
  timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
436
- detach_watcher(tw)
492
+ detach_watcher(tw, ino)
437
493
  end
438
494
  end
439
495
 
@@ -601,10 +657,12 @@ module Fluent::Plugin
601
657
  end
602
658
 
603
659
  class TailWatcher
604
- def initialize(path, pe, log, read_from_head, update_watcher, line_buffer_timer_flusher, io_handler_build)
605
- @path = path
660
+ def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
661
+ @path = target_info.path
662
+ @ino = target_info.ino
606
663
  @pe = pe || MemoryPositionEntry.new
607
664
  @read_from_head = read_from_head
665
+ @follow_inodes = follow_inodes
608
666
  @update_watcher = update_watcher
609
667
  @log = log
610
668
  @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
@@ -614,7 +672,7 @@ module Fluent::Plugin
614
672
  @watchers = []
615
673
  end
616
674
 
617
- attr_reader :path
675
+ attr_reader :path, :ino
618
676
  attr_reader :pe
619
677
  attr_reader :line_buffer_timer_flusher
620
678
  attr_accessor :unwatched # This is used for removing position entry from PositionFile
@@ -709,7 +767,17 @@ module Fluent::Plugin
709
767
  end
710
768
 
711
769
  if watcher_needs_update
712
- @update_watcher.call(@path, swap_state(@pe))
770
+ # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
771
+ # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
772
+ if stat
773
+ target_info = TargetInfo.new(@path, stat.ino)
774
+ if @follow_inodes
775
+ # don't want to swap state because we need latest read offset in pos file even after rotate_wait
776
+ @update_watcher.call(target_info, @pe)
777
+ else
778
+ @update_watcher.call(target_info, swap_state(@pe))
779
+ end
780
+ end
713
781
  else
714
782
  @log.info "detected rotation of #{@path}"
715
783
  @io_handler = io_handler