fluentd 1.14.0-x86-mingw32 → 1.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +6 -6
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/CHANGELOG.md +101 -0
- data/README.md +2 -0
- data/SECURITY.md +18 -0
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +62 -8
- data/lib/fluent/plugin/in_tail.rb +53 -7
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +13 -1
- data/lib/fluent/plugin/out_forward.rb +15 -7
- data/lib/fluent/plugin/output.rb +6 -5
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin_id.rb +2 -1
- data/lib/fluent/supervisor.rb +2 -2
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_cat.rb +31 -2
- data/test/plugin/in_tail/test_io_handler.rb +12 -4
- data/test/plugin/test_bare_output.rb +1 -1
- data/test/plugin/test_buffer.rb +149 -1
- data/test/plugin/test_filter.rb +1 -1
- data/test/plugin/test_in_tail.rb +109 -9
- data/test/plugin/test_input.rb +1 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_exec_filter.rb +4 -0
- data/test/plugin/test_out_file.rb +29 -13
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +9 -9
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_time_parser.rb +22 -0
- metadata +10 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '0735519c797e326cb6fcd64a4e27a0e0872af520ae52d24802460c51869417dc'
|
4
|
+
data.tar.gz: a6524d76380af765663e85378c6e1bda25a83f01bfd5ee034152f0f672d44071
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: af913a0847c7bd11dc534cf935ae1ae73f7b447836a8bca8abd8cd4a151dfc7cdf4e72fefe577c265dbccc1829f2c0dc6fa31d23a046b869b1f69ac972570435
|
7
|
+
data.tar.gz: b0360be02ae35ba1b1a5212fbb5815a7b2fb1bf620b6cf5459f5acf4768dd19c1f7825b17ae9bc00983eaea13bcd7c3036ae7a6f802e4684dc7963f7387d9ffe
|
data/.drone.yml
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
kind: pipeline
|
2
|
-
name: fluentd-test-arm64-
|
2
|
+
name: fluentd-test-arm64-3-0
|
3
3
|
|
4
4
|
platform:
|
5
5
|
os: linux
|
6
6
|
arch: arm64
|
7
7
|
|
8
8
|
steps:
|
9
|
-
- name: fluentd-test-arm64-
|
10
|
-
image: arm64v8/ruby:
|
9
|
+
- name: fluentd-test-arm64-3-0
|
10
|
+
image: arm64v8/ruby:3.0
|
11
11
|
commands:
|
12
12
|
- apt update
|
13
13
|
- apt -y install libgmp3-dev
|
@@ -17,15 +17,15 @@ steps:
|
|
17
17
|
- bundle exec rake test
|
18
18
|
---
|
19
19
|
kind: pipeline
|
20
|
-
name: fluentd-test-arm64-
|
20
|
+
name: fluentd-test-arm64-2-7
|
21
21
|
|
22
22
|
platform:
|
23
23
|
os: linux
|
24
24
|
arch: arm64
|
25
25
|
|
26
26
|
steps:
|
27
|
-
- name: fluentd-test-arm64-
|
28
|
-
image: arm64v8/ruby:
|
27
|
+
- name: fluentd-test-arm64-2-7
|
28
|
+
image: arm64v8/ruby:2.7
|
29
29
|
commands:
|
30
30
|
- apt update
|
31
31
|
- apt -y install libgmp3-dev
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,104 @@
|
|
1
|
+
# v1.14.4
|
2
|
+
|
3
|
+
## Release v1.14.4 - 2022/01/06
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* `in_tail`: Add option to skip long lines (`max_line_size`)
|
8
|
+
https://github.com/fluent/fluentd/pull/3565
|
9
|
+
|
10
|
+
### Bug fix
|
11
|
+
|
12
|
+
* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
|
13
|
+
https://github.com/fluent/fluentd/pull/3560
|
14
|
+
* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
|
15
|
+
https://github.com/fluent/fluentd/pull/3579
|
16
|
+
* test: Fix unstable test cases
|
17
|
+
https://github.com/fluent/fluentd/pull/3574
|
18
|
+
https://github.com/fluent/fluentd/pull/3577
|
19
|
+
|
20
|
+
# v1.14.3
|
21
|
+
|
22
|
+
## Release v1.14.3 - 2021/11/26
|
23
|
+
|
24
|
+
### Enhancement
|
25
|
+
|
26
|
+
* Changed to accept `http_parser.rb` 0.8.0.
|
27
|
+
`http_parser.rb` 0.8.0 is ready for Ractor.
|
28
|
+
https://github.com/fluent/fluentd/pull/3544
|
29
|
+
|
30
|
+
### Bug fix
|
31
|
+
|
32
|
+
* in_tail: Fixed a bug that no new logs are read when
|
33
|
+
`enable_stat_watcher true` and `enable_watch_timer false` is set.
|
34
|
+
https://github.com/fluent/fluentd/pull/3541
|
35
|
+
* in_tail: Fixed a bug that the beginning and initial lines are lost
|
36
|
+
after startup when `read_from_head false` and path includes wildcard '*'.
|
37
|
+
https://github.com/fluent/fluentd/pull/3542
|
38
|
+
* Fixed a bug that processing messages were lost when
|
39
|
+
BufferChunkOverflowError was thrown even though only a specific
|
40
|
+
message size exceeds chunk_limit_size.
|
41
|
+
https://github.com/fluent/fluentd/pull/3553
|
42
|
+
https://github.com/fluent/fluentd/pull/3562
|
43
|
+
|
44
|
+
### Misc
|
45
|
+
|
46
|
+
* Bump up required version of `win32-service` gem.
|
47
|
+
newer version is required to implement additional `fluent-ctl` commands.
|
48
|
+
https://github.com/fluent/fluentd/pull/3556
|
49
|
+
|
50
|
+
# v1.14.2
|
51
|
+
|
52
|
+
## Release v1.14.2 - 2021/10/29
|
53
|
+
|
54
|
+
IMPORTANT: This release contain the fix for CVE-2021-41186 -
|
55
|
+
ReDoS vulnerability in `parser_apache2`.
|
56
|
+
This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
|
57
|
+
We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
58
|
+
`parser_apache2` plugin.
|
59
|
+
|
60
|
+
### Enhancement
|
61
|
+
|
62
|
+
* fluent-cat: Add `--event-time` option to send specified event time for testing.
|
63
|
+
|
64
|
+
### Bug fix
|
65
|
+
|
66
|
+
* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
|
67
|
+
https://github.com/fluent/fluentd/pull/3524
|
68
|
+
* Fixed ReDoS vulnerability in parser_apache2.
|
69
|
+
This vulnerability is caused by a certain pattern of a broken apache log.
|
70
|
+
|
71
|
+
# v1.14.1
|
72
|
+
|
73
|
+
## Release v1.14.1 - 2021/09/29
|
74
|
+
|
75
|
+
### Enhancement
|
76
|
+
|
77
|
+
* in_tail: Added file related metrics.
|
78
|
+
These metrics should be collected same as fluent-bit's in_tail.
|
79
|
+
https://github.com/fluent/fluentd/pull/3504
|
80
|
+
* out_forward: Changed to use metrics mechanism for node statistics
|
81
|
+
https://github.com/fluent/fluentd/pull/3506
|
82
|
+
|
83
|
+
### Bug fix
|
84
|
+
|
85
|
+
* in_tail: Fixed a crash bug that it raise undefined method of eof? error.
|
86
|
+
This error may happen only when `read_bytes_limit_per_second` was specified.
|
87
|
+
https://github.com/fluent/fluentd/pull/3500
|
88
|
+
* out_forward: Fixed a bug that node statistics information is not included correctly.
|
89
|
+
https://github.com/fluent/fluentd/pull/3503
|
90
|
+
https://github.com/fluent/fluentd/pull/3507
|
91
|
+
* Fixed a error when using `@include` directive
|
92
|
+
It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
|
93
|
+
https://github.com/fluent/fluentd/pull/3517
|
94
|
+
* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
|
95
|
+
It didn't work even if a user set it.
|
96
|
+
https://github.com/fluent/fluentd/pull/3515
|
97
|
+
* Fixed not to output nanoseconds field of next retry time in warning log
|
98
|
+
Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
|
99
|
+
next_retry_seconds -> next_retry_time)
|
100
|
+
https://github.com/fluent/fluentd/pull/3518
|
101
|
+
|
1
102
|
# v1.14.0
|
2
103
|
|
3
104
|
## Release v1.14.0 - 2021/08/30
|
data/README.md
CHANGED
@@ -88,6 +88,8 @@ You can run specified test via `TEST` environment variable:
|
|
88
88
|
|
89
89
|
A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
|
90
90
|
|
91
|
+
See [SECURITY](SECURITY.md) to contact us about vulnerability.
|
92
|
+
|
91
93
|
## Contributors:
|
92
94
|
|
93
95
|
Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
|
data/SECURITY.md
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# Security Policy
|
2
|
+
|
3
|
+
## Supported Versions
|
4
|
+
|
5
|
+
| Version | Supported |
|
6
|
+
| ------- | ------------------ |
|
7
|
+
| 1.14.x | :white_check_mark: |
|
8
|
+
| <= 1.13.x | :x: |
|
9
|
+
|
10
|
+
## Reporting a Vulnerability
|
11
|
+
|
12
|
+
Please contact to current active maintainers. (in alphabetical order)
|
13
|
+
|
14
|
+
* ashie@clear-code.com
|
15
|
+
* fujimoto@clear-code.com
|
16
|
+
* hatake@calyptia.com
|
17
|
+
* hayashi@clear-code.com
|
18
|
+
|
data/fluentd.gemspec
CHANGED
@@ -23,11 +23,11 @@ Gem::Specification.new do |gem|
|
|
23
23
|
gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
|
24
24
|
gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
|
25
25
|
gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
|
26
|
-
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.
|
26
|
+
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
|
27
27
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
28
28
|
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
29
29
|
gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
|
30
|
-
gem.add_runtime_dependency("strptime", [">= 0.2.
|
30
|
+
gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
|
31
31
|
gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
|
32
32
|
|
33
33
|
# build gem for a certain platform. see also Rakefile
|
@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
|
|
35
35
|
gem.platform = fake_platform unless fake_platform.empty?
|
36
36
|
if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
|
37
37
|
gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
|
38
|
-
gem.add_runtime_dependency("win32-service", ["~> 2.
|
38
|
+
gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
|
39
39
|
gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
|
40
40
|
gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
|
41
41
|
gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
|
data/lib/fluent/command/cat.rb
CHANGED
@@ -35,6 +35,7 @@ format = 'json'
|
|
35
35
|
message_key = 'message'
|
36
36
|
time_as_integer = false
|
37
37
|
retry_limit = 5
|
38
|
+
event_time = nil
|
38
39
|
|
39
40
|
op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
|
40
41
|
port = i
|
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
|
|
80
81
|
retry_limit = n
|
81
82
|
}
|
82
83
|
|
84
|
+
op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
|
85
|
+
event_time = v
|
86
|
+
}
|
87
|
+
|
83
88
|
singleton_class.module_eval do
|
84
89
|
define_method(:usage) do |msg|
|
85
90
|
puts op.to_s
|
@@ -134,7 +139,7 @@ class Writer
|
|
134
139
|
end
|
135
140
|
end
|
136
141
|
|
137
|
-
def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
|
142
|
+
def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
|
138
143
|
@tag = tag
|
139
144
|
@connector = connector
|
140
145
|
@socket = false
|
@@ -148,6 +153,7 @@ class Writer
|
|
148
153
|
@retry_wait = 1
|
149
154
|
@retry_limit = retry_limit
|
150
155
|
@time_as_integer = time_as_integer
|
156
|
+
@event_time = event_time
|
151
157
|
|
152
158
|
super()
|
153
159
|
end
|
@@ -166,7 +172,11 @@ class Writer
|
|
166
172
|
end
|
167
173
|
end
|
168
174
|
|
169
|
-
time =
|
175
|
+
time = if @event_time
|
176
|
+
Fluent::EventTime.parse(@event_time)
|
177
|
+
else
|
178
|
+
Fluent::EventTime.now
|
179
|
+
end
|
170
180
|
time = time.to_i if @time_as_integer
|
171
181
|
entry = if secondary_record?(record)
|
172
182
|
# Even though secondary contains Fluent::EventTime in record,
|
@@ -309,7 +319,7 @@ else
|
|
309
319
|
}
|
310
320
|
end
|
311
321
|
|
312
|
-
w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
|
322
|
+
w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
|
313
323
|
w.start
|
314
324
|
|
315
325
|
case format
|
data/lib/fluent/config/parser.rb
CHANGED
@@ -172,7 +172,7 @@ module Fluent
|
|
172
172
|
require 'open-uri'
|
173
173
|
basepath = '/'
|
174
174
|
fname = path
|
175
|
-
data = open(uri) { |f| f.read }
|
175
|
+
data = URI.open(uri) { |f| f.read }
|
176
176
|
data.force_encoding('UTF-8')
|
177
177
|
ss = StringScanner.new(data)
|
178
178
|
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
data/lib/fluent/env.rb
CHANGED
@@ -39,8 +39,8 @@ module Fluent
|
|
39
39
|
config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
|
40
40
|
config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
|
41
41
|
|
42
|
-
config_param :file_permission, :string, default: nil # '0644'
|
43
|
-
config_param :dir_permission, :string, default: nil # '0755'
|
42
|
+
config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
|
43
|
+
config_param :dir_permission, :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
|
44
44
|
|
45
45
|
def initialize
|
46
46
|
super
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -332,12 +332,14 @@ module Fluent
|
|
332
332
|
unstaged_chunks = {} # metadata => [chunk, chunk, ...]
|
333
333
|
chunks_to_enqueue = []
|
334
334
|
staged_bytesizes_by_chunk = {}
|
335
|
+
# track internal BufferChunkOverflowError in write_step_by_step
|
336
|
+
buffer_chunk_overflow_errors = []
|
335
337
|
|
336
338
|
begin
|
337
339
|
# sort metadata to get lock of chunks in same order with other threads
|
338
340
|
metadata_and_data.keys.sort.each do |metadata|
|
339
341
|
data = metadata_and_data[metadata]
|
340
|
-
write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
|
342
|
+
write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize, error|
|
341
343
|
chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
|
342
344
|
operated_chunks << chunk
|
343
345
|
if chunk.staged?
|
@@ -352,6 +354,9 @@ module Fluent
|
|
352
354
|
unstaged_chunks[metadata] ||= []
|
353
355
|
unstaged_chunks[metadata] << chunk
|
354
356
|
end
|
357
|
+
if error && !error.empty?
|
358
|
+
buffer_chunk_overflow_errors << error
|
359
|
+
end
|
355
360
|
end
|
356
361
|
end
|
357
362
|
|
@@ -444,6 +449,10 @@ module Fluent
|
|
444
449
|
end
|
445
450
|
chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
|
446
451
|
end
|
452
|
+
unless buffer_chunk_overflow_errors.empty?
|
453
|
+
# Notify delayed BufferChunkOverflowError here
|
454
|
+
raise BufferChunkOverflowError, buffer_chunk_overflow_errors.join(", ")
|
455
|
+
end
|
447
456
|
end
|
448
457
|
end
|
449
458
|
|
@@ -716,6 +725,7 @@ module Fluent
|
|
716
725
|
|
717
726
|
def write_step_by_step(metadata, data, format, splits_count, &block)
|
718
727
|
splits = []
|
728
|
+
errors = []
|
719
729
|
if splits_count > data.size
|
720
730
|
splits_count = data.size
|
721
731
|
end
|
@@ -757,22 +767,65 @@ module Fluent
|
|
757
767
|
raise ShouldRetry unless chunk.writable?
|
758
768
|
staged_chunk_used = true if chunk.staged?
|
759
769
|
|
760
|
-
original_bytesize = chunk.bytesize
|
770
|
+
original_bytesize = committed_bytesize = chunk.bytesize
|
761
771
|
begin
|
762
772
|
while writing_splits_index < splits.size
|
763
773
|
split = splits[writing_splits_index]
|
774
|
+
formatted_split = format ? format.call(split) : nil
|
775
|
+
|
776
|
+
if split.size == 1 # Check BufferChunkOverflowError
|
777
|
+
determined_bytesize = nil
|
778
|
+
if @compress != :text
|
779
|
+
determined_bytesize = nil
|
780
|
+
elsif formatted_split
|
781
|
+
determined_bytesize = formatted_split.bytesize
|
782
|
+
elsif split.first.respond_to?(:bytesize)
|
783
|
+
determined_bytesize = split.first.bytesize
|
784
|
+
end
|
785
|
+
|
786
|
+
if determined_bytesize && determined_bytesize > @chunk_limit_size
|
787
|
+
# It is a obvious case that BufferChunkOverflowError should be raised here.
|
788
|
+
# But if it raises here, already processed 'split' or
|
789
|
+
# the proceeding 'split' will be lost completely.
|
790
|
+
# So it is a last resort to delay raising such a exception
|
791
|
+
errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
|
792
|
+
writing_splits_index += 1
|
793
|
+
next
|
794
|
+
end
|
795
|
+
|
796
|
+
if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
|
797
|
+
# The split will (might) cause size over so keep already processed
|
798
|
+
# 'split' content here (allow performance regression a bit).
|
799
|
+
chunk.commit
|
800
|
+
committed_bytesize = chunk.bytesize
|
801
|
+
end
|
802
|
+
end
|
803
|
+
|
764
804
|
if format
|
765
|
-
chunk.concat(
|
805
|
+
chunk.concat(formatted_split, split.size)
|
766
806
|
else
|
767
807
|
chunk.append(split, compress: @compress)
|
768
808
|
end
|
809
|
+
adding_bytes = chunk.bytesize - committed_bytesize
|
769
810
|
|
770
811
|
if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
|
771
812
|
chunk.rollback
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
813
|
+
committed_bytesize = chunk.bytesize
|
814
|
+
|
815
|
+
if split.size == 1 # Check BufferChunkOverflowError again
|
816
|
+
if adding_bytes > @chunk_limit_size
|
817
|
+
errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
|
818
|
+
writing_splits_index += 1
|
819
|
+
next
|
820
|
+
else
|
821
|
+
# As already processed content is kept after rollback, then unstaged chunk should be queued.
|
822
|
+
# After that, re-process current split again.
|
823
|
+
# New chunk should be allocated, to do it, modify @stage and so on.
|
824
|
+
synchronize { @stage.delete(modified_metadata) }
|
825
|
+
staged_chunk_used = false
|
826
|
+
chunk.unstaged!
|
827
|
+
break
|
828
|
+
end
|
776
829
|
end
|
777
830
|
|
778
831
|
if chunk_size_full?(chunk) || split.size == 1
|
@@ -795,7 +848,8 @@ module Fluent
|
|
795
848
|
raise
|
796
849
|
end
|
797
850
|
|
798
|
-
block.call(chunk, chunk.bytesize - original_bytesize)
|
851
|
+
block.call(chunk, chunk.bytesize - original_bytesize, errors)
|
852
|
+
errors = []
|
799
853
|
end
|
800
854
|
end
|
801
855
|
rescue ShouldRetry
|
@@ -38,6 +38,7 @@ module Fluent::Plugin
|
|
38
38
|
helpers :timer, :event_loop, :parser, :compat_parameters
|
39
39
|
|
40
40
|
RESERVED_CHARS = ['/', '*', '%'].freeze
|
41
|
+
MetricsInfo = Struct.new(:opened, :closed, :rotated)
|
41
42
|
|
42
43
|
class WatcherSetupError < StandardError
|
43
44
|
def initialize(msg)
|
@@ -57,6 +58,8 @@ module Fluent::Plugin
|
|
57
58
|
@pf = nil
|
58
59
|
@ignore_list = []
|
59
60
|
@shutdown_start_time = nil
|
61
|
+
@metrics = nil
|
62
|
+
@startup = true
|
60
63
|
end
|
61
64
|
|
62
65
|
desc 'The paths to read. Multiple paths can be specified, separated by comma.'
|
@@ -110,6 +113,8 @@ module Fluent::Plugin
|
|
110
113
|
config_param :path_timezone, :string, default: nil
|
111
114
|
desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
|
112
115
|
config_param :follow_inodes, :bool, default: false
|
116
|
+
desc 'Maximum length of line. The longer line is just skipped.'
|
117
|
+
config_param :max_line_size, :size, default: nil
|
113
118
|
|
114
119
|
config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
|
115
120
|
config_argument :usage, :string, default: 'in_tail_parser'
|
@@ -191,6 +196,10 @@ module Fluent::Plugin
|
|
191
196
|
@read_bytes_limit_per_second = min_bytes
|
192
197
|
end
|
193
198
|
end
|
199
|
+
opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
|
200
|
+
closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
|
201
|
+
rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
|
202
|
+
@metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
|
194
203
|
end
|
195
204
|
|
196
205
|
def configure_tag
|
@@ -363,19 +372,25 @@ module Fluent::Plugin
|
|
363
372
|
def refresh_watchers
|
364
373
|
target_paths_hash = expand_paths
|
365
374
|
existence_paths_hash = existence_path
|
366
|
-
|
367
|
-
log.debug {
|
375
|
+
|
376
|
+
log.debug {
|
377
|
+
target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
|
378
|
+
existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
|
379
|
+
"tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
|
380
|
+
}
|
368
381
|
|
369
382
|
unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
|
370
383
|
added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
|
371
384
|
|
372
385
|
stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
|
373
386
|
start_watchers(added_hash) unless added_hash.empty?
|
387
|
+
@startup = false if @startup
|
374
388
|
end
|
375
389
|
|
376
390
|
def setup_watcher(target_info, pe)
|
377
391
|
line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
|
378
|
-
|
392
|
+
read_from_head = !@startup || @read_from_head
|
393
|
+
tw = TailWatcher.new(target_info, pe, log, read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
|
379
394
|
|
380
395
|
if @enable_watch_timer
|
381
396
|
tt = TimerTrigger.new(1, log) { tw.on_notify }
|
@@ -383,7 +398,7 @@ module Fluent::Plugin
|
|
383
398
|
end
|
384
399
|
|
385
400
|
if @enable_stat_watcher
|
386
|
-
tt = StatWatcher.new(path, log) { tw.on_notify }
|
401
|
+
tt = StatWatcher.new(target_info.path, log) { tw.on_notify }
|
387
402
|
tw.register_watcher(tt)
|
388
403
|
end
|
389
404
|
|
@@ -581,6 +596,14 @@ module Fluent::Plugin
|
|
581
596
|
|
582
597
|
# @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
|
583
598
|
def receive_lines(lines, tail_watcher)
|
599
|
+
lines = lines.reject do |line|
|
600
|
+
skip_line = @max_line_size ? line.bytesize > @max_line_size : false
|
601
|
+
if skip_line
|
602
|
+
log.warn "received line length is longer than #{@max_line_size}"
|
603
|
+
log.debug "skipped line: #{line.chomp}"
|
604
|
+
end
|
605
|
+
skip_line
|
606
|
+
end
|
584
607
|
es = @receive_handler.call(lines, tail_watcher)
|
585
608
|
unless es.empty?
|
586
609
|
tag = if @tag_prefix || @tag_suffix
|
@@ -670,6 +693,19 @@ module Fluent::Plugin
|
|
670
693
|
es
|
671
694
|
end
|
672
695
|
|
696
|
+
def statistics
|
697
|
+
stats = super
|
698
|
+
|
699
|
+
stats = {
|
700
|
+
'input' => stats["input"].merge({
|
701
|
+
'opened_file_count' => @metrics.opened.get,
|
702
|
+
'closed_file_count' => @metrics.closed.get,
|
703
|
+
'rotated_file_count' => @metrics.rotated.get,
|
704
|
+
})
|
705
|
+
}
|
706
|
+
stats
|
707
|
+
end
|
708
|
+
|
673
709
|
private
|
674
710
|
|
675
711
|
def io_handler(watcher, path)
|
@@ -682,6 +718,7 @@ module Fluent::Plugin
|
|
682
718
|
open_on_every_update: @open_on_every_update,
|
683
719
|
from_encoding: @from_encoding,
|
684
720
|
encoding: @encoding,
|
721
|
+
metrics: @metrics,
|
685
722
|
&method(:receive_lines)
|
686
723
|
)
|
687
724
|
end
|
@@ -717,7 +754,7 @@ module Fluent::Plugin
|
|
717
754
|
end
|
718
755
|
|
719
756
|
class TailWatcher
|
720
|
-
def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
|
757
|
+
def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
|
721
758
|
@path = target_info.path
|
722
759
|
@ino = target_info.ino
|
723
760
|
@pe = pe || MemoryPositionEntry.new
|
@@ -729,6 +766,7 @@ module Fluent::Plugin
|
|
729
766
|
@line_buffer_timer_flusher = line_buffer_timer_flusher
|
730
767
|
@io_handler = nil
|
731
768
|
@io_handler_build = io_handler_build
|
769
|
+
@metrics = metrics
|
732
770
|
@watchers = []
|
733
771
|
end
|
734
772
|
|
@@ -762,7 +800,7 @@ module Fluent::Plugin
|
|
762
800
|
end
|
763
801
|
|
764
802
|
def eof?
|
765
|
-
@io_handler.eof?
|
803
|
+
@io_handler.nil? || @io_handler.eof?
|
766
804
|
end
|
767
805
|
|
768
806
|
def on_notify
|
@@ -855,6 +893,7 @@ module Fluent::Plugin
|
|
855
893
|
@log.info "detected rotation of #{@path}"
|
856
894
|
@io_handler = io_handler
|
857
895
|
end
|
896
|
+
@metrics.rotated.inc
|
858
897
|
end
|
859
898
|
end
|
860
899
|
|
@@ -934,7 +973,7 @@ module Fluent::Plugin
|
|
934
973
|
|
935
974
|
attr_accessor :shutdown_timeout
|
936
975
|
|
937
|
-
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
|
976
|
+
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
|
938
977
|
@watcher = watcher
|
939
978
|
@path = path
|
940
979
|
@read_lines_limit = read_lines_limit
|
@@ -953,6 +992,7 @@ module Fluent::Plugin
|
|
953
992
|
@shutdown_timeout = SHUTDOWN_TIMEOUT
|
954
993
|
@shutdown_mutex = Mutex.new
|
955
994
|
@eof = false
|
995
|
+
@metrics = metrics
|
956
996
|
|
957
997
|
@log.info "following tail of #{@path}"
|
958
998
|
end
|
@@ -972,6 +1012,7 @@ module Fluent::Plugin
|
|
972
1012
|
if @io && !@io.closed?
|
973
1013
|
@io.close
|
974
1014
|
@io = nil
|
1015
|
+
@metrics.closed.inc
|
975
1016
|
end
|
976
1017
|
end
|
977
1018
|
|
@@ -1059,6 +1100,7 @@ module Fluent::Plugin
|
|
1059
1100
|
def open
|
1060
1101
|
io = Fluent::FileWrapper.open(@path)
|
1061
1102
|
io.seek(@watcher.pe.read_pos + @fifo.bytesize)
|
1103
|
+
@metrics.opened.inc
|
1062
1104
|
io
|
1063
1105
|
rescue RangeError
|
1064
1106
|
io.close if io
|
@@ -1111,6 +1153,10 @@ module Fluent::Plugin
|
|
1111
1153
|
def opened?
|
1112
1154
|
false
|
1113
1155
|
end
|
1156
|
+
|
1157
|
+
def eof?
|
1158
|
+
true
|
1159
|
+
end
|
1114
1160
|
end
|
1115
1161
|
|
1116
1162
|
class RotateHandler
|
@@ -46,7 +46,7 @@ module Fluent::Plugin
|
|
46
46
|
@ignore_errors << (store.arg.include?('ignore_error'))
|
47
47
|
@ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
|
48
48
|
}
|
49
|
-
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) &&
|
49
|
+
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
|
50
50
|
log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
|
51
51
|
end
|
52
52
|
end
|
@@ -181,6 +181,13 @@ module Fluent::Plugin
|
|
181
181
|
@dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
|
182
182
|
@file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
|
183
183
|
@need_lock = system_config.workers > 1
|
184
|
+
|
185
|
+
# https://github.com/fluent/fluentd/issues/3569
|
186
|
+
@need_ruby_on_macos_workaround = false
|
187
|
+
if @append && Fluent.macos?
|
188
|
+
condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
|
189
|
+
@need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
|
190
|
+
end
|
184
191
|
end
|
185
192
|
|
186
193
|
def multi_workers_ready?
|
@@ -223,7 +230,12 @@ module Fluent::Plugin
|
|
223
230
|
|
224
231
|
def write_without_compression(path, chunk)
|
225
232
|
File.open(path, "ab", @file_perm) do |f|
|
226
|
-
|
233
|
+
if @need_ruby_on_macos_workaround
|
234
|
+
content = chunk.read()
|
235
|
+
f.puts content
|
236
|
+
else
|
237
|
+
chunk.write_to(f)
|
238
|
+
end
|
227
239
|
end
|
228
240
|
end
|
229
241
|
|