fluentd 1.9.1-x64-mingw32 → 1.9.2-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/CHANGELOG.md +22 -0
- data/lib/fluent/plugin/in_http.rb +10 -4
- data/lib/fluent/plugin/in_tail.rb +10 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +78 -38
- data/lib/fluent/plugin/parser_syslog.rb +11 -4
- data/lib/fluent/supervisor.rb +37 -7
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +82 -8
- data/test/plugin/in_tail/test_position_file.rb +64 -15
- data/test/plugin/test_in_tail.rb +1 -1
- data/test/plugin/test_parser_syslog.rb +6 -6
- data/test/test_supervisor.rb +27 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 976b5918a9761704f4c808761a5988c6f95f1b86e6fd7f7fa68f68f5de4b64ce
|
4
|
+
data.tar.gz: 37a6608a19b50fc7d4815df34fb9321edfd48547d32513a0c631d297580ca71b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce2d17a96d496d47848082e7ed980b3c8bd111d071ff256f9788db0d2c13fb41da543eeee39cd3764b6b18d16a53039a49968f5797264be7dcf3b8d472a24aa2
|
7
|
+
data.tar.gz: 0b86f77306d5b5b79d75d56d013eabf81e0714446a9b8cdf814b66893e4baea1004866060c45d0b2ef1914bf852f7592cde72eba15446d2e881f839e989c7039
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,27 @@
|
|
1
1
|
# v1.9
|
2
2
|
|
3
|
+
## Release v1.9.2 - 2020/02/13
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* in_tail: Add `pos_file_compaction_interval` parameter for auto compaction
|
8
|
+
https://github.com/fluent/fluentd/pull/2805
|
9
|
+
* command: Use given encoding when RUBYOPT has `-E`
|
10
|
+
https://github.com/fluent/fluentd/pull/2814
|
11
|
+
|
12
|
+
### Bug fix
|
13
|
+
|
14
|
+
* command: Accept RUBYOPT with two or more options
|
15
|
+
https://github.com/fluent/fluentd/pull/2807
|
16
|
+
* command: Fix infinite loop bug when RUBYOPT is invalid
|
17
|
+
https://github.com/fluent/fluentd/pull/2813
|
18
|
+
* log: serverengine's log should be formatted with the same format of fluentd
|
19
|
+
https://github.com/fluent/fluentd/pull/2812
|
20
|
+
* in_http: Fix `NoMethodError` when `OPTIONS` request doesn't have 'Origin' header
|
21
|
+
https://github.com/fluent/fluentd/pull/2823
|
22
|
+
* parser_syslog: Improved for parsing RFC5424 structured data in `parser_syslog`
|
23
|
+
https://github.com/fluent/fluentd/pull/2816
|
24
|
+
|
3
25
|
## Release v1.9.1 - 2020/01/31
|
4
26
|
|
5
27
|
### Enhancement
|
@@ -526,14 +526,20 @@ module Fluent::Plugin
|
|
526
526
|
end
|
527
527
|
|
528
528
|
def include_cors_allow_origin
|
529
|
+
if @origin.nil?
|
530
|
+
return false
|
531
|
+
end
|
532
|
+
|
529
533
|
if @cors_allow_origins.include?(@origin)
|
530
534
|
return true
|
531
535
|
end
|
532
536
|
filtered_cors_allow_origins = @cors_allow_origins.select {|origin| origin != ""}
|
533
|
-
|
534
|
-
(start_str,end_str) = origin.split("*",2)
|
535
|
-
@origin.start_with?(start_str)
|
536
|
-
end
|
537
|
+
r = filtered_cors_allow_origins.find do |origin|
|
538
|
+
(start_str, end_str) = origin.split("*", 2)
|
539
|
+
@origin.start_with?(start_str) && @origin.end_with?(end_str)
|
540
|
+
end
|
541
|
+
|
542
|
+
!r.nil?
|
537
543
|
end
|
538
544
|
end
|
539
545
|
end
|
@@ -71,6 +71,8 @@ module Fluent::Plugin
|
|
71
71
|
config_param :rotate_wait, :time, default: 5
|
72
72
|
desc 'Fluentd will record the position it last read into this file.'
|
73
73
|
config_param :pos_file, :string, default: nil
|
74
|
+
desc 'The cleanup interval of pos file'
|
75
|
+
config_param :pos_file_compaction_interval, :integer, default: nil
|
74
76
|
desc 'Start to read the logs from the head of file, not bottom.'
|
75
77
|
config_param :read_from_head, :bool, default: false
|
76
78
|
# When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
|
@@ -211,7 +213,14 @@ module Fluent::Plugin
|
|
211
213
|
FileUtils.mkdir_p(pos_file_dir) unless Dir.exist?(pos_file_dir)
|
212
214
|
@pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
|
213
215
|
@pf_file.sync = true
|
214
|
-
@pf = PositionFile.
|
216
|
+
@pf = PositionFile.load(@pf_file, logger: log)
|
217
|
+
|
218
|
+
if @pos_file_compaction_interval
|
219
|
+
timer_execute(:in_tail_refresh_compact_pos_file, @pos_file_compaction_interval) do
|
220
|
+
log.info('Clean up the pos file')
|
221
|
+
@pf.try_compact
|
222
|
+
end
|
223
|
+
end
|
215
224
|
end
|
216
225
|
|
217
226
|
refresh_watchers unless @skip_refresh_on_startup
|
@@ -14,7 +14,7 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'fluent/plugin/
|
17
|
+
require 'fluent/plugin/input'
|
18
18
|
|
19
19
|
module Fluent::Plugin
|
20
20
|
class TailInput < Fluent::Plugin::Input
|
@@ -23,10 +23,17 @@ module Fluent::Plugin
|
|
23
23
|
POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
|
24
24
|
POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
|
25
25
|
|
26
|
-
def
|
26
|
+
def self.load(file, logger:)
|
27
|
+
pf = new(file, logger: logger)
|
28
|
+
pf.load
|
29
|
+
pf
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize(file, logger: nil)
|
27
33
|
@file = file
|
28
|
-
@
|
29
|
-
@
|
34
|
+
@logger = logger
|
35
|
+
@file_mutex = Mutex.new
|
36
|
+
@map = {}
|
30
37
|
end
|
31
38
|
|
32
39
|
def [](path)
|
@@ -48,56 +55,89 @@ module Fluent::Plugin
|
|
48
55
|
end
|
49
56
|
end
|
50
57
|
|
51
|
-
def
|
52
|
-
compact
|
58
|
+
def load
|
59
|
+
compact
|
53
60
|
|
54
|
-
file_mutex = Mutex.new
|
55
61
|
map = {}
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
next
|
62
|
+
@file_mutex.synchronize do
|
63
|
+
@file.pos = 0
|
64
|
+
|
65
|
+
@file.each_line do |line|
|
66
|
+
m = POSITION_FILE_ENTRY_REGEX.match(line)
|
67
|
+
next if m.nil?
|
68
|
+
|
69
|
+
path = m[1]
|
70
|
+
pos = m[2].to_i(16)
|
71
|
+
ino = m[3].to_i(16)
|
72
|
+
seek = @file.pos - line.bytesize + path.bytesize + 1
|
73
|
+
map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
|
62
74
|
end
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
75
|
+
end
|
76
|
+
|
77
|
+
@map = map
|
78
|
+
end
|
79
|
+
|
80
|
+
# This method is similer to #compact but it tries to get less lock to avoid a lock contention
|
81
|
+
def try_compact
|
82
|
+
last_modified = nil
|
83
|
+
size = nil
|
84
|
+
|
85
|
+
@file_mutex.synchronize do
|
86
|
+
last_modified = @file.mtime
|
87
|
+
size = @file.size
|
88
|
+
end
|
89
|
+
|
90
|
+
entries = fetch_compacted_entries
|
91
|
+
|
92
|
+
@file_mutex.synchronize do
|
93
|
+
if last_modified == @file.mtime && size == @file.size
|
94
|
+
@file.pos = 0
|
95
|
+
@file.truncate(0)
|
96
|
+
@file.write(entries.join)
|
97
|
+
else
|
98
|
+
# skip
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
def compact
|
106
|
+
@file_mutex.synchronize do
|
107
|
+
entries = fetch_compacted_entries
|
108
|
+
|
109
|
+
@file.pos = 0
|
110
|
+
@file.truncate(0)
|
111
|
+
@file.write(entries.join)
|
112
|
+
end
|
70
113
|
end
|
71
114
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
file.pos = 0
|
76
|
-
file.each_line do |line|
|
115
|
+
def fetch_compacted_entries
|
116
|
+
entries = {}
|
117
|
+
|
118
|
+
@file.pos = 0
|
119
|
+
@file.each_line do |line|
|
77
120
|
m = POSITION_FILE_ENTRY_REGEX.match(line)
|
78
|
-
|
79
|
-
|
121
|
+
if m.nil?
|
122
|
+
@logger.warn "Unparsable line in pos_file: #{line}" if @logger
|
80
123
|
next
|
81
124
|
end
|
125
|
+
|
82
126
|
path = m[1]
|
83
127
|
pos = m[2].to_i(16)
|
84
128
|
ino = m[3].to_i(16)
|
85
|
-
|
86
129
|
if pos == UNWATCHED_POSITION
|
87
|
-
|
88
|
-
|
130
|
+
@logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
|
131
|
+
else
|
132
|
+
if entries.include?(path)
|
133
|
+
@logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
|
134
|
+
end
|
89
135
|
|
90
|
-
|
91
|
-
$log.warn("#{path} already exists. use latest one: deleted #{existent_entries[path]}")
|
136
|
+
entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
|
92
137
|
end
|
93
|
-
|
94
|
-
# 32bit inode converted to 64bit at this phase
|
95
|
-
existent_entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
|
96
138
|
end
|
97
139
|
|
98
|
-
|
99
|
-
file.truncate(0)
|
100
|
-
file.write(existent_entries.values.join)
|
140
|
+
entries.values
|
101
141
|
end
|
102
142
|
end
|
103
143
|
|
@@ -27,8 +27,15 @@ module Fluent
|
|
27
27
|
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
28
28
|
# From in_syslog default pattern
|
29
29
|
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
30
|
-
REGEXP_RFC5424 =
|
31
|
-
|
30
|
+
REGEXP_RFC5424 = <<~'EOS'.chomp
|
31
|
+
(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))(?: (?<message>.+))?
|
32
|
+
EOS
|
33
|
+
REGEXP_RFC5424_NO_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
|
34
|
+
\A%s\z
|
35
|
+
EOS
|
36
|
+
REGEXP_RFC5424_WITH_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
|
37
|
+
\A<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} %s\z
|
38
|
+
EOS
|
32
39
|
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
|
33
40
|
|
34
41
|
config_set_default :time_format, "%b %d %H:%M:%S"
|
@@ -73,7 +80,7 @@ module Fluent
|
|
73
80
|
end
|
74
81
|
@time_format = @rfc5424_time_format unless conf.has_key?('time_format')
|
75
82
|
@support_rfc5424_without_subseconds = true
|
76
|
-
@with_priority ? REGEXP_RFC5424_WITH_PRI :
|
83
|
+
@with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
|
77
84
|
when :auto
|
78
85
|
class << self
|
79
86
|
alias_method :parse, :parse_auto
|
@@ -96,7 +103,7 @@ module Fluent
|
|
96
103
|
|
97
104
|
def parse_auto(text, &block)
|
98
105
|
if REGEXP_DETECT_RFC5424.match(text)
|
99
|
-
@regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI :
|
106
|
+
@regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
|
100
107
|
@time_parser = @time_parser_rfc5424
|
101
108
|
@support_rfc5424_without_subseconds = true
|
102
109
|
parse_plain(text, &block)
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -15,6 +15,7 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
require 'fileutils'
|
18
|
+
require 'open3'
|
18
19
|
|
19
20
|
require 'fluent/config'
|
20
21
|
require 'fluent/counter'
|
@@ -315,6 +316,7 @@ module Fluent
|
|
315
316
|
)
|
316
317
|
# this #init sets initialized logger to $log
|
317
318
|
logger_initializer.init(:supervisor, 0)
|
319
|
+
logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
|
318
320
|
logger = $log
|
319
321
|
|
320
322
|
command_sender = Fluent.windows? ? "pipe" : "signal"
|
@@ -659,13 +661,7 @@ module Fluent
|
|
659
661
|
Process.setproctitle("supervisor:#{@system_config.process_name}") if @system_config.process_name
|
660
662
|
$log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
|
661
663
|
|
662
|
-
|
663
|
-
fluentd_spawn_cmd = [ServerEngine.ruby_bin_path, "-Eascii-8bit:ascii-8bit"]
|
664
|
-
fluentd_spawn_cmd << rubyopt if rubyopt
|
665
|
-
fluentd_spawn_cmd << $0
|
666
|
-
fluentd_spawn_cmd += $fluentdargv
|
667
|
-
fluentd_spawn_cmd << "--under-supervisor"
|
668
|
-
|
664
|
+
fluentd_spawn_cmd = build_spawn_command
|
669
665
|
$log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
|
670
666
|
|
671
667
|
params = {
|
@@ -689,6 +685,8 @@ module Fluent
|
|
689
685
|
'rpc_endpoint' => @system_config.rpc_endpoint,
|
690
686
|
'enable_get_dump' => @system_config.enable_get_dump,
|
691
687
|
'counter_server' => @system_config.counter_server,
|
688
|
+
'log_format' => @system_config.log.format,
|
689
|
+
'log_time_format' => @system_config.log.time_format,
|
692
690
|
}
|
693
691
|
|
694
692
|
se = ServerEngine.create(ServerModule, WorkerModule){
|
@@ -872,5 +870,37 @@ module Fluent
|
|
872
870
|
system_config.overwrite_variables(**opt)
|
873
871
|
system_config
|
874
872
|
end
|
873
|
+
|
874
|
+
RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
|
875
|
+
|
876
|
+
def build_spawn_command
|
877
|
+
fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
|
878
|
+
|
879
|
+
rubyopt = ENV['RUBYOPT']
|
880
|
+
if rubyopt
|
881
|
+
encodes, others = rubyopt.split(' ').partition { |e| e.match?(RUBY_ENCODING_OPTIONS_REGEX) }
|
882
|
+
fluentd_spawn_cmd.concat(others)
|
883
|
+
|
884
|
+
adopted_encodes = encodes.empty? ? ['-Eascii-8bit:ascii-8bit'] : encodes
|
885
|
+
fluentd_spawn_cmd.concat(adopted_encodes)
|
886
|
+
else
|
887
|
+
fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
|
888
|
+
end
|
889
|
+
|
890
|
+
# Adding `-h` so that it can avoid ruby's command blocking
|
891
|
+
# e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
|
892
|
+
cmd = fluentd_spawn_cmd.join(' ')
|
893
|
+
_, e, s = Open3.capture3("#{cmd} -h")
|
894
|
+
if s.exitstatus != 0
|
895
|
+
$log.error('Invalid option is passed to RUBYOPT', command: cmd, error: e)
|
896
|
+
exit s.exitstatus
|
897
|
+
end
|
898
|
+
|
899
|
+
fluentd_spawn_cmd << $0
|
900
|
+
fluentd_spawn_cmd += $fluentdargv
|
901
|
+
fluentd_spawn_cmd << '--under-supervisor'
|
902
|
+
|
903
|
+
fluentd_spawn_cmd
|
904
|
+
end
|
875
905
|
end
|
876
906
|
end
|
data/lib/fluent/version.rb
CHANGED
@@ -47,16 +47,14 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
47
47
|
|
48
48
|
def create_cmdline(conf_path, *fluentd_options)
|
49
49
|
cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
|
50
|
-
["bundle", "exec",
|
50
|
+
["bundle", "exec", cmd_path, "-c", conf_path, *fluentd_options]
|
51
51
|
end
|
52
52
|
|
53
|
-
def execute_command(cmdline, chdir=TMP_DIR)
|
53
|
+
def execute_command(cmdline, chdir=TMP_DIR, env = {})
|
54
54
|
null_stream = File.open(File::NULL, 'w')
|
55
55
|
gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
|
56
56
|
|
57
|
-
env = {
|
58
|
-
"BUNDLE_GEMFILE" => gemfile_path,
|
59
|
-
}
|
57
|
+
env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
|
60
58
|
cmdname = cmdline.shift
|
61
59
|
arg0 = "testing-fluentd"
|
62
60
|
# p(here: "executing process", env: env, cmdname: cmdname, arg0: arg0, args: cmdline)
|
@@ -81,12 +79,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
81
79
|
null_stream.close rescue nil
|
82
80
|
end
|
83
81
|
|
84
|
-
def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10)
|
82
|
+
def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
|
85
83
|
matched = false
|
86
84
|
assert_error_msg = "matched correctly"
|
87
85
|
stdio_buf = ""
|
88
86
|
begin
|
89
|
-
execute_command(cmdline) do |pid, stdout|
|
87
|
+
execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
|
90
88
|
begin
|
91
89
|
waiting(timeout) do
|
92
90
|
while process_exist?(pid) && !matched
|
@@ -285,7 +283,7 @@ CONF
|
|
285
283
|
|
286
284
|
assert_fluentd_fails_to_start(
|
287
285
|
create_cmdline(conf_path),
|
288
|
-
"non directory entry exists:#{@root_path}
|
286
|
+
"non directory entry exists:#{@root_path}",
|
289
287
|
)
|
290
288
|
end
|
291
289
|
end
|
@@ -780,6 +778,82 @@ CONF
|
|
780
778
|
)
|
781
779
|
end
|
782
780
|
|
781
|
+
test "multiple values are set to RUBYOPT" do
|
782
|
+
conf = <<CONF
|
783
|
+
<source>
|
784
|
+
@type dummy
|
785
|
+
tag dummy
|
786
|
+
</source>
|
787
|
+
<match>
|
788
|
+
@type null
|
789
|
+
</match>
|
790
|
+
CONF
|
791
|
+
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
792
|
+
assert_log_matches(
|
793
|
+
create_cmdline(conf_path),
|
794
|
+
'#0 fluentd worker is now running worker=0',
|
795
|
+
patterns_not_match: ['(LoadError)'],
|
796
|
+
env: { 'RUBYOPT' => '-rtest-unit -rbundler/setup' },
|
797
|
+
)
|
798
|
+
end
|
799
|
+
|
800
|
+
data(
|
801
|
+
'-E' => '-Eutf-8',
|
802
|
+
'-encoding' => '--encoding=utf-8',
|
803
|
+
'-external-encoding' => '--external-encoding=utf-8',
|
804
|
+
'-internal-encoding' => '--internal-encoding=utf-8',
|
805
|
+
)
|
806
|
+
test "-E option is set to RUBYOPT3" do |opt|
|
807
|
+
conf = <<CONF
|
808
|
+
<source>
|
809
|
+
@type dummy
|
810
|
+
tag dummy
|
811
|
+
</source>
|
812
|
+
<match>
|
813
|
+
@type null
|
814
|
+
</match>
|
815
|
+
CONF
|
816
|
+
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
817
|
+
assert_log_matches(
|
818
|
+
create_cmdline(conf_path),
|
819
|
+
*opt.split(' '),
|
820
|
+
patterns_not_match: ['-Eascii-8bit:ascii-8bit'],
|
821
|
+
env: { 'RUBYOPT' => opt },
|
822
|
+
)
|
823
|
+
end
|
824
|
+
|
825
|
+
test "without RUBYOPT" do
|
826
|
+
conf = <<CONF
|
827
|
+
<source>
|
828
|
+
@type dummy
|
829
|
+
tag dummy
|
830
|
+
</source>
|
831
|
+
<match>
|
832
|
+
@type null
|
833
|
+
</match>
|
834
|
+
CONF
|
835
|
+
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
836
|
+
assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
|
837
|
+
end
|
838
|
+
|
839
|
+
test 'invalid values are set to RUBYOPT' do
|
840
|
+
conf = <<CONF
|
841
|
+
<source>
|
842
|
+
@type dummy
|
843
|
+
tag dummy
|
844
|
+
</source>
|
845
|
+
<match>
|
846
|
+
@type null
|
847
|
+
</match>
|
848
|
+
CONF
|
849
|
+
conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
|
850
|
+
assert_log_matches(
|
851
|
+
create_cmdline(conf_path),
|
852
|
+
'Invalid option is passed to RUBYOPT',
|
853
|
+
env: { 'RUBYOPT' => 'a' },
|
854
|
+
)
|
855
|
+
end
|
856
|
+
|
783
857
|
test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
|
784
858
|
conf = <<CONF
|
785
859
|
<system>
|
@@ -1,9 +1,17 @@
|
|
1
1
|
require_relative '../../helper'
|
2
2
|
require 'fluent/plugin/in_tail/position_file'
|
3
3
|
|
4
|
+
require 'fileutils'
|
5
|
+
require 'tempfile'
|
6
|
+
|
4
7
|
class IntailPositionFileTest < Test::Unit::TestCase
|
5
8
|
setup do
|
6
|
-
@file =
|
9
|
+
@file = Tempfile.new('intail_position_file_test')
|
10
|
+
end
|
11
|
+
|
12
|
+
teardown do
|
13
|
+
@file.close rescue nil
|
14
|
+
@file.unlink rescue nil
|
7
15
|
end
|
8
16
|
|
9
17
|
UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
|
@@ -19,10 +27,21 @@ class IntailPositionFileTest < Test::Unit::TestCase
|
|
19
27
|
f.seek(0)
|
20
28
|
end
|
21
29
|
|
22
|
-
|
30
|
+
test '.load' do
|
31
|
+
write_data(@file, TEST_CONTENT)
|
32
|
+
Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
|
33
|
+
|
34
|
+
@file.seek(0)
|
35
|
+
lines = @file.readlines
|
36
|
+
assert_equal 2, lines.size
|
37
|
+
assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
|
38
|
+
assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
|
39
|
+
end
|
40
|
+
|
41
|
+
sub_test_case '#try_compact' do
|
23
42
|
test 'compact invalid and convert 32 bit inode value' do
|
24
43
|
write_data(@file, TEST_CONTENT)
|
25
|
-
Fluent::Plugin::TailInput::PositionFile.
|
44
|
+
Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
|
26
45
|
|
27
46
|
@file.seek(0)
|
28
47
|
lines = @file.readlines
|
@@ -36,29 +55,59 @@ class IntailPositionFileTest < Test::Unit::TestCase
|
|
36
55
|
valid_path\t0000000000000002\t0000000000000001
|
37
56
|
valid_path\t0000000000000003\t0000000000000004
|
38
57
|
EOF
|
39
|
-
Fluent::Plugin::TailInput::PositionFile.
|
58
|
+
Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
|
40
59
|
|
41
60
|
@file.seek(0)
|
42
61
|
lines = @file.readlines
|
43
62
|
assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
|
44
63
|
end
|
64
|
+
|
65
|
+
test 'does not change when the file is changed' do
|
66
|
+
write_data(@file, TEST_CONTENT)
|
67
|
+
pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
|
68
|
+
|
69
|
+
mock.proxy(pf).fetch_compacted_entries do |r|
|
70
|
+
@file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
|
71
|
+
r
|
72
|
+
end
|
73
|
+
|
74
|
+
pf.try_compact
|
75
|
+
|
76
|
+
@file.seek(0)
|
77
|
+
lines = @file.readlines
|
78
|
+
assert_equal 5, lines.size
|
79
|
+
end
|
45
80
|
end
|
46
81
|
|
47
|
-
|
48
|
-
|
49
|
-
|
82
|
+
sub_test_case '#load' do
|
83
|
+
test 'compact invalid and convert 32 bit inode value' do
|
84
|
+
write_data(@file, TEST_CONTENT)
|
85
|
+
Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
|
50
86
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
87
|
+
@file.seek(0)
|
88
|
+
lines = @file.readlines
|
89
|
+
assert_equal 2, lines.size
|
90
|
+
assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
|
91
|
+
assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
|
92
|
+
end
|
93
|
+
|
94
|
+
test 'compact data if duplicated line' do
|
95
|
+
write_data(@file, <<~EOF)
|
96
|
+
valid_path\t0000000000000002\t0000000000000001
|
97
|
+
valid_path\t0000000000000003\t0000000000000004
|
98
|
+
EOF
|
99
|
+
Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
|
100
|
+
|
101
|
+
@file.seek(0)
|
102
|
+
lines = @file.readlines
|
103
|
+
assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
|
104
|
+
end
|
56
105
|
end
|
57
106
|
|
58
107
|
sub_test_case '#[]' do
|
59
108
|
test 'return entry' do
|
60
109
|
write_data(@file, TEST_CONTENT)
|
61
|
-
pf = Fluent::Plugin::TailInput::PositionFile.
|
110
|
+
pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
|
62
111
|
|
63
112
|
f = pf['valid_path']
|
64
113
|
assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
|
@@ -82,7 +131,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
|
|
82
131
|
|
83
132
|
test 'does not change other value position if other entry try to write' do
|
84
133
|
write_data(@file, TEST_CONTENT)
|
85
|
-
pf = Fluent::Plugin::TailInput::PositionFile.
|
134
|
+
pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
|
86
135
|
|
87
136
|
f = pf['nonexist_path']
|
88
137
|
assert_equal 0, f.read_inode
|
@@ -103,7 +152,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
|
|
103
152
|
sub_test_case '#unwatch' do
|
104
153
|
test 'deletes entry by path' do
|
105
154
|
write_data(@file, TEST_CONTENT)
|
106
|
-
pf = Fluent::Plugin::TailInput::PositionFile.
|
155
|
+
pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
|
107
156
|
p1 = pf['valid_path']
|
108
157
|
assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
|
109
158
|
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -1088,7 +1088,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1088
1088
|
plugin = create_driver(EX_CONFIG, false).instance
|
1089
1089
|
sio = StringIO.new
|
1090
1090
|
plugin.instance_eval do
|
1091
|
-
@pf = Fluent::Plugin::TailInput::PositionFile.
|
1091
|
+
@pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
|
1092
1092
|
@loop = Coolio::Loop.new
|
1093
1093
|
end
|
1094
1094
|
|
@@ -237,7 +237,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
237
237
|
assert_equal "-", record["extradata"]
|
238
238
|
assert_equal "Hi, from Fluentd!", record["message"]
|
239
239
|
end
|
240
|
-
assert_equal(Fluent::Plugin::SyslogParser::
|
240
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
|
241
241
|
@parser.instance.patterns['format'])
|
242
242
|
end
|
243
243
|
|
@@ -254,7 +254,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
254
254
|
assert_equal "-", record["extradata"]
|
255
255
|
assert_nil record["message"]
|
256
256
|
end
|
257
|
-
assert_equal(Fluent::Plugin::SyslogParser::
|
257
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
|
258
258
|
@parser.instance.patterns['format'])
|
259
259
|
end
|
260
260
|
|
@@ -294,14 +294,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
294
294
|
'message_format' => 'rfc5424',
|
295
295
|
'with_priority' => true,
|
296
296
|
)
|
297
|
-
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi
|
297
|
+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd!'
|
298
298
|
@parser.instance.parse(text) do |time, record|
|
299
299
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
300
300
|
assert_equal "11111", record["pid"]
|
301
301
|
assert_equal "ID24224", record["msgid"]
|
302
302
|
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
|
303
303
|
record["extradata"]
|
304
|
-
assert_equal "Hi
|
304
|
+
assert_equal "[Hi] from Fluentd!", record["message"]
|
305
305
|
end
|
306
306
|
end
|
307
307
|
|
@@ -328,14 +328,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
328
328
|
'message_format' => 'rfc5424',
|
329
329
|
'with_priority' => true,
|
330
330
|
)
|
331
|
-
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi
|
331
|
+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd]!'
|
332
332
|
@parser.instance.parse(text) do |time, record|
|
333
333
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
334
334
|
assert_equal "11111", record["pid"]
|
335
335
|
assert_equal "ID24224", record["msgid"]
|
336
336
|
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
|
337
337
|
record["extradata"]
|
338
|
-
assert_equal "Hi
|
338
|
+
assert_equal "[Hi] from Fluentd]!", record["message"]
|
339
339
|
end
|
340
340
|
end
|
341
341
|
|
data/test/test_supervisor.rb
CHANGED
@@ -235,6 +235,33 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
235
235
|
Timecop.return
|
236
236
|
end
|
237
237
|
|
238
|
+
def test_load_config_for_logger
|
239
|
+
tmp_dir = "#{TMP_DIR}/dir/test_load_config_log.conf"
|
240
|
+
conf_info_str = %[
|
241
|
+
<system>
|
242
|
+
<log>
|
243
|
+
format json
|
244
|
+
time_format %FT%T.%L%z
|
245
|
+
</log>
|
246
|
+
</system>
|
247
|
+
]
|
248
|
+
write_config tmp_dir, conf_info_str
|
249
|
+
params = {
|
250
|
+
'use_v1_config' => true,
|
251
|
+
'conf_encoding' => 'utf8',
|
252
|
+
'log_level' => Fluent::Log::LEVEL_INFO,
|
253
|
+
'log_path' => 'test/tmp/supervisor/log',
|
254
|
+
|
255
|
+
'workers' => 1,
|
256
|
+
'log_format' => :json,
|
257
|
+
'log_time_format' => '%FT%T.%L%z',
|
258
|
+
}
|
259
|
+
|
260
|
+
r = Fluent::Supervisor.load_config(tmp_dir, params)
|
261
|
+
assert_equal :json, r[:logger].format
|
262
|
+
assert_equal '%FT%T.%L%z', r[:logger].time_format
|
263
|
+
end
|
264
|
+
|
238
265
|
def test_load_config_for_daemonize
|
239
266
|
tmp_dir = "#{TMP_DIR}/dir/test_load_config.conf"
|
240
267
|
conf_info_str = %[
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.9.
|
4
|
+
version: 1.9.2
|
5
5
|
platform: x64-mingw32
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|