fluentd 1.16.3 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
- data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
- data/.github/workflows/test-ruby-head.yml +31 -0
- data/.github/workflows/{linux-test.yaml → test.yml} +9 -13
- data/CHANGELOG.md +66 -0
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/fluentd.gemspec +9 -1
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/config/configure_proxy.rb +2 -2
- data/lib/fluent/config/types.rb +1 -1
- data/lib/fluent/configurable.rb +2 -2
- data/lib/fluent/counter/mutex_hash.rb +1 -1
- data/lib/fluent/fluent_log_event_router.rb +0 -2
- data/lib/fluent/plugin/buf_file.rb +1 -1
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
- data/lib/fluent/plugin/buffer.rb +95 -86
- data/lib/fluent/plugin/filter_parser.rb +26 -8
- data/lib/fluent/plugin/in_http.rb +18 -53
- data/lib/fluent/plugin/in_tail.rb +52 -8
- data/lib/fluent/plugin/out_http.rb +125 -13
- data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
- data/lib/fluent/plugin/parser_json.rb +22 -5
- data/lib/fluent/plugin/parser_msgpack.rb +24 -3
- data/lib/fluent/plugin_helper/metrics.rb +2 -2
- data/lib/fluent/registry.rb +6 -6
- data/lib/fluent/test/output_test.rb +1 -1
- data/lib/fluent/unique_id.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +8 -2
- data/test/log/test_console_adapter.rb +10 -3
- data/test/plugin/data/log_numeric/01.log +0 -0
- data/test/plugin/data/log_numeric/02.log +0 -0
- data/test/plugin/data/log_numeric/12.log +0 -0
- data/test/plugin/data/log_numeric/14.log +0 -0
- data/test/plugin/test_buffer.rb +110 -0
- data/test/plugin/test_in_http.rb +23 -1
- data/test/plugin/test_in_tail.rb +307 -3
- data/test/plugin/test_out_forward.rb +34 -39
- data/test/plugin/test_out_http.rb +128 -0
- data/test/plugin/test_owned_by.rb +0 -1
- data/test/plugin/test_parser_json.rb +106 -0
- data/test/plugin/test_parser_msgpack.rb +127 -0
- data/test/plugin/test_storage.rb +0 -1
- data/test/plugin_helper/test_child_process.rb +17 -7
- metadata +104 -9
- data/.github/workflows/macos-test.yaml +0 -34
- data/.github/workflows/windows-test.yaml +0 -49
- /data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +0 -0
- /data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +0 -0
data/test/plugin/test_in_tail.rb
CHANGED
|
@@ -1538,6 +1538,147 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1538
1538
|
assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
|
|
1539
1539
|
end
|
|
1540
1540
|
|
|
1541
|
+
sub_test_case "expand_paths with glob" do |data|
|
|
1542
|
+
sub_test_case "extended_glob" do
|
|
1543
|
+
data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
|
|
1544
|
+
"square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
|
1545
|
+
"asterisk" => [true, "always", "test/plugin/data/log/*.log"],
|
|
1546
|
+
"one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
|
|
1547
|
+
)
|
|
1548
|
+
def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
|
|
1549
|
+
result, option, path = data
|
|
1550
|
+
config = config_element("", "", {
|
|
1551
|
+
"tag" => "tail",
|
|
1552
|
+
"path" => path,
|
|
1553
|
+
"format" => "none",
|
|
1554
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1555
|
+
"read_from_head" => true,
|
|
1556
|
+
"refresh_interval" => 30,
|
|
1557
|
+
"glob_policy" => option,
|
|
1558
|
+
"path_delimiter" => "|",
|
|
1559
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1560
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1561
|
+
})
|
|
1562
|
+
plugin = create_driver(config, false).instance
|
|
1563
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
|
1564
|
+
end
|
|
1565
|
+
|
|
1566
|
+
data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
|
|
1567
|
+
"square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
|
1568
|
+
"asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
|
|
1569
|
+
"one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
|
|
1570
|
+
)
|
|
1571
|
+
def test_expand_paths_with_use_glob_p
|
|
1572
|
+
emit_exception_p, result, option, path = data
|
|
1573
|
+
config = config_element("", "", {
|
|
1574
|
+
"tag" => "tail",
|
|
1575
|
+
"path" => path,
|
|
1576
|
+
"format" => "none",
|
|
1577
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1578
|
+
"read_from_head" => true,
|
|
1579
|
+
"refresh_interval" => 30,
|
|
1580
|
+
"glob_policy" => option,
|
|
1581
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1582
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1583
|
+
})
|
|
1584
|
+
if emit_exception_p
|
|
1585
|
+
assert_raise(Fluent::ConfigError) do
|
|
1586
|
+
plugin = create_driver(config, false).instance
|
|
1587
|
+
end
|
|
1588
|
+
else
|
|
1589
|
+
plugin = create_driver(config, false).instance
|
|
1590
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
|
1591
|
+
end
|
|
1592
|
+
end
|
|
1593
|
+
end
|
|
1594
|
+
|
|
1595
|
+
sub_test_case "only_use_backward_compatible" do
|
|
1596
|
+
data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
|
1597
|
+
"asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
|
|
1598
|
+
"one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
|
|
1599
|
+
)
|
|
1600
|
+
def test_expand_paths_with_use_glob_p
|
|
1601
|
+
result, option, path = data
|
|
1602
|
+
config = config_element("", "", {
|
|
1603
|
+
"tag" => "tail",
|
|
1604
|
+
"path" => path,
|
|
1605
|
+
"format" => "none",
|
|
1606
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1607
|
+
"read_from_head" => true,
|
|
1608
|
+
"refresh_interval" => 30,
|
|
1609
|
+
"glob_policy" => option,
|
|
1610
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1611
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1612
|
+
})
|
|
1613
|
+
plugin = create_driver(config, false).instance
|
|
1614
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
|
1615
|
+
end
|
|
1616
|
+
end
|
|
1617
|
+
end
|
|
1618
|
+
|
|
1619
|
+
def ex_config_with_brackets
|
|
1620
|
+
config_element("", "", {
|
|
1621
|
+
"tag" => "tail",
|
|
1622
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
|
1623
|
+
"format" => "none",
|
|
1624
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1625
|
+
"read_from_head" => true,
|
|
1626
|
+
"refresh_interval" => 30,
|
|
1627
|
+
"glob_policy" => "extended",
|
|
1628
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1629
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1630
|
+
})
|
|
1631
|
+
end
|
|
1632
|
+
|
|
1633
|
+
def test_config_with_always_with_default_delimiter
|
|
1634
|
+
assert_raise(Fluent::ConfigError) do
|
|
1635
|
+
config = config_element("", "", {
|
|
1636
|
+
"tag" => "tail",
|
|
1637
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
|
1638
|
+
"format" => "none",
|
|
1639
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1640
|
+
"read_from_head" => true,
|
|
1641
|
+
"refresh_interval" => 30,
|
|
1642
|
+
"glob_policy" => "always",
|
|
1643
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1644
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1645
|
+
})
|
|
1646
|
+
|
|
1647
|
+
create_driver(config, false).instance
|
|
1648
|
+
end
|
|
1649
|
+
end
|
|
1650
|
+
|
|
1651
|
+
def test_config_with_always_with_custom_delimiter
|
|
1652
|
+
assert_nothing_raised do
|
|
1653
|
+
config = config_element("", "", {
|
|
1654
|
+
"tag" => "tail",
|
|
1655
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
|
1656
|
+
"format" => "none",
|
|
1657
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
1658
|
+
"read_from_head" => true,
|
|
1659
|
+
"refresh_interval" => 30,
|
|
1660
|
+
"glob_policy" => "always",
|
|
1661
|
+
"path_delimiter" => "|",
|
|
1662
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
|
1663
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
|
1664
|
+
})
|
|
1665
|
+
|
|
1666
|
+
create_driver(config, false).instance
|
|
1667
|
+
end
|
|
1668
|
+
end
|
|
1669
|
+
|
|
1670
|
+
def test_expand_paths_with_brackets
|
|
1671
|
+
expanded_paths = [
|
|
1672
|
+
create_target_info('test/plugin/data/log_numeric/01.log'),
|
|
1673
|
+
create_target_info('test/plugin/data/log_numeric/02.log'),
|
|
1674
|
+
create_target_info('test/plugin/data/log_numeric/12.log'),
|
|
1675
|
+
create_target_info('test/plugin/data/log_numeric/14.log'),
|
|
1676
|
+
]
|
|
1677
|
+
|
|
1678
|
+
plugin = create_driver(ex_config_with_brackets, false).instance
|
|
1679
|
+
assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
|
|
1680
|
+
end
|
|
1681
|
+
|
|
1541
1682
|
def test_expand_paths_with_duplicate_configuration
|
|
1542
1683
|
expanded_paths = [
|
|
1543
1684
|
create_target_info('test/plugin/data/log/foo/bar.log'),
|
|
@@ -3016,6 +3157,92 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
3016
3157
|
},
|
|
3017
3158
|
)
|
|
3018
3159
|
end
|
|
3160
|
+
|
|
3161
|
+
def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
|
|
3162
|
+
config = config_element(
|
|
3163
|
+
"ROOT",
|
|
3164
|
+
"",
|
|
3165
|
+
{
|
|
3166
|
+
"path" => "#{@tmp_dir}/tail.txt*",
|
|
3167
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
3168
|
+
"tag" => "t1",
|
|
3169
|
+
"format" => "none",
|
|
3170
|
+
"read_from_head" => "true",
|
|
3171
|
+
"follow_inodes" => "true",
|
|
3172
|
+
"rotate_wait" => "3s",
|
|
3173
|
+
"refresh_interval" => "1h",
|
|
3174
|
+
# stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
|
|
3175
|
+
# so disable it in order to reproduce the same condition stably.
|
|
3176
|
+
"enable_stat_watcher" => "false",
|
|
3177
|
+
}
|
|
3178
|
+
)
|
|
3179
|
+
d = create_driver(config, false)
|
|
3180
|
+
|
|
3181
|
+
tail_watchers = []
|
|
3182
|
+
stub.proxy(d.instance).setup_watcher do |tw|
|
|
3183
|
+
tail_watchers.append(tw)
|
|
3184
|
+
mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
|
|
3185
|
+
tw
|
|
3186
|
+
end
|
|
3187
|
+
|
|
3188
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
|
|
3189
|
+
|
|
3190
|
+
d.run(expect_records: 6, timeout: 15) do
|
|
3191
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
|
|
3192
|
+
|
|
3193
|
+
sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
|
|
3194
|
+
|
|
3195
|
+
FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
|
|
3196
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
|
|
3197
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
|
|
3198
|
+
|
|
3199
|
+
sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
|
|
3200
|
+
|
|
3201
|
+
# Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
|
|
3202
|
+
[1, 0].each do |i|
|
|
3203
|
+
FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
|
|
3204
|
+
end
|
|
3205
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
|
|
3206
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
|
|
3207
|
+
|
|
3208
|
+
# Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
|
|
3209
|
+
# (Note: Currently, there is no harm in duplicate calls)
|
|
3210
|
+
sleep 4
|
|
3211
|
+
end
|
|
3212
|
+
|
|
3213
|
+
inode_0 = tail_watchers[0]&.ino
|
|
3214
|
+
inode_1 = tail_watchers[1]&.ino
|
|
3215
|
+
inode_2 = tail_watchers[2]&.ino
|
|
3216
|
+
record_values = d.events.collect { |event| event[2]["message"] }.sort
|
|
3217
|
+
position_entries = []
|
|
3218
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
|
|
3219
|
+
f.readlines(chomp: true).each do |line|
|
|
3220
|
+
values = line.split("\t")
|
|
3221
|
+
position_entries.append([values[0], values[1], values[2].to_i(16)])
|
|
3222
|
+
end
|
|
3223
|
+
end
|
|
3224
|
+
|
|
3225
|
+
assert_equal(
|
|
3226
|
+
{
|
|
3227
|
+
record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
|
|
3228
|
+
tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
|
|
3229
|
+
tail_watcher_inodes: [inode_0, inode_1, inode_2],
|
|
3230
|
+
tail_watcher_io_handler_opened_statuses: [false, false, false],
|
|
3231
|
+
position_entries: [
|
|
3232
|
+
["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
|
|
3233
|
+
["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
|
|
3234
|
+
["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
|
|
3235
|
+
],
|
|
3236
|
+
},
|
|
3237
|
+
{
|
|
3238
|
+
record_values: record_values,
|
|
3239
|
+
tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
|
|
3240
|
+
tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
|
|
3241
|
+
tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
|
|
3242
|
+
position_entries: position_entries
|
|
3243
|
+
},
|
|
3244
|
+
)
|
|
3245
|
+
end
|
|
3019
3246
|
end
|
|
3020
3247
|
|
|
3021
3248
|
sub_test_case "Update watchers for rotation without follow_inodes" do
|
|
@@ -3084,9 +3311,6 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
3084
3311
|
sleep 3
|
|
3085
3312
|
|
|
3086
3313
|
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
|
|
3087
|
-
|
|
3088
|
-
# Wait `rotate_wait` for file2 to make sure to close all IO handlers
|
|
3089
|
-
sleep 3
|
|
3090
3314
|
end
|
|
3091
3315
|
|
|
3092
3316
|
inode_0 = tail_watchers[0]&.ino
|
|
@@ -3121,5 +3345,85 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
3121
3345
|
},
|
|
3122
3346
|
)
|
|
3123
3347
|
end
|
|
3348
|
+
|
|
3349
|
+
def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
|
|
3350
|
+
config = config_element(
|
|
3351
|
+
"ROOT",
|
|
3352
|
+
"",
|
|
3353
|
+
{
|
|
3354
|
+
"path" => "#{@tmp_dir}/tail.txt0",
|
|
3355
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
|
3356
|
+
"tag" => "t1",
|
|
3357
|
+
"format" => "none",
|
|
3358
|
+
"read_from_head" => "true",
|
|
3359
|
+
"rotate_wait" => "3s",
|
|
3360
|
+
"refresh_interval" => "1h",
|
|
3361
|
+
}
|
|
3362
|
+
)
|
|
3363
|
+
d = create_driver(config, false)
|
|
3364
|
+
|
|
3365
|
+
tail_watchers = []
|
|
3366
|
+
stub.proxy(d.instance).setup_watcher do |tw|
|
|
3367
|
+
tail_watchers.append(tw)
|
|
3368
|
+
mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
|
|
3369
|
+
tw
|
|
3370
|
+
end
|
|
3371
|
+
|
|
3372
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
|
|
3373
|
+
|
|
3374
|
+
d.run(expect_records: 6, timeout: 15) do
|
|
3375
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
|
|
3376
|
+
|
|
3377
|
+
sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
|
|
3378
|
+
|
|
3379
|
+
FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
|
|
3380
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
|
|
3381
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
|
|
3382
|
+
|
|
3383
|
+
sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
|
|
3384
|
+
|
|
3385
|
+
# Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
|
|
3386
|
+
[1, 0].each do |i|
|
|
3387
|
+
FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
|
|
3388
|
+
end
|
|
3389
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
|
|
3390
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
|
|
3391
|
+
|
|
3392
|
+
# Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
|
|
3393
|
+
# (Note: Currently, there is no harm in duplicate calls)
|
|
3394
|
+
sleep 4
|
|
3395
|
+
end
|
|
3396
|
+
|
|
3397
|
+
inode_0 = tail_watchers[0]&.ino
|
|
3398
|
+
inode_1 = tail_watchers[1]&.ino
|
|
3399
|
+
inode_2 = tail_watchers[2]&.ino
|
|
3400
|
+
record_values = d.events.collect { |event| event[2]["message"] }.sort
|
|
3401
|
+
position_entries = []
|
|
3402
|
+
Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
|
|
3403
|
+
f.readlines(chomp: true).each do |line|
|
|
3404
|
+
values = line.split("\t")
|
|
3405
|
+
position_entries.append([values[0], values[1], values[2].to_i(16)])
|
|
3406
|
+
end
|
|
3407
|
+
end
|
|
3408
|
+
|
|
3409
|
+
assert_equal(
|
|
3410
|
+
{
|
|
3411
|
+
record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
|
|
3412
|
+
tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
|
|
3413
|
+
tail_watcher_inodes: [inode_0, inode_1, inode_2],
|
|
3414
|
+
tail_watcher_io_handler_opened_statuses: [false, false, false],
|
|
3415
|
+
position_entries: [
|
|
3416
|
+
["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
|
|
3417
|
+
],
|
|
3418
|
+
},
|
|
3419
|
+
{
|
|
3420
|
+
record_values: record_values,
|
|
3421
|
+
tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
|
|
3422
|
+
tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
|
|
3423
|
+
tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
|
|
3424
|
+
position_entries: position_entries
|
|
3425
|
+
},
|
|
3426
|
+
)
|
|
3427
|
+
end
|
|
3124
3428
|
end
|
|
3125
3429
|
end
|
|
@@ -156,7 +156,14 @@ EOL
|
|
|
156
156
|
normal_conf = config_element('match', '**', {}, [
|
|
157
157
|
config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})
|
|
158
158
|
])
|
|
159
|
-
|
|
159
|
+
|
|
160
|
+
if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3
|
|
161
|
+
error_class = Socket::ResolutionError
|
|
162
|
+
else
|
|
163
|
+
error_class = SocketError
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
assert_raise error_class do
|
|
160
167
|
create_driver(normal_conf)
|
|
161
168
|
end
|
|
162
169
|
|
|
@@ -165,7 +172,7 @@ EOL
|
|
|
165
172
|
])
|
|
166
173
|
@d = d = create_driver(conf)
|
|
167
174
|
expected_log = "failed to resolve node name when configured"
|
|
168
|
-
expected_detail =
|
|
175
|
+
expected_detail = "server=\"test\" error_class=#{error_class.name}"
|
|
169
176
|
logs = d.logs
|
|
170
177
|
assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
|
|
171
178
|
end
|
|
@@ -1241,27 +1248,22 @@ EOL
|
|
|
1241
1248
|
target_input_driver = create_target_input_driver(conf: target_config)
|
|
1242
1249
|
output_conf = config
|
|
1243
1250
|
d = create_driver(output_conf)
|
|
1244
|
-
d.instance_start
|
|
1245
1251
|
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
) { |sock| mock(sock).close.once; sock }.twice
|
|
1252
|
+
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
|
1253
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
|
|
1254
|
+
linger_timeout: anything,
|
|
1255
|
+
send_timeout: anything,
|
|
1256
|
+
recv_timeout: anything,
|
|
1257
|
+
connect_timeout: anything
|
|
1258
|
+
) { |sock| mock(sock).close.once; sock }.twice
|
|
1254
1259
|
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
end
|
|
1260
|
+
target_input_driver.run(timeout: 15) do
|
|
1261
|
+
d.run do
|
|
1262
|
+
node = d.instance.nodes.first
|
|
1263
|
+
2.times do
|
|
1264
|
+
node.send_data('test', chunk) rescue nil
|
|
1261
1265
|
end
|
|
1262
1266
|
end
|
|
1263
|
-
ensure
|
|
1264
|
-
d.instance_shutdown
|
|
1265
1267
|
end
|
|
1266
1268
|
end
|
|
1267
1269
|
|
|
@@ -1275,7 +1277,6 @@ EOL
|
|
|
1275
1277
|
port #{@target_port}
|
|
1276
1278
|
</server>
|
|
1277
1279
|
])
|
|
1278
|
-
d.instance_start
|
|
1279
1280
|
assert_nothing_raised { d.run }
|
|
1280
1281
|
end
|
|
1281
1282
|
|
|
@@ -1287,33 +1288,28 @@ EOL
|
|
|
1287
1288
|
keepalive_timeout 2
|
|
1288
1289
|
]
|
|
1289
1290
|
d = create_driver(output_conf)
|
|
1290
|
-
d.instance_start
|
|
1291
1291
|
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
) { |sock| mock(sock).close.once; sock }.once
|
|
1292
|
+
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
|
1293
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
|
|
1294
|
+
linger_timeout: anything,
|
|
1295
|
+
send_timeout: anything,
|
|
1296
|
+
recv_timeout: anything,
|
|
1297
|
+
connect_timeout: anything
|
|
1298
|
+
) { |sock| mock(sock).close.once; sock }.once
|
|
1300
1299
|
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
end
|
|
1300
|
+
target_input_driver.run(timeout: 15) do
|
|
1301
|
+
d.run do
|
|
1302
|
+
node = d.instance.nodes.first
|
|
1303
|
+
2.times do
|
|
1304
|
+
node.send_data('test', chunk) rescue nil
|
|
1307
1305
|
end
|
|
1308
1306
|
end
|
|
1309
|
-
ensure
|
|
1310
|
-
d.instance_shutdown
|
|
1311
1307
|
end
|
|
1312
1308
|
end
|
|
1313
1309
|
|
|
1314
1310
|
test 'create timer of purging obsolete sockets' do
|
|
1315
1311
|
output_conf = config + %[keepalive true]
|
|
1316
|
-
d = create_driver(output_conf)
|
|
1312
|
+
@d = d = create_driver(output_conf)
|
|
1317
1313
|
|
|
1318
1314
|
mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
|
|
1319
1315
|
mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
|
|
@@ -1329,7 +1325,6 @@ EOL
|
|
|
1329
1325
|
keepalive_timeout 2
|
|
1330
1326
|
]
|
|
1331
1327
|
d = create_driver(output_conf)
|
|
1332
|
-
d.instance_start
|
|
1333
1328
|
|
|
1334
1329
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
|
1335
1330
|
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
|
|
@@ -7,6 +7,7 @@ require 'webrick/https'
|
|
|
7
7
|
require 'net/http'
|
|
8
8
|
require 'uri'
|
|
9
9
|
require 'json'
|
|
10
|
+
require 'aws-sdk-core'
|
|
10
11
|
|
|
11
12
|
# WEBrick's ProcHandler doesn't handle PUT by default
|
|
12
13
|
module WEBrick::HTTPServlet
|
|
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
|
390
391
|
end
|
|
391
392
|
end
|
|
392
393
|
|
|
394
|
+
|
|
395
|
+
sub_test_case 'aws sigv4 auth' do
|
|
396
|
+
setup do
|
|
397
|
+
@@fake_aws_credentials = Aws::Credentials.new(
|
|
398
|
+
'fakeaccess',
|
|
399
|
+
'fakesecret',
|
|
400
|
+
'fake session token'
|
|
401
|
+
)
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
def server_port
|
|
405
|
+
19883
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
def test_aws_sigv4_sts_role_arn
|
|
409
|
+
stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
|
|
410
|
+
stub(credentials_provider).credentials {
|
|
411
|
+
@@fake_aws_credentials
|
|
412
|
+
}
|
|
413
|
+
credentials_provider
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
d = create_driver(config + %[
|
|
417
|
+
<auth>
|
|
418
|
+
method aws_sigv4
|
|
419
|
+
aws_service someservice
|
|
420
|
+
aws_region my-region-1
|
|
421
|
+
aws_role_arn arn:aws:iam::123456789012:role/MyRole
|
|
422
|
+
</auth>
|
|
423
|
+
])
|
|
424
|
+
d.run(default_tag: 'test.http') do
|
|
425
|
+
test_events.each { |event|
|
|
426
|
+
d.feed(event)
|
|
427
|
+
}
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
result = @@result
|
|
431
|
+
assert_equal 'POST', result.method
|
|
432
|
+
assert_equal 'application/x-ndjson', result.content_type
|
|
433
|
+
assert_equal test_events, result.data
|
|
434
|
+
assert_not_empty result.headers
|
|
435
|
+
assert_not_nil result.headers['authorization']
|
|
436
|
+
assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
|
|
437
|
+
assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
|
|
438
|
+
assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
|
|
439
|
+
assert_not_nil result.headers['x-amz-content-sha256']
|
|
440
|
+
assert_not_empty result.headers['x-amz-content-sha256']
|
|
441
|
+
assert_not_nil result.headers['x-amz-security-token']
|
|
442
|
+
assert_not_empty result.headers['x-amz-security-token']
|
|
443
|
+
assert_not_nil result.headers['x-amz-date']
|
|
444
|
+
assert_not_empty result.headers['x-amz-date']
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def test_aws_sigv4_no_role
|
|
448
|
+
stub(Aws::CredentialProviderChain).new do |provider_chain|
|
|
449
|
+
stub(provider_chain).resolve {
|
|
450
|
+
@@fake_aws_credentials
|
|
451
|
+
}
|
|
452
|
+
provider_chain
|
|
453
|
+
end
|
|
454
|
+
d = create_driver(config + %[
|
|
455
|
+
<auth>
|
|
456
|
+
method aws_sigv4
|
|
457
|
+
aws_service someservice
|
|
458
|
+
aws_region my-region-1
|
|
459
|
+
</auth>
|
|
460
|
+
])
|
|
461
|
+
d.run(default_tag: 'test.http') do
|
|
462
|
+
test_events.each { |event|
|
|
463
|
+
d.feed(event)
|
|
464
|
+
}
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
result = @@result
|
|
468
|
+
assert_equal 'POST', result.method
|
|
469
|
+
assert_equal 'application/x-ndjson', result.content_type
|
|
470
|
+
assert_equal test_events, result.data
|
|
471
|
+
assert_not_empty result.headers
|
|
472
|
+
assert_not_nil result.headers['authorization']
|
|
473
|
+
assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
|
|
474
|
+
assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
|
|
475
|
+
assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
|
|
476
|
+
assert_not_nil result.headers['x-amz-content-sha256']
|
|
477
|
+
assert_not_empty result.headers['x-amz-content-sha256']
|
|
478
|
+
assert_not_nil result.headers['x-amz-security-token']
|
|
479
|
+
assert_not_empty result.headers['x-amz-security-token']
|
|
480
|
+
assert_not_nil result.headers['x-amz-date']
|
|
481
|
+
assert_not_empty result.headers['x-amz-date']
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
|
|
393
485
|
sub_test_case 'HTTPS' do
|
|
394
486
|
def server_port
|
|
395
487
|
19882
|
|
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
|
426
518
|
assert_not_empty result.headers
|
|
427
519
|
end
|
|
428
520
|
end
|
|
521
|
+
|
|
522
|
+
sub_test_case 'connection_reuse' do
|
|
523
|
+
def server_port
|
|
524
|
+
19883
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
def test_connection_recreation
|
|
528
|
+
d = create_driver(%[
|
|
529
|
+
endpoint http://127.0.0.1:#{server_port}/test
|
|
530
|
+
reuse_connections true
|
|
531
|
+
])
|
|
532
|
+
|
|
533
|
+
d.run(default_tag: 'test.http', shutdown: false) do
|
|
534
|
+
d.feed(test_events[0])
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
data = @@result.data
|
|
538
|
+
|
|
539
|
+
# Restart server to simulate connection loss
|
|
540
|
+
@@http_server_thread.kill
|
|
541
|
+
@@http_server_thread.join
|
|
542
|
+
@@http_server_thread = Thread.new do
|
|
543
|
+
run_http_server
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
d.run(default_tag: 'test.http') do
|
|
547
|
+
d.feed(test_events[1])
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
result = @@result
|
|
551
|
+
assert_equal 'POST', result.method
|
|
552
|
+
assert_equal 'application/x-ndjson', result.content_type
|
|
553
|
+
assert_equal test_events, data.concat(result.data)
|
|
554
|
+
assert_not_empty result.headers
|
|
555
|
+
end
|
|
556
|
+
end
|
|
429
557
|
end
|
|
@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
|
|
|
26
26
|
|
|
27
27
|
assert_equal parent.object_id, child.owner.object_id
|
|
28
28
|
|
|
29
|
-
assert child.instance_eval{ @_plugin_id_configured }
|
|
30
29
|
assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
|
|
31
30
|
|
|
32
31
|
assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
|