fluentd 1.16.3 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
  3. data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
  4. data/.github/workflows/test-ruby-head.yml +31 -0
  5. data/.github/workflows/{linux-test.yaml → test.yml} +9 -13
  6. data/CHANGELOG.md +66 -0
  7. data/README.md +1 -1
  8. data/Rakefile +1 -1
  9. data/fluentd.gemspec +9 -1
  10. data/lib/fluent/command/binlog_reader.rb +1 -1
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/configurable.rb +2 -2
  14. data/lib/fluent/counter/mutex_hash.rb +1 -1
  15. data/lib/fluent/fluent_log_event_router.rb +0 -2
  16. data/lib/fluent/plugin/buf_file.rb +1 -1
  17. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  18. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  19. data/lib/fluent/plugin/buffer.rb +95 -86
  20. data/lib/fluent/plugin/filter_parser.rb +26 -8
  21. data/lib/fluent/plugin/in_http.rb +18 -53
  22. data/lib/fluent/plugin/in_tail.rb +52 -8
  23. data/lib/fluent/plugin/out_http.rb +125 -13
  24. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  25. data/lib/fluent/plugin/parser_json.rb +22 -5
  26. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  27. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  28. data/lib/fluent/registry.rb +6 -6
  29. data/lib/fluent/test/output_test.rb +1 -1
  30. data/lib/fluent/unique_id.rb +1 -1
  31. data/lib/fluent/version.rb +1 -1
  32. data/test/command/test_fluentd.rb +8 -2
  33. data/test/log/test_console_adapter.rb +10 -3
  34. data/test/plugin/data/log_numeric/01.log +0 -0
  35. data/test/plugin/data/log_numeric/02.log +0 -0
  36. data/test/plugin/data/log_numeric/12.log +0 -0
  37. data/test/plugin/data/log_numeric/14.log +0 -0
  38. data/test/plugin/test_buffer.rb +110 -0
  39. data/test/plugin/test_in_http.rb +23 -1
  40. data/test/plugin/test_in_tail.rb +307 -3
  41. data/test/plugin/test_out_forward.rb +34 -39
  42. data/test/plugin/test_out_http.rb +128 -0
  43. data/test/plugin/test_owned_by.rb +0 -1
  44. data/test/plugin/test_parser_json.rb +106 -0
  45. data/test/plugin/test_parser_msgpack.rb +127 -0
  46. data/test/plugin/test_storage.rb +0 -1
  47. data/test/plugin_helper/test_child_process.rb +17 -7
  48. metadata +104 -9
  49. data/.github/workflows/macos-test.yaml +0 -34
  50. data/.github/workflows/windows-test.yaml +0 -49
  51. /data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +0 -0
  52. /data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +0 -0
@@ -1538,6 +1538,147 @@ class TailInputTest < Test::Unit::TestCase
1538
1538
  assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1539
1539
  end
1540
1540
 
1541
+ sub_test_case "expand_paths with glob" do |data|
1542
+ sub_test_case "extended_glob" do
1543
+ data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
1544
+ "square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1545
+ "asterisk" => [true, "always", "test/plugin/data/log/*.log"],
1546
+ "one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
1547
+ )
1548
+ def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
1549
+ result, option, path = data
1550
+ config = config_element("", "", {
1551
+ "tag" => "tail",
1552
+ "path" => path,
1553
+ "format" => "none",
1554
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1555
+ "read_from_head" => true,
1556
+ "refresh_interval" => 30,
1557
+ "glob_policy" => option,
1558
+ "path_delimiter" => "|",
1559
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1560
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1561
+ })
1562
+ plugin = create_driver(config, false).instance
1563
+ assert_equal(result, !!plugin.use_glob?(path))
1564
+ end
1565
+
1566
+ data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
1567
+ "square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1568
+ "asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
1569
+ "one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
1570
+ )
1571
+ def test_expand_paths_with_use_glob_p
1572
+ emit_exception_p, result, option, path = data
1573
+ config = config_element("", "", {
1574
+ "tag" => "tail",
1575
+ "path" => path,
1576
+ "format" => "none",
1577
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1578
+ "read_from_head" => true,
1579
+ "refresh_interval" => 30,
1580
+ "glob_policy" => option,
1581
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1582
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1583
+ })
1584
+ if emit_exception_p
1585
+ assert_raise(Fluent::ConfigError) do
1586
+ plugin = create_driver(config, false).instance
1587
+ end
1588
+ else
1589
+ plugin = create_driver(config, false).instance
1590
+ assert_equal(result, !!plugin.use_glob?(path))
1591
+ end
1592
+ end
1593
+ end
1594
+
1595
+ sub_test_case "only_use_backward_compatible" do
1596
+ data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1597
+ "asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
1598
+ "one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
1599
+ )
1600
+ def test_expand_paths_with_use_glob_p
1601
+ result, option, path = data
1602
+ config = config_element("", "", {
1603
+ "tag" => "tail",
1604
+ "path" => path,
1605
+ "format" => "none",
1606
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1607
+ "read_from_head" => true,
1608
+ "refresh_interval" => 30,
1609
+ "glob_policy" => option,
1610
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1611
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1612
+ })
1613
+ plugin = create_driver(config, false).instance
1614
+ assert_equal(result, !!plugin.use_glob?(path))
1615
+ end
1616
+ end
1617
+ end
1618
+
1619
+ def ex_config_with_brackets
1620
+ config_element("", "", {
1621
+ "tag" => "tail",
1622
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1623
+ "format" => "none",
1624
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1625
+ "read_from_head" => true,
1626
+ "refresh_interval" => 30,
1627
+ "glob_policy" => "extended",
1628
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1629
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1630
+ })
1631
+ end
1632
+
1633
+ def test_config_with_always_with_default_delimiter
1634
+ assert_raise(Fluent::ConfigError) do
1635
+ config = config_element("", "", {
1636
+ "tag" => "tail",
1637
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1638
+ "format" => "none",
1639
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1640
+ "read_from_head" => true,
1641
+ "refresh_interval" => 30,
1642
+ "glob_policy" => "always",
1643
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1644
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1645
+ })
1646
+
1647
+ create_driver(config, false).instance
1648
+ end
1649
+ end
1650
+
1651
+ def test_config_with_always_with_custom_delimiter
1652
+ assert_nothing_raised do
1653
+ config = config_element("", "", {
1654
+ "tag" => "tail",
1655
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1656
+ "format" => "none",
1657
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1658
+ "read_from_head" => true,
1659
+ "refresh_interval" => 30,
1660
+ "glob_policy" => "always",
1661
+ "path_delimiter" => "|",
1662
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1663
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1664
+ })
1665
+
1666
+ create_driver(config, false).instance
1667
+ end
1668
+ end
1669
+
1670
+ def test_expand_paths_with_brackets
1671
+ expanded_paths = [
1672
+ create_target_info('test/plugin/data/log_numeric/01.log'),
1673
+ create_target_info('test/plugin/data/log_numeric/02.log'),
1674
+ create_target_info('test/plugin/data/log_numeric/12.log'),
1675
+ create_target_info('test/plugin/data/log_numeric/14.log'),
1676
+ ]
1677
+
1678
+ plugin = create_driver(ex_config_with_brackets, false).instance
1679
+ assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1680
+ end
1681
+
1541
1682
  def test_expand_paths_with_duplicate_configuration
1542
1683
  expanded_paths = [
1543
1684
  create_target_info('test/plugin/data/log/foo/bar.log'),
@@ -3016,6 +3157,92 @@ class TailInputTest < Test::Unit::TestCase
3016
3157
  },
3017
3158
  )
3018
3159
  end
3160
+
3161
+ def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3162
+ config = config_element(
3163
+ "ROOT",
3164
+ "",
3165
+ {
3166
+ "path" => "#{@tmp_dir}/tail.txt*",
3167
+ "pos_file" => "#{@tmp_dir}/tail.pos",
3168
+ "tag" => "t1",
3169
+ "format" => "none",
3170
+ "read_from_head" => "true",
3171
+ "follow_inodes" => "true",
3172
+ "rotate_wait" => "3s",
3173
+ "refresh_interval" => "1h",
3174
+ # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3175
+ # so disable it in order to reproduce the same condition stably.
3176
+ "enable_stat_watcher" => "false",
3177
+ }
3178
+ )
3179
+ d = create_driver(config, false)
3180
+
3181
+ tail_watchers = []
3182
+ stub.proxy(d.instance).setup_watcher do |tw|
3183
+ tail_watchers.append(tw)
3184
+ mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3185
+ tw
3186
+ end
3187
+
3188
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3189
+
3190
+ d.run(expect_records: 6, timeout: 15) do
3191
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3192
+
3193
+ sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3194
+
3195
+ FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3196
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3197
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3198
+
3199
+ sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3200
+
3201
+ # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3202
+ [1, 0].each do |i|
3203
+ FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3204
+ end
3205
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3206
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3207
+
3208
+ # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3209
+ # (Note: Currently, there is no harm in duplicate calls)
3210
+ sleep 4
3211
+ end
3212
+
3213
+ inode_0 = tail_watchers[0]&.ino
3214
+ inode_1 = tail_watchers[1]&.ino
3215
+ inode_2 = tail_watchers[2]&.ino
3216
+ record_values = d.events.collect { |event| event[2]["message"] }.sort
3217
+ position_entries = []
3218
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3219
+ f.readlines(chomp: true).each do |line|
3220
+ values = line.split("\t")
3221
+ position_entries.append([values[0], values[1], values[2].to_i(16)])
3222
+ end
3223
+ end
3224
+
3225
+ assert_equal(
3226
+ {
3227
+ record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3228
+ tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3229
+ tail_watcher_inodes: [inode_0, inode_1, inode_2],
3230
+ tail_watcher_io_handler_opened_statuses: [false, false, false],
3231
+ position_entries: [
3232
+ ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
3233
+ ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
3234
+ ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3235
+ ],
3236
+ },
3237
+ {
3238
+ record_values: record_values,
3239
+ tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3240
+ tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3241
+ tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3242
+ position_entries: position_entries
3243
+ },
3244
+ )
3245
+ end
3019
3246
  end
3020
3247
 
3021
3248
  sub_test_case "Update watchers for rotation without follow_inodes" do
@@ -3084,9 +3311,6 @@ class TailInputTest < Test::Unit::TestCase
3084
3311
  sleep 3
3085
3312
 
3086
3313
  Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3087
-
3088
- # Wait `rotate_wait` for file2 to make sure to close all IO handlers
3089
- sleep 3
3090
3314
  end
3091
3315
 
3092
3316
  inode_0 = tail_watchers[0]&.ino
@@ -3121,5 +3345,85 @@ class TailInputTest < Test::Unit::TestCase
3121
3345
  },
3122
3346
  )
3123
3347
  end
3348
+
3349
+ def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3350
+ config = config_element(
3351
+ "ROOT",
3352
+ "",
3353
+ {
3354
+ "path" => "#{@tmp_dir}/tail.txt0",
3355
+ "pos_file" => "#{@tmp_dir}/tail.pos",
3356
+ "tag" => "t1",
3357
+ "format" => "none",
3358
+ "read_from_head" => "true",
3359
+ "rotate_wait" => "3s",
3360
+ "refresh_interval" => "1h",
3361
+ }
3362
+ )
3363
+ d = create_driver(config, false)
3364
+
3365
+ tail_watchers = []
3366
+ stub.proxy(d.instance).setup_watcher do |tw|
3367
+ tail_watchers.append(tw)
3368
+ mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3369
+ tw
3370
+ end
3371
+
3372
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3373
+
3374
+ d.run(expect_records: 6, timeout: 15) do
3375
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3376
+
3377
+ sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3378
+
3379
+ FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3380
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3381
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3382
+
3383
+ sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3384
+
3385
+ # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3386
+ [1, 0].each do |i|
3387
+ FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3388
+ end
3389
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3390
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3391
+
3392
+ # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3393
+ # (Note: Currently, there is no harm in duplicate calls)
3394
+ sleep 4
3395
+ end
3396
+
3397
+ inode_0 = tail_watchers[0]&.ino
3398
+ inode_1 = tail_watchers[1]&.ino
3399
+ inode_2 = tail_watchers[2]&.ino
3400
+ record_values = d.events.collect { |event| event[2]["message"] }.sort
3401
+ position_entries = []
3402
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3403
+ f.readlines(chomp: true).each do |line|
3404
+ values = line.split("\t")
3405
+ position_entries.append([values[0], values[1], values[2].to_i(16)])
3406
+ end
3407
+ end
3408
+
3409
+ assert_equal(
3410
+ {
3411
+ record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3412
+ tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3413
+ tail_watcher_inodes: [inode_0, inode_1, inode_2],
3414
+ tail_watcher_io_handler_opened_statuses: [false, false, false],
3415
+ position_entries: [
3416
+ ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3417
+ ],
3418
+ },
3419
+ {
3420
+ record_values: record_values,
3421
+ tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3422
+ tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3423
+ tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3424
+ position_entries: position_entries
3425
+ },
3426
+ )
3427
+ end
3124
3428
  end
3125
3429
  end
@@ -156,7 +156,14 @@ EOL
156
156
  normal_conf = config_element('match', '**', {}, [
157
157
  config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})
158
158
  ])
159
- assert_raise SocketError do
159
+
160
+ if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3
161
+ error_class = Socket::ResolutionError
162
+ else
163
+ error_class = SocketError
164
+ end
165
+
166
+ assert_raise error_class do
160
167
  create_driver(normal_conf)
161
168
  end
162
169
 
@@ -165,7 +172,7 @@ EOL
165
172
  ])
166
173
  @d = d = create_driver(conf)
167
174
  expected_log = "failed to resolve node name when configured"
168
- expected_detail = 'server="test" error_class=SocketError'
175
+ expected_detail = "server=\"test\" error_class=#{error_class.name}"
169
176
  logs = d.logs
170
177
  assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
171
178
  end
@@ -1241,27 +1248,22 @@ EOL
1241
1248
  target_input_driver = create_target_input_driver(conf: target_config)
1242
1249
  output_conf = config
1243
1250
  d = create_driver(output_conf)
1244
- d.instance_start
1245
1251
 
1246
- begin
1247
- chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1248
- mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1249
- linger_timeout: anything,
1250
- send_timeout: anything,
1251
- recv_timeout: anything,
1252
- connect_timeout: anything
1253
- ) { |sock| mock(sock).close.once; sock }.twice
1252
+ chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1253
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1254
+ linger_timeout: anything,
1255
+ send_timeout: anything,
1256
+ recv_timeout: anything,
1257
+ connect_timeout: anything
1258
+ ) { |sock| mock(sock).close.once; sock }.twice
1254
1259
 
1255
- target_input_driver.run(timeout: 15) do
1256
- d.run(shutdown: false) do
1257
- node = d.instance.nodes.first
1258
- 2.times do
1259
- node.send_data('test', chunk) rescue nil
1260
- end
1260
+ target_input_driver.run(timeout: 15) do
1261
+ d.run do
1262
+ node = d.instance.nodes.first
1263
+ 2.times do
1264
+ node.send_data('test', chunk) rescue nil
1261
1265
  end
1262
1266
  end
1263
- ensure
1264
- d.instance_shutdown
1265
1267
  end
1266
1268
  end
1267
1269
 
@@ -1275,7 +1277,6 @@ EOL
1275
1277
  port #{@target_port}
1276
1278
  </server>
1277
1279
  ])
1278
- d.instance_start
1279
1280
  assert_nothing_raised { d.run }
1280
1281
  end
1281
1282
 
@@ -1287,33 +1288,28 @@ EOL
1287
1288
  keepalive_timeout 2
1288
1289
  ]
1289
1290
  d = create_driver(output_conf)
1290
- d.instance_start
1291
1291
 
1292
- begin
1293
- chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1294
- mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1295
- linger_timeout: anything,
1296
- send_timeout: anything,
1297
- recv_timeout: anything,
1298
- connect_timeout: anything
1299
- ) { |sock| mock(sock).close.once; sock }.once
1292
+ chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1293
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1294
+ linger_timeout: anything,
1295
+ send_timeout: anything,
1296
+ recv_timeout: anything,
1297
+ connect_timeout: anything
1298
+ ) { |sock| mock(sock).close.once; sock }.once
1300
1299
 
1301
- target_input_driver.run(timeout: 15) do
1302
- d.run(shutdown: false) do
1303
- node = d.instance.nodes.first
1304
- 2.times do
1305
- node.send_data('test', chunk) rescue nil
1306
- end
1300
+ target_input_driver.run(timeout: 15) do
1301
+ d.run do
1302
+ node = d.instance.nodes.first
1303
+ 2.times do
1304
+ node.send_data('test', chunk) rescue nil
1307
1305
  end
1308
1306
  end
1309
- ensure
1310
- d.instance_shutdown
1311
1307
  end
1312
1308
  end
1313
1309
 
1314
1310
  test 'create timer of purging obsolete sockets' do
1315
1311
  output_conf = config + %[keepalive true]
1316
- d = create_driver(output_conf)
1312
+ @d = d = create_driver(output_conf)
1317
1313
 
1318
1314
  mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
1319
1315
  mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
@@ -1329,7 +1325,6 @@ EOL
1329
1325
  keepalive_timeout 2
1330
1326
  ]
1331
1327
  d = create_driver(output_conf)
1332
- d.instance_start
1333
1328
 
1334
1329
  chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1335
1330
  mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
@@ -7,6 +7,7 @@ require 'webrick/https'
7
7
  require 'net/http'
8
8
  require 'uri'
9
9
  require 'json'
10
+ require 'aws-sdk-core'
10
11
 
11
12
  # WEBrick's ProcHandler doesn't handle PUT by default
12
13
  module WEBrick::HTTPServlet
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
390
391
  end
391
392
  end
392
393
 
394
+
395
+ sub_test_case 'aws sigv4 auth' do
396
+ setup do
397
+ @@fake_aws_credentials = Aws::Credentials.new(
398
+ 'fakeaccess',
399
+ 'fakesecret',
400
+ 'fake session token'
401
+ )
402
+ end
403
+
404
+ def server_port
405
+ 19883
406
+ end
407
+
408
+ def test_aws_sigv4_sts_role_arn
409
+ stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
410
+ stub(credentials_provider).credentials {
411
+ @@fake_aws_credentials
412
+ }
413
+ credentials_provider
414
+ end
415
+
416
+ d = create_driver(config + %[
417
+ <auth>
418
+ method aws_sigv4
419
+ aws_service someservice
420
+ aws_region my-region-1
421
+ aws_role_arn arn:aws:iam::123456789012:role/MyRole
422
+ </auth>
423
+ ])
424
+ d.run(default_tag: 'test.http') do
425
+ test_events.each { |event|
426
+ d.feed(event)
427
+ }
428
+ end
429
+
430
+ result = @@result
431
+ assert_equal 'POST', result.method
432
+ assert_equal 'application/x-ndjson', result.content_type
433
+ assert_equal test_events, result.data
434
+ assert_not_empty result.headers
435
+ assert_not_nil result.headers['authorization']
436
+ assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
437
+ assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
438
+ assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
439
+ assert_not_nil result.headers['x-amz-content-sha256']
440
+ assert_not_empty result.headers['x-amz-content-sha256']
441
+ assert_not_nil result.headers['x-amz-security-token']
442
+ assert_not_empty result.headers['x-amz-security-token']
443
+ assert_not_nil result.headers['x-amz-date']
444
+ assert_not_empty result.headers['x-amz-date']
445
+ end
446
+
447
+ def test_aws_sigv4_no_role
448
+ stub(Aws::CredentialProviderChain).new do |provider_chain|
449
+ stub(provider_chain).resolve {
450
+ @@fake_aws_credentials
451
+ }
452
+ provider_chain
453
+ end
454
+ d = create_driver(config + %[
455
+ <auth>
456
+ method aws_sigv4
457
+ aws_service someservice
458
+ aws_region my-region-1
459
+ </auth>
460
+ ])
461
+ d.run(default_tag: 'test.http') do
462
+ test_events.each { |event|
463
+ d.feed(event)
464
+ }
465
+ end
466
+
467
+ result = @@result
468
+ assert_equal 'POST', result.method
469
+ assert_equal 'application/x-ndjson', result.content_type
470
+ assert_equal test_events, result.data
471
+ assert_not_empty result.headers
472
+ assert_not_nil result.headers['authorization']
473
+ assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
474
+ assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
475
+ assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
476
+ assert_not_nil result.headers['x-amz-content-sha256']
477
+ assert_not_empty result.headers['x-amz-content-sha256']
478
+ assert_not_nil result.headers['x-amz-security-token']
479
+ assert_not_empty result.headers['x-amz-security-token']
480
+ assert_not_nil result.headers['x-amz-date']
481
+ assert_not_empty result.headers['x-amz-date']
482
+ end
483
+ end
484
+
393
485
  sub_test_case 'HTTPS' do
394
486
  def server_port
395
487
  19882
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
426
518
  assert_not_empty result.headers
427
519
  end
428
520
  end
521
+
522
+ sub_test_case 'connection_reuse' do
523
+ def server_port
524
+ 19883
525
+ end
526
+
527
+ def test_connection_recreation
528
+ d = create_driver(%[
529
+ endpoint http://127.0.0.1:#{server_port}/test
530
+ reuse_connections true
531
+ ])
532
+
533
+ d.run(default_tag: 'test.http', shutdown: false) do
534
+ d.feed(test_events[0])
535
+ end
536
+
537
+ data = @@result.data
538
+
539
+ # Restart server to simulate connection loss
540
+ @@http_server_thread.kill
541
+ @@http_server_thread.join
542
+ @@http_server_thread = Thread.new do
543
+ run_http_server
544
+ end
545
+
546
+ d.run(default_tag: 'test.http') do
547
+ d.feed(test_events[1])
548
+ end
549
+
550
+ result = @@result
551
+ assert_equal 'POST', result.method
552
+ assert_equal 'application/x-ndjson', result.content_type
553
+ assert_equal test_events, data.concat(result.data)
554
+ assert_not_empty result.headers
555
+ end
556
+ end
429
557
  end
@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
26
26
 
27
27
  assert_equal parent.object_id, child.owner.object_id
28
28
 
29
- assert child.instance_eval{ @_plugin_id_configured }
30
29
  assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
31
30
 
32
31
  assert_equal Fluent::Log::LEVEL_TRACE, child.log.level