fluentd 1.16.5-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
@@ -1,346 +0,0 @@
1
- require_relative '../../helper'
2
- require 'fluent/plugin/in_tail/position_file'
3
- require 'fluent/plugin/in_tail'
4
-
5
- require 'fileutils'
6
- require 'tempfile'
7
-
8
- class IntailPositionFileTest < Test::Unit::TestCase
9
- setup do
10
- @file = Tempfile.new('intail_position_file_test').binmode
11
- end
12
-
13
- teardown do
14
- @file.close rescue nil
15
- @file.unlink rescue nil
16
- end
17
-
18
- UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
19
- TEST_CONTENT = <<~EOF
20
- valid_path\t0000000000000002\t0000000000000001
21
- inode23bit\t0000000000000000\t00000000
22
- invalidpath100000000000000000000000000000000
23
- unwatched\t#{UNWATCHED_STR}\t0000000000000000
24
- EOF
25
- TEST_CONTENT_PATHS = {
26
- "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
27
- "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
28
- }
29
- TEST_CONTENT_INODES = {
30
- 1 => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
31
- 0 => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
32
- }
33
-
34
- def write_data(f, content)
35
- f.write(content)
36
- f.seek(0)
37
- end
38
-
39
- def follow_inodes_block
40
- [true, false].each do |follow_inodes|
41
- yield follow_inodes
42
- end
43
- end
44
-
45
- test '.load' do
46
- write_data(@file, TEST_CONTENT)
47
- Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
48
-
49
- @file.seek(0)
50
- lines = @file.readlines
51
- assert_equal 2, lines.size
52
- assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
53
- assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
54
- end
55
-
56
- sub_test_case '#try_compact' do
57
- test 'compact invalid and convert 32 bit inode value' do
58
- write_data(@file, TEST_CONTENT)
59
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
60
-
61
- @file.seek(0)
62
- lines = @file.readlines
63
- assert_equal 2, lines.size
64
- assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
65
- assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
66
- end
67
-
68
- test 'compact data if duplicated line' do
69
- write_data(@file, <<~EOF)
70
- valid_path\t0000000000000002\t0000000000000001
71
- valid_path\t0000000000000003\t0000000000000004
72
- EOF
73
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
74
-
75
- @file.seek(0)
76
- lines = @file.readlines
77
- assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
78
- end
79
-
80
- test 'does not change when the file is changed' do
81
- write_data(@file, TEST_CONTENT)
82
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
83
-
84
- mock.proxy(pf).fetch_compacted_entries do |r|
85
- @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
86
- r
87
- end
88
-
89
- pf.try_compact
90
-
91
- @file.seek(0)
92
- lines = @file.readlines
93
- assert_equal 5, lines.size
94
- end
95
-
96
- test 'update seek position of remained position entry' do
97
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
98
- target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
99
- target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
100
- target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
101
- pf[target_info1]
102
- pf[target_info2]
103
- pf[target_info3]
104
-
105
- target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
106
- pf.unwatch(target_info1_2)
107
-
108
- pf.try_compact
109
-
110
- @file.seek(0)
111
- lines = @file.readlines
112
- assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0]
113
- assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
114
- assert_equal 2, lines.size
115
-
116
- target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
117
- target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
118
- pf.unwatch(target_info2_2)
119
- pf.unwatch(target_info3_2)
120
- @file.seek(0)
121
- lines = @file.readlines
122
- assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
123
- assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
124
- assert_equal 2, lines.size
125
- end
126
-
127
- test 'should ignore initial existing files on follow_inode' do
128
- write_data(@file, TEST_CONTENT)
129
- pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})
130
- @file.seek(0)
131
- assert_equal([], @file.readlines)
132
-
133
- @file.seek(0)
134
- write_data(@file, TEST_CONTENT)
135
- pos_file.try_compact
136
-
137
- @file.seek(0)
138
- assert_equal([
139
- "valid_path\t0000000000000002\t0000000000000001\n",
140
- "inode23bit\t0000000000000000\t0000000000000000\n",
141
- ],
142
- @file.readlines)
143
- end
144
- end
145
-
146
- sub_test_case '#load' do
147
- test 'compact invalid and convert 32 bit inode value' do
148
- write_data(@file, TEST_CONTENT)
149
- Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
150
-
151
- @file.seek(0)
152
- lines = @file.readlines
153
- assert_equal 2, lines.size
154
- assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
155
- assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
156
- end
157
-
158
- test 'compact deleted paths' do
159
- write_data(@file, TEST_CONTENT)
160
- Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
161
-
162
- @file.seek(0)
163
- lines = @file.readlines
164
- assert_equal [], lines
165
- end
166
-
167
- test 'compact data if duplicated line' do
168
- write_data(@file, <<~EOF)
169
- valid_path\t0000000000000002\t0000000000000001
170
- valid_path\t0000000000000003\t0000000000000004
171
- EOF
172
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load
173
-
174
- @file.seek(0)
175
- lines = @file.readlines
176
- assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
177
- end
178
- end
179
-
180
- sub_test_case '#[]' do
181
- test 'return entry' do
182
- write_data(@file, TEST_CONTENT)
183
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
184
-
185
- valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
186
- f = pf[valid_target_info]
187
- assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
188
- assert_equal 2, f.read_pos
189
- assert_equal 1, f.read_inode
190
-
191
- @file.seek(0)
192
- lines = @file.readlines
193
- assert_equal 2, lines.size
194
-
195
- nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
196
- f = pf[nonexistent_target_info]
197
- assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
198
- assert_equal 0, f.read_pos
199
- assert_equal 0, f.read_inode
200
-
201
- @file.seek(0)
202
- lines = @file.readlines
203
- assert_equal 3, lines.size
204
- assert_equal "nonexist_path\t0000000000000000\t0000000000000000\n", lines[2]
205
- end
206
-
207
- test 'does not change other value position if other entry try to write' do
208
- write_data(@file, TEST_CONTENT)
209
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
210
-
211
- f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
212
- assert_equal 0, f.read_inode
213
- assert_equal 0, f.read_pos
214
-
215
- pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
216
-
217
- f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
218
- assert_equal 0, f.read_inode
219
- assert_equal 0, f.read_pos
220
-
221
- pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
222
- assert_equal 1, f.read_inode
223
- assert_equal 2, f.read_pos
224
- end
225
- end
226
-
227
- sub_test_case '#unwatch' do
228
- test 'unwatch entry by path' do
229
- write_data(@file, TEST_CONTENT)
230
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
231
- inode1 = File.stat(@file).ino
232
- target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
233
- p1 = pf[target_info1]
234
- assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
235
-
236
- pf.unwatch(target_info1)
237
- assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
238
-
239
- inode2 = File.stat(@file).ino
240
- target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
241
- p2 = pf[target_info2]
242
- assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
243
-
244
- assert_not_equal p1, p2
245
- end
246
-
247
- test 'unwatch entries by inode' do
248
- write_data(@file, TEST_CONTENT)
249
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log)
250
-
251
- existing_targets = TEST_CONTENT_INODES.select do |inode, target_info|
252
- inode == 1
253
- end
254
- pe_to_unwatch = pf[TEST_CONTENT_INODES[0]]
255
-
256
- pf.unwatch_removed_targets(existing_targets)
257
-
258
- assert_equal(
259
- {
260
- map_keys: [TEST_CONTENT_INODES[1].ino],
261
- unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION,
262
- },
263
- {
264
- map_keys: pf.instance_variable_get(:@map).keys,
265
- unwatched_pe_pos: pe_to_unwatch.read_pos,
266
- }
267
- )
268
-
269
- unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]]
270
- assert_not_equal pe_to_unwatch, unwatched_pe_retaken
271
- end
272
- end
273
-
274
- sub_test_case 'FilePositionEntry' do
275
- FILE_POS_CONTENT = <<~EOF
276
- valid_path\t0000000000000002\t0000000000000001
277
- valid_path2\t0000000000000003\t0000000000000002
278
- EOF
279
-
280
- def build_files(file)
281
- r = {}
282
-
283
- file.each_line do |line|
284
- m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
285
- path = m[1]
286
- pos = m[2].to_i(16)
287
- ino = m[3].to_i(16)
288
- seek = file.pos - line.bytesize + path.bytesize + 1
289
- r[path] = Fluent::Plugin::TailInput::FilePositionEntry.new(@file, Mutex.new, seek, pos, ino)
290
- end
291
-
292
- r
293
- end
294
-
295
- test '#update' do
296
- write_data(@file, FILE_POS_CONTENT)
297
- fs = build_files(@file)
298
- f = fs['valid_path']
299
- f.update(11, 10)
300
-
301
- @file.seek(0)
302
- lines = @file.readlines
303
- assert_equal 2, lines.size
304
- assert_equal "valid_path\t000000000000000a\t000000000000000b\n", lines[0]
305
- assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1]
306
- end
307
-
308
- test '#update_pos' do
309
- write_data(@file, FILE_POS_CONTENT)
310
- fs = build_files(@file)
311
- f = fs['valid_path']
312
- f.update_pos(10)
313
-
314
- @file.seek(0)
315
- lines = @file.readlines
316
- assert_equal 2, lines.size
317
- assert_equal "valid_path\t000000000000000a\t0000000000000001\n", lines[0]
318
- assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1]
319
- end
320
-
321
- test '#read_pos' do
322
- write_data(@file, FILE_POS_CONTENT)
323
- fs = build_files(@file)
324
- f = fs['valid_path']
325
- assert_equal 2, f.read_pos
326
-
327
- f.update_pos(10)
328
- assert_equal 10, f.read_pos
329
-
330
- f.update(2, 11)
331
- assert_equal 11, f.read_pos
332
- end
333
-
334
- test '#read_inode' do
335
- write_data(@file, FILE_POS_CONTENT)
336
- fs = build_files(@file)
337
- f = fs['valid_path']
338
- assert_equal 1, f.read_inode
339
- f.update_pos(10)
340
- assert_equal 1, f.read_inode
341
-
342
- f.update(2, 11)
343
- assert_equal 2, f.read_inode
344
- end
345
- end
346
- end
@@ -1,140 +0,0 @@
1
- require_relative '../../helper'
2
- require 'fluent/test/driver/output'
3
- require 'flexmock/test_unit'
4
-
5
- require 'fluent/plugin/out_forward'
6
- require 'fluent/plugin/out_forward/ack_handler'
7
-
8
- class AckHandlerTest < Test::Unit::TestCase
9
- data(
10
- 'chunk_id is matched' => [MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }), Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS],
11
- 'chunk_id is not matched' => [MessagePack.pack({ 'ack' => 'unmatched' }), Fluent::Plugin::ForwardOutput::AckHandler::Result::CHUNKID_UNMATCHED],
12
- 'chunk_id is empty' => ['', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED],
13
- )
14
- test 'returns chunk_id, node, sock and result status' do |args|
15
- receved, state = args
16
- ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
17
-
18
- node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
19
- chunk_id = 'chunk_id 111'
20
- ack = ack_handler.create_ack(chunk_id, node)
21
-
22
- r, w = IO.pipe
23
- begin
24
- w.write(chunk_id)
25
- mock(r).recv(anything) { |_| receved } # IO does not have recv
26
- ack.enqueue(r)
27
-
28
- a1 = a2 = a3 = a4 = nil
29
- ack_handler.collect_response(1) do |cid, n, s, ret|
30
- # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
31
- a1 = cid; a2 = n; a3 = s; a4 = ret
32
- end
33
-
34
- assert_equal chunk_id, a1
35
- assert_equal node, a2
36
- assert_equal r, a3
37
- assert_equal state, a4
38
- ensure
39
- r.close rescue nil
40
- w.close rescue nil
41
- end
42
- end
43
-
44
- test 'returns nil if raise an error' do
45
- ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
46
-
47
- node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
48
- chunk_id = 'chunk_id 111'
49
- ack = ack_handler.create_ack(chunk_id, node)
50
-
51
- r, w = IO.pipe
52
- begin
53
- w.write(chunk_id)
54
- mock(r).recv(anything) { |_| raise 'unexpected error' } # IO does not have recv
55
- ack.enqueue(r)
56
-
57
- a1 = a2 = a3 = a4 = nil
58
- ack_handler.collect_response(1) do |cid, n, s, ret|
59
- # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
60
- a1 = cid; a2 = n; a3 = s; a4 = ret
61
- end
62
-
63
- assert_nil a1
64
- assert_nil a2
65
- assert_nil a3
66
- assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
67
- ensure
68
- r.close rescue nil
69
- w.close rescue nil
70
- end
71
- end
72
-
73
- test 'when ack is expired' do
74
- ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100)
75
-
76
- node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
77
- chunk_id = 'chunk_id 111'
78
- ack = ack_handler.create_ack(chunk_id, node)
79
-
80
- r, w = IO.pipe
81
- begin
82
- w.write(chunk_id)
83
- mock(r).recv(anything).never
84
- ack.enqueue(r)
85
-
86
- a1 = a2 = a3 = a4 = nil
87
- ack_handler.collect_response(1) do |cid, n, s, ret|
88
- # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
89
- a1 = cid; a2 = n; a3 = s; a4 = ret
90
- end
91
-
92
- assert_equal chunk_id, a1
93
- assert_equal node, a2
94
- assert_equal r, a3
95
- assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
96
- ensure
97
- r.close rescue nil
98
- w.close rescue nil
99
- end
100
- end
101
-
102
- # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case.
103
- # If exclusive control for this case is implemented, this test may not be necessary.
104
- test 'raises no error when another thread closes a socket' do
105
- ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
106
-
107
- node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
108
- chunk_id = 'chunk_id 111'
109
- ack = ack_handler.create_ack(chunk_id, node)
110
-
111
- r, w = IO.pipe
112
- begin
113
- w.write(chunk_id)
114
- stub(r).recv { |_|
115
- sleep(1) # To ensure that multiple threads select the socket before closing.
116
- raise IOError, 'stream closed in another thread' if r.closed?
117
- MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })
118
- }
119
- ack.enqueue(r)
120
-
121
- threads = []
122
- 2.times do
123
- threads << Thread.new do
124
- ack_handler.collect_response(1) do |cid, n, s, ret|
125
- s&.close
126
- end
127
- end
128
- end
129
-
130
- assert_true threads.map{ |t| t.join(10) }.all?
131
- assert_false(
132
- $log.out.logs.any? { |log| log.include?('[error]') },
133
- $log.out.logs.select{ |log| log.include?('[error]') }.join('\n')
134
- )
135
- ensure
136
- r.close rescue nil
137
- w.close rescue nil
138
- end
139
- end
140
- end
@@ -1,145 +0,0 @@
1
- require_relative '../../helper'
2
- require 'fluent/test/driver/output'
3
- require 'flexmock/test_unit'
4
-
5
- require 'fluent/plugin/out_forward'
6
- require 'fluent/plugin/out_forward/connection_manager'
7
- require 'fluent/plugin/out_forward/socket_cache'
8
-
9
- class ConnectionManager < Test::Unit::TestCase
10
- sub_test_case '#connect' do
11
- sub_test_case 'when socket_cache is nil' do
12
- test 'creates socket and does not close when block is not given' do
13
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
14
- log: $log,
15
- secure: false,
16
- connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
17
- socket_cache: nil,
18
- )
19
-
20
- mock.proxy(cm).connect_keepalive(anything).never
21
- sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
22
- assert_equal(sock, 'sock')
23
- assert_equal(ri.state, :established)
24
- end
25
-
26
- test 'creates socket and calls close when block is given' do
27
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
28
- log: $log,
29
- secure: false,
30
- connection_factory: -> (_, _, _) {
31
- sock = 'sock'
32
- mock(sock).close.once
33
- mock(sock).close_write.once
34
- sock
35
- },
36
- socket_cache: nil,
37
- )
38
-
39
- mock.proxy(cm).connect_keepalive(anything).never
40
- cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
41
- assert_equal(sock, 'sock')
42
- assert_equal(ri.state, :established)
43
- end
44
- end
45
-
46
- test 'when secure is true, state is helo' do
47
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
48
- log: $log,
49
- secure: true,
50
- connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
51
- socket_cache: nil,
52
- )
53
-
54
- mock.proxy(cm).connect_keepalive(anything).never
55
- sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
56
- assert_equal(sock, 'sock')
57
- assert_equal(ri.state, :helo)
58
- end
59
-
60
- test 'when passed ack' do
61
- sock = 'sock'
62
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
63
- log: $log,
64
- secure: false,
65
- connection_factory: -> (_, _, _) {
66
- mock(sock).close.never
67
- mock(sock).close_write.never
68
- sock
69
- },
70
- socket_cache: nil,
71
- )
72
-
73
- mock.proxy(cm).connect_keepalive(anything).never
74
- ack = mock('ack').enqueue(sock).once.subject
75
- cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
76
- assert_equal(sock, 'sock')
77
- assert_equal(ri.state, :established)
78
- end
79
- end
80
- end
81
-
82
- sub_test_case 'when socket_cache exists' do
83
- test 'calls connect_keepalive' do
84
- cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
85
- mock(cache).checkin('sock').never
86
-
87
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
88
- log: $log,
89
- secure: false,
90
- connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
91
- socket_cache: cache,
92
- )
93
-
94
- mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
95
-
96
- sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
97
- assert_equal(sock, 'sock')
98
- assert_equal(ri.state, :established)
99
- end
100
-
101
- test 'calls connect_keepalive and closes socket with block' do
102
- cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
103
- mock(cache).checkin('sock').once
104
-
105
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
106
- log: $log,
107
- secure: false,
108
- connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock); sock },
109
- socket_cache: cache,
110
- )
111
-
112
- mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
113
-
114
- cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
115
- assert_equal(sock, 'sock')
116
- assert_equal(ri.state, :established)
117
- end
118
- end
119
-
120
- test 'does not call dec_ref when ack is passed' do
121
- cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
122
- mock(cache).checkin('sock').never
123
- sock = 'sock'
124
- ack = stub('ack').enqueue(sock).once.subject
125
-
126
- cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
127
- log: $log,
128
- secure: false,
129
- connection_factory: -> (_, _, _) {
130
- mock(sock).close.never
131
- mock(sock).close_write.never
132
- sock
133
- },
134
- socket_cache: cache,
135
- )
136
-
137
- mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: ack).once
138
- cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
139
- assert_equal(sock, 'sock')
140
- assert_equal(ri.state, :established)
141
- end
142
- end
143
- end
144
- end
145
- end