fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,833 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buf_file_single'
3
+ require 'fluent/plugin/output'
4
+ require 'fluent/unique_id'
5
+ require 'fluent/system_config'
6
+ require 'fluent/env'
7
+ require 'fluent/test/driver/output'
8
+
9
+ require 'msgpack'
10
+
11
+ module FluentPluginFileSingleBufferTest
12
+ class DummyOutputPlugin < Fluent::Plugin::Output
13
+ Fluent::Plugin.register_output('buf_file_single_test', self)
14
+ config_section :buffer do
15
+ config_set_default :@type, 'file_single'
16
+ end
17
+ def multi_workers_ready?
18
+ true
19
+ end
20
+ def write(chunk)
21
+ # drop
22
+ end
23
+ end
24
+
25
+ class DummyOutputMPPlugin < Fluent::Plugin::Output
26
+ Fluent::Plugin.register_output('buf_file_single_mp_test', self)
27
+ config_section :buffer do
28
+ config_set_default :@type, 'file_single'
29
+ end
30
+ def formatted_to_msgpack_binary?
31
+ true
32
+ end
33
+ def multi_workers_ready?
34
+ true
35
+ end
36
+ def write(chunk)
37
+ # drop
38
+ end
39
+ end
40
+ end
41
+
42
+ class FileSingleBufferTest < Test::Unit::TestCase
43
+ def metadata(timekey: nil, tag: 'testing', variables: nil)
44
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
45
+ end
46
+
47
+ PATH = File.expand_path('../../tmp/buffer_file_single_dir', __FILE__)
48
+ TAG_CONF = %[
49
+ <buffer tag>
50
+ @type file_single
51
+ path #{PATH}
52
+ </buffer>
53
+ ]
54
+ FIELD_CONF = %[
55
+ <buffer k>
56
+ @type file_single
57
+ path #{PATH}
58
+ </buffer>
59
+ ]
60
+
61
+ setup do
62
+ Fluent::Test.setup
63
+
64
+ @d = nil
65
+ @bufdir = PATH
66
+ FileUtils.rm_rf(@bufdir) rescue nil
67
+ FileUtils.mkdir_p(@bufdir)
68
+ end
69
+
70
+ teardown do
71
+ FileUtils.rm_rf(@bufdir) rescue nil
72
+ end
73
+
74
+ def create_driver(conf = TAG_CONF, klass = FluentPluginFileSingleBufferTest::DummyOutputPlugin)
75
+ Fluent::Test::Driver::Output.new(klass).configure(conf)
76
+ end
77
+
78
+ sub_test_case 'configuration' do
79
+ test 'path has "fsb" prefix and "buf" suffix by default' do
80
+ @d = create_driver
81
+ p = @d.instance.buffer
82
+ assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path
83
+ end
84
+
85
+ data('text based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputPlugin, :text],
86
+ 'msgpack based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputMPPlugin, :msgpack])
87
+ test 'detect chunk_format' do |param|
88
+ klass, expected = param
89
+ @d = create_driver(TAG_CONF, klass)
90
+ p = @d.instance.buffer
91
+ assert_equal expected, p.chunk_format
92
+ end
93
+
94
+ test '"prefix.*.suffix" path will be replaced with default' do
95
+ @d = create_driver(%[
96
+ <buffer tag>
97
+ @type file_single
98
+ path #{@bufdir}/foo.*.bar
99
+ </buffer>
100
+ ])
101
+ p = @d.instance.buffer
102
+ assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path
103
+ end
104
+ end
105
+
106
+ sub_test_case 'buffer configurations and workers' do
107
+ setup do
108
+ @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
109
+ end
110
+
111
+ test 'enables multi worker configuration with unexisting directory path' do
112
+ FileUtils.rm_rf(@bufdir)
113
+ buf_conf = config_element('buffer', '', {'path' => @bufdir})
114
+ assert_nothing_raised do
115
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
116
+ @d.configure(config_element('ROOT', '', {}, [buf_conf]))
117
+ end
118
+ end
119
+ end
120
+
121
+ test 'enables multi worker configuration with existing directory path' do
122
+ FileUtils.mkdir_p @bufdir
123
+ buf_conf = config_element('buffer', '', {'path' => @bufdir})
124
+ assert_nothing_raised do
125
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
126
+ @d.configure(config_element('ROOT', '', {}, [buf_conf]))
127
+ end
128
+ end
129
+ end
130
+
131
+ test 'enables multi worker configuration with root dir' do
132
+ buf_conf = config_element('buffer', '')
133
+ assert_nothing_raised do
134
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
135
+ @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ test 'raise config error when using same file path' do
142
+ d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
143
+ d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
144
+ Fluent::SystemConfig.overwrite_system_config({}) do
145
+ d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))
146
+ end
147
+
148
+ any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|
149
+ stub(klass).called_in_test? { false }
150
+ end
151
+
152
+ err = assert_raise(Fluent::ConfigError) do
153
+ Fluent::SystemConfig.overwrite_system_config({}) do
154
+ d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))
155
+ end
156
+ end
157
+ assert_match(/plugin already uses same buffer path/, err.message)
158
+ end
159
+
160
+ sub_test_case 'buffer plugin configured only with path' do
161
+ setup do
162
+ @bufpath = File.join(@bufdir, 'testbuf.*.buf')
163
+ FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
164
+
165
+ @d = create_driver
166
+ @p = @d.instance.buffer
167
+ end
168
+
169
+ teardown do
170
+ if @p
171
+ @p.stop unless @p.stopped?
172
+ @p.before_shutdown unless @p.before_shutdown?
173
+ @p.shutdown unless @p.shutdown?
174
+ @p.after_shutdown unless @p.after_shutdown?
175
+ @p.close unless @p.closed?
176
+ @p.terminate unless @p.terminated?
177
+ end
178
+ end
179
+
180
+ test 'this is persistent plugin' do
181
+ assert @p.persistent?
182
+ end
183
+
184
+ test '#start creates directory for buffer chunks' do
185
+ @d = create_driver
186
+ @p = @d.instance.buffer
187
+
188
+ FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
189
+ assert !File.exist?(@bufdir)
190
+
191
+ @p.start
192
+ assert File.exist?(@bufdir)
193
+ assert { File.stat(@bufdir).mode.to_s(8).end_with?('755') }
194
+ end
195
+
196
+ test '#start creates directory for buffer chunks with specified permission' do
197
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
198
+
199
+ @d = create_driver(%[
200
+ <buffer tag>
201
+ @type file_single
202
+ path #{PATH}
203
+ dir_permission 700
204
+ </buffer>
205
+ ])
206
+ @p = @d.instance.buffer
207
+
208
+ FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
209
+ assert !File.exist?(@bufdir)
210
+
211
+ @p.start
212
+ assert File.exist?(@bufdir)
213
+ assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }
214
+ end
215
+
216
+ test '#start creates directory for buffer chunks with specified permission via system config' do
217
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
218
+
219
+ sysconf = {'dir_permission' => '700'}
220
+ Fluent::SystemConfig.overwrite_system_config(sysconf) do
221
+ @d = create_driver
222
+ @p = @d.instance.buffer
223
+
224
+ FileUtils.rm_r @bufdir if File.exist?(@bufdir)
225
+ assert !File.exist?(@bufdir)
226
+
227
+ @p.start
228
+ assert File.exist?(@bufdir)
229
+ assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }
230
+ end
231
+ end
232
+
233
+ test '#generate_chunk generates blank file chunk on path with unique_id and tag' do
234
+ FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)
235
+
236
+ m1 = metadata()
237
+ c1 = @p.generate_chunk(m1)
238
+ assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk
239
+ assert_equal m1, c1.metadata
240
+ assert c1.empty?
241
+ assert_equal :unstaged, c1.state
242
+ assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
243
+ assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
244
+ assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
245
+
246
+ c1.purge
247
+ end
248
+
249
+ test '#generate_chunk generates blank file chunk on path with unique_id and field key' do
250
+ FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)
251
+
252
+ @d = create_driver(FIELD_CONF)
253
+ @p = @d.instance.buffer
254
+
255
+ m1 = metadata(tag: nil, variables: {:k => 'foo_bar'})
256
+ c1 = @p.generate_chunk(m1)
257
+ assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk
258
+ assert_equal m1, c1.metadata
259
+ assert c1.empty?
260
+ assert_equal :unstaged, c1.state
261
+ assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
262
+ assert_equal File.join(@bufdir, "fsb.foo_bar.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
263
+
264
+ c1.purge
265
+ end
266
+
267
+ test '#generate_chunk generates blank file chunk with specified permission' do
268
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
269
+
270
+ @d = create_driver(%[
271
+ <buffer tag>
272
+ @type file_single
273
+ path #{PATH}
274
+ file_permission 600
275
+ </buffer>
276
+ ])
277
+ @p = @d.instance.buffer
278
+
279
+ FileUtils.rm_r @bufdir if File.exist?(@bufdir)
280
+ assert !File.exist?(@bufdir)
281
+
282
+ @p.start
283
+
284
+ m = metadata()
285
+ c = @p.generate_chunk(m)
286
+ assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk
287
+ assert_equal m, c.metadata
288
+ assert c.empty?
289
+ assert_equal :unstaged, c.state
290
+ assert_equal 0600, c.permission
291
+ assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path
292
+ assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
293
+
294
+ c.purge
295
+ end
296
+
297
+ test '#generate_chunk generates blank file chunk with specified permission with system_config' do
298
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
299
+
300
+ @d = create_driver(%[
301
+ <buffer tag>
302
+ @type file_single
303
+ path #{PATH}
304
+ </buffer>
305
+ ])
306
+ @p = @d.instance.buffer
307
+
308
+ FileUtils.rm_r @bufdir if File.exist?(@bufdir)
309
+ assert !File.exist?(@bufdir)
310
+
311
+ @p.start
312
+
313
+ m = metadata()
314
+ c = nil
315
+ Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do
316
+ c = @p.generate_chunk(m)
317
+ end
318
+ assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk
319
+ assert_equal m, c.metadata
320
+ assert c.empty?
321
+ assert_equal :unstaged, c.state
322
+ assert_equal 0700, c.permission
323
+ assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path
324
+ assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }
325
+
326
+ c.purge
327
+ end
328
+ end
329
+
330
+ sub_test_case 'configured with system root directory and plugin @id' do
331
+ setup do
332
+ @root_dir = File.expand_path('../../tmp/buffer_file_single_root', __FILE__)
333
+ FileUtils.rm_rf(@root_dir)
334
+
335
+ @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
336
+ @p = nil
337
+ end
338
+
339
+ teardown do
340
+ if @p
341
+ @p.stop unless @p.stopped?
342
+ @p.before_shutdown unless @p.before_shutdown?
343
+ @p.shutdown unless @p.shutdown?
344
+ @p.after_shutdown unless @p.after_shutdown?
345
+ @p.close unless @p.closed?
346
+ @p.terminate unless @p.terminated?
347
+ end
348
+ end
349
+
350
+ test '#start creates directory for buffer chunks' do
351
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
352
+ @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [config_element('buffer', '', {})]))
353
+ @p = @d.buffer
354
+ end
355
+
356
+ expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "fsb.*.buf")
357
+ expected_buffer_dir = File.dirname(expected_buffer_path)
358
+ assert_equal expected_buffer_path, @d.buffer.path
359
+ assert_false Dir.exist?(expected_buffer_dir)
360
+
361
+ @p.start
362
+ assert Dir.exist?(expected_buffer_dir)
363
+ end
364
+ end
365
+
366
+ sub_test_case 'buffer plugin configuration errors' do
367
+ data('tag and key' => 'tag,key',
368
+ 'multiple keys' => 'key1,key2')
369
+ test 'invalid chunk keys' do |param|
370
+ assert_raise Fluent::ConfigError do
371
+ @d = create_driver(%[
372
+ <buffer #{param}>
373
+ @type file_single
374
+ path #{PATH}
375
+ calc_num_records false
376
+ </buffer>
377
+ ])
378
+ end
379
+ end
380
+
381
+ test 'path is not specified' do
382
+ assert_raise Fluent::ConfigError do
383
+ @d = create_driver(%[
384
+ <buffer tag>
385
+ @type file_single
386
+ </buffer>
387
+ ])
388
+ end
389
+ end
390
+ end
391
+
392
+ sub_test_case 'there are no existing file chunks' do
393
+ setup do
394
+ FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
395
+
396
+ @d = create_driver
397
+ @p = @d.instance.buffer
398
+ @p.start
399
+ end
400
+ teardown do
401
+ if @p
402
+ @p.stop unless @p.stopped?
403
+ @p.before_shutdown unless @p.before_shutdown?
404
+ @p.shutdown unless @p.shutdown?
405
+ @p.after_shutdown unless @p.after_shutdown?
406
+ @p.close unless @p.closed?
407
+ @p.terminate unless @p.terminated?
408
+ end
409
+ if @bufdir
410
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
411
+ next if ['.', '..'].include?(File.basename(path))
412
+ File.delete(path)
413
+ end
414
+ end
415
+ end
416
+
417
+ test '#resume returns empty buffer state' do
418
+ ary = @p.resume
419
+ assert_equal({}, ary[0])
420
+ assert_equal([], ary[1])
421
+ end
422
+ end
423
+
424
+ sub_test_case 'there are some existing file chunks' do
425
+ setup do
426
+ @c1id = Fluent::UniqueId.generate
427
+ p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
428
+ File.open(p1, 'wb') do |f|
429
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
430
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
431
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
432
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
433
+ end
434
+ t = Time.now - 50000
435
+ File.utime(t, t, p1)
436
+
437
+ @c2id = Fluent::UniqueId.generate
438
+ p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf")
439
+ File.open(p2, 'wb') do |f|
440
+ f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
441
+ f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
442
+ f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
443
+ end
444
+ t = Time.now - 40000
445
+ File.utime(t, t, p2)
446
+
447
+ @c3id = Fluent::UniqueId.generate
448
+ p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf")
449
+ File.open(p3, 'wb') do |f|
450
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
451
+ f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
452
+ f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
453
+ f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
454
+ end
455
+
456
+ @c4id = Fluent::UniqueId.generate
457
+ p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf")
458
+ File.open(p4, 'wb') do |f|
459
+ f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
460
+ f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
461
+ f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
462
+ end
463
+ end
464
+
465
+ teardown do
466
+ if @p
467
+ @p.stop unless @p.stopped?
468
+ @p.before_shutdown unless @p.before_shutdown?
469
+ @p.shutdown unless @p.shutdown?
470
+ @p.after_shutdown unless @p.after_shutdown?
471
+ @p.close unless @p.closed?
472
+ @p.terminate unless @p.terminated?
473
+ end
474
+ if @bufdir
475
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
476
+ next if ['.', '..'].include?(File.basename(path))
477
+ File.delete(path)
478
+ end
479
+ end
480
+ end
481
+
482
+ test '#resume returns staged/queued chunks with metadata' do
483
+ @d = create_driver
484
+ @p = @d.instance.buffer
485
+ @p.start
486
+
487
+ assert_equal 2, @p.stage.size
488
+ assert_equal 2, @p.queue.size
489
+
490
+ stage = @p.stage
491
+
492
+ m3 = metadata()
493
+ assert_equal @c3id, stage[m3].unique_id
494
+ assert_equal 4, stage[m3].size
495
+ assert_equal :staged, stage[m3].state
496
+
497
+ m4 = metadata(tag: 'foo')
498
+ assert_equal @c4id, stage[m4].unique_id
499
+ assert_equal 3, stage[m4].size
500
+ assert_equal :staged, stage[m4].state
501
+ end
502
+
503
+ test '#resume returns queued chunks ordered by last modified time (FIFO)' do
504
+ @d = create_driver
505
+ @p = @d.instance.buffer
506
+ @p.start
507
+
508
+ assert_equal 2, @p.stage.size
509
+ assert_equal 2, @p.queue.size
510
+
511
+ queue = @p.queue
512
+
513
+ assert{ queue[0].modified_at <= queue[1].modified_at }
514
+
515
+ assert_equal @c1id, queue[0].unique_id
516
+ assert_equal :queued, queue[0].state
517
+ assert_equal 'testing', queue[0].metadata.tag
518
+ assert_nil queue[0].metadata.variables
519
+ assert_equal 4, queue[0].size
520
+
521
+ assert_equal @c2id, queue[1].unique_id
522
+ assert_equal :queued, queue[1].state
523
+ assert_equal 'testing', queue[1].metadata.tag
524
+ assert_nil queue[1].metadata.variables
525
+ assert_equal 3, queue[1].size
526
+ end
527
+
528
+ test '#resume returns staged/queued chunks but skips size calculation by calc_num_records' do
529
+ @d = create_driver(%[
530
+ <buffer tag>
531
+ @type file_single
532
+ path #{PATH}
533
+ calc_num_records false
534
+ </buffer>
535
+ ])
536
+ @p = @d.instance.buffer
537
+ @p.start
538
+
539
+ assert_equal 2, @p.stage.size
540
+ assert_equal 2, @p.queue.size
541
+
542
+ stage = @p.stage
543
+
544
+ m3 = metadata()
545
+ assert_equal @c3id, stage[m3].unique_id
546
+ assert_equal 0, stage[m3].size
547
+ assert_equal :staged, stage[m3].state
548
+
549
+ m4 = metadata(tag: 'foo')
550
+ assert_equal @c4id, stage[m4].unique_id
551
+ assert_equal 0, stage[m4].size
552
+ assert_equal :staged, stage[m4].state
553
+ end
554
+ end
555
+
556
+ sub_test_case 'there are some existing file chunks with placeholders path' do
557
+ setup do
558
+ @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)
559
+ FileUtils.rm_rf(@buf_ph_dir)
560
+ FileUtils.mkdir_p(@buf_ph_dir)
561
+
562
+ @c1id = Fluent::UniqueId.generate
563
+ p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
564
+ File.open(p1, 'wb') do |f|
565
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
566
+ end
567
+ t = Time.now - 50000
568
+ File.utime(t, t, p1)
569
+
570
+ @c2id = Fluent::UniqueId.generate
571
+ p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf")
572
+ File.open(p2, 'wb') do |f|
573
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
574
+ end
575
+ end
576
+
577
+ teardown do
578
+ if @p
579
+ @p.stop unless @p.stopped?
580
+ @p.before_shutdown unless @p.before_shutdown?
581
+ @p.shutdown unless @p.shutdown?
582
+ @p.after_shutdown unless @p.after_shutdown?
583
+ @p.close unless @p.closed?
584
+ @p.terminate unless @p.terminated?
585
+ end
586
+ FileUtils.rm_rf(@buf_ph_dir)
587
+ end
588
+
589
+ test '#resume returns staged/queued chunks with metadata' do
590
+ @d = create_driver(%[
591
+ <buffer tag>
592
+ @type file_single
593
+ path #{@buf_ph_dir}
594
+ </buffer>
595
+ ])
596
+ @p = @d.instance.buffer
597
+ @p.start
598
+
599
+ assert_equal 1, @p.stage.size
600
+ assert_equal 1, @p.queue.size
601
+ end
602
+ end
603
+
604
+ sub_test_case 'there are some existing msgpack file chunks' do
605
+ setup do
606
+ packer = Fluent::MessagePackFactory.packer
607
+ @c1id = Fluent::UniqueId.generate
608
+ p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
609
+ File.open(p1, 'wb') do |f|
610
+ packer.write(["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}])
611
+ packer.write(["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}])
612
+ packer.write(["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}])
613
+ packer.write(["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}])
614
+ f.write packer.full_pack
615
+ end
616
+ t = Time.now - 50000
617
+ File.utime(t, t, p1)
618
+
619
+ @c2id = Fluent::UniqueId.generate
620
+ p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf")
621
+ File.open(p2, 'wb') do |f|
622
+ packer.write(["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}])
623
+ packer.write(["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}])
624
+ packer.write(["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}])
625
+ f.write packer.full_pack
626
+ end
627
+ t = Time.now - 40000
628
+ File.utime(t, t, p2)
629
+
630
+ @c3id = Fluent::UniqueId.generate
631
+ p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf")
632
+ File.open(p3, 'wb') do |f|
633
+ packer.write(["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}])
634
+ packer.write(["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}])
635
+ packer.write(["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}])
636
+ packer.write(["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}])
637
+ f.write packer.full_pack
638
+ end
639
+
640
+ @c4id = Fluent::UniqueId.generate
641
+ p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf")
642
+ File.open(p4, 'wb') do |f|
643
+ packer.write(["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}])
644
+ packer.write(["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}])
645
+ packer.write(["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}])
646
+ f.write packer.full_pack
647
+ end
648
+ end
649
+
650
+ teardown do
651
+ if @p
652
+ @p.stop unless @p.stopped?
653
+ @p.before_shutdown unless @p.before_shutdown?
654
+ @p.shutdown unless @p.shutdown?
655
+ @p.after_shutdown unless @p.after_shutdown?
656
+ @p.close unless @p.closed?
657
+ @p.terminate unless @p.terminated?
658
+ end
659
+ if @bufdir
660
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
661
+ next if ['.', '..'].include?(File.basename(path))
662
+ File.delete(path)
663
+ end
664
+ end
665
+ end
666
+
667
+ test '#resume returns staged/queued chunks with msgpack format' do
668
+ @d = create_driver(%[
669
+ <buffer tag>
670
+ @type file_single
671
+ path #{PATH}
672
+ chunk_format msgpack
673
+ </buffer>
674
+ ])
675
+ @p = @d.instance.buffer
676
+ @p.start
677
+
678
+ assert_equal 2, @p.stage.size
679
+ assert_equal 2, @p.queue.size
680
+
681
+ stage = @p.stage
682
+
683
+ m3 = metadata()
684
+ assert_equal @c3id, stage[m3].unique_id
685
+ assert_equal 4, stage[m3].size
686
+ assert_equal :staged, stage[m3].state
687
+
688
+ m4 = metadata(tag: 'foo')
689
+ assert_equal @c4id, stage[m4].unique_id
690
+ assert_equal 3, stage[m4].size
691
+ assert_equal :staged, stage[m4].state
692
+ end
693
+ end
694
+
695
+ sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
696
+ setup do
697
+ @worker0_dir = File.join(@bufdir, "worker0")
698
+ @worker1_dir = File.join(@bufdir, "worker1")
699
+ FileUtils.rm_rf(@bufdir)
700
+ FileUtils.mkdir_p(@worker0_dir)
701
+ FileUtils.mkdir_p(@worker1_dir)
702
+
703
+ @bufdir_chunk_1 = Fluent::UniqueId.generate
704
+ bc1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.buf")
705
+ File.open(bc1, 'wb') do |f|
706
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
707
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
708
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
709
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
710
+ end
711
+
712
+ @bufdir_chunk_2 = Fluent::UniqueId.generate
713
+ bc2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.buf")
714
+ File.open(bc2, 'wb') do |f|
715
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
716
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
717
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
718
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
719
+ end
720
+
721
+ @worker_dir_chunk_1 = Fluent::UniqueId.generate
722
+ wc0_1 = File.join(@worker0_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf")
723
+ wc1_1 = File.join(@worker1_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf")
724
+ [wc0_1, wc1_1].each do |chunk_path|
725
+ File.open(chunk_path, 'wb') do |f|
726
+ f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
727
+ f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
728
+ f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
729
+ end
730
+ end
731
+
732
+ @worker_dir_chunk_2 = Fluent::UniqueId.generate
733
+ wc0_2 = File.join(@worker0_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf")
734
+ wc1_2 = File.join(@worker1_dir, "fsb.foo.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf")
735
+ [wc0_2, wc1_2].each do |chunk_path|
736
+ File.open(chunk_path, 'wb') do |f|
737
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
738
+ f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
739
+ f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
740
+ f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
741
+ end
742
+ end
743
+
744
+ @worker_dir_chunk_3 = Fluent::UniqueId.generate
745
+ wc0_3 = File.join(@worker0_dir, "fsb.bar.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf")
746
+ wc1_3 = File.join(@worker1_dir, "fsb.baz.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf")
747
+ [wc0_3, wc1_3].each do |chunk_path|
748
+ File.open(chunk_path, 'wb') do |f|
749
+ f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
750
+ f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
751
+ f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
752
+ end
753
+ end
754
+ end
755
+
756
+ teardown do
757
+ if @p
758
+ @p.stop unless @p.stopped?
759
+ @p.before_shutdown unless @p.before_shutdown?
760
+ @p.shutdown unless @p.shutdown?
761
+ @p.after_shutdown unless @p.after_shutdown?
762
+ @p.close unless @p.closed?
763
+ @p.terminate unless @p.terminated?
764
+ end
765
+ end
766
+
767
+ test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do
768
+ ENV['SERVERENGINE_WORKER_ID'] = '0'
769
+
770
+ buf_conf = config_element('buffer', '', {'path' => @bufdir})
771
+ @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
772
+ with_worker_config(workers: 2, worker_id: 0) do
773
+ @d.configure(config_element('output', '', {}, [buf_conf]))
774
+ end
775
+
776
+ @d.start
777
+ @p = @d.buffer
778
+
779
+ assert_equal 2, @p.stage.size
780
+ assert_equal 3, @p.queue.size
781
+
782
+ stage = @p.stage
783
+
784
+ m1 = metadata(tag: 'testing')
785
+ assert_equal @worker_dir_chunk_2, stage[m1].unique_id
786
+ assert_equal 4, stage[m1].size
787
+ assert_equal :staged, stage[m1].state
788
+
789
+ m2 = metadata(tag: 'bar')
790
+ assert_equal @worker_dir_chunk_3, stage[m2].unique_id
791
+ assert_equal 3, stage[m2].size
792
+ assert_equal :staged, stage[m2].state
793
+
794
+ queue = @p.queue
795
+
796
+ assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort
797
+ assert_equal [3, 4, 4], queue.map(&:size).sort
798
+ assert_equal [:queued, :queued, :queued], queue.map(&:state)
799
+ end
800
+
801
+ test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do
802
+ buf_conf = config_element('buffer', '', {'path' => @bufdir})
803
+ @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
804
+ with_worker_config(workers: 2, worker_id: 1) do
805
+ @d.configure(config_element('output', '', {}, [buf_conf]))
806
+ end
807
+
808
+ @d.start
809
+ @p = @d.buffer
810
+
811
+ assert_equal 2, @p.stage.size
812
+ assert_equal 1, @p.queue.size
813
+
814
+ stage = @p.stage
815
+
816
+ m1 = metadata(tag: 'foo')
817
+ assert_equal @worker_dir_chunk_2, stage[m1].unique_id
818
+ assert_equal 4, stage[m1].size
819
+ assert_equal :staged, stage[m1].state
820
+
821
+ m2 = metadata(tag: 'baz')
822
+ assert_equal @worker_dir_chunk_3, stage[m2].unique_id
823
+ assert_equal 3, stage[m2].size
824
+ assert_equal :staged, stage[m2].state
825
+
826
+ queue = @p.queue
827
+
828
+ assert_equal @worker_dir_chunk_1, queue[0].unique_id
829
+ assert_equal 3, queue[0].size
830
+ assert_equal :queued, queue[0].state
831
+ end
832
+ end
833
+ end