fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,882 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/event'
5
+
6
+ require 'json'
7
+ require 'time'
8
+ require 'timeout'
9
+ require 'timecop'
10
+
11
+ module FluentPluginOutputAsBufferedSecondaryTest
12
+ class DummyBareOutput < Fluent::Plugin::Output
13
+ def register(name, &block)
14
+ instance_variable_set("@#{name}", block)
15
+ end
16
+ end
17
+ class DummySyncOutput < DummyBareOutput
18
+ def initialize
19
+ super
20
+ @process = nil
21
+ end
22
+ def process(tag, es)
23
+ @process ? @process.call(tag, es) : nil
24
+ end
25
+ end
26
+ class DummyFullFeatureOutput < DummyBareOutput
27
+ def initialize
28
+ super
29
+ @prefer_buffered_processing = nil
30
+ @prefer_delayed_commit = nil
31
+ @process = nil
32
+ @format = nil
33
+ @write = nil
34
+ @try_write = nil
35
+ end
36
+ def prefer_buffered_processing
37
+ @prefer_buffered_processing ? @prefer_buffered_processing.call : false
38
+ end
39
+ def prefer_delayed_commit
40
+ @prefer_delayed_commit ? @prefer_delayed_commit.call : false
41
+ end
42
+ def process(tag, es)
43
+ @process ? @process.call(tag, es) : nil
44
+ end
45
+ def format(tag, time, record)
46
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
47
+ end
48
+ def write(chunk)
49
+ @write ? @write.call(chunk) : nil
50
+ end
51
+ def try_write(chunk)
52
+ @try_write ? @try_write.call(chunk) : nil
53
+ end
54
+ end
55
+ class DummyFullFeatureOutput2 < DummyFullFeatureOutput
56
+ def prefer_buffered_processing; true; end
57
+ def prefer_delayed_commit; super; end
58
+ def format(tag, time, record); super; end
59
+ def write(chunk); super; end
60
+ def try_write(chunk); super; end
61
+ end
62
+ end
63
+
64
+ class BufferedOutputSecondaryTest < Test::Unit::TestCase
65
+ def create_output(type=:full)
66
+ case type
67
+ when :bare then FluentPluginOutputAsBufferedSecondaryTest::DummyBareOutput.new
68
+ when :sync then FluentPluginOutputAsBufferedSecondaryTest::DummySyncOutput.new
69
+ when :full then FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput.new
70
+ else
71
+ raise ArgumentError, "unknown type: #{type}"
72
+ end
73
+ end
74
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
75
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
76
+ end
77
+ def waiting(seconds)
78
+ begin
79
+ Timeout.timeout(seconds) do
80
+ yield
81
+ end
82
+ rescue Timeout::Error
83
+ STDERR.print(*@i.log.out.logs)
84
+ raise
85
+ end
86
+ end
87
+ def dummy_event_stream
88
+ Fluent::ArrayEventStream.new([
89
+ [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
90
+ [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
91
+ [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
92
+ ])
93
+ end
94
+
95
+ setup do
96
+ @i = create_output
97
+ end
98
+
99
+ teardown do
100
+ if @i
101
+ @i.stop unless @i.stopped?
102
+ @i.before_shutdown unless @i.before_shutdown?
103
+ @i.shutdown unless @i.shutdown?
104
+ @i.after_shutdown unless @i.after_shutdown?
105
+ @i.close unless @i.closed?
106
+ @i.terminate unless @i.terminated?
107
+ end
108
+ Timecop.return
109
+ end
110
+
111
+ sub_test_case 'secondary plugin feature for buffered output with periodical retry' do
112
+ setup do
113
+ Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)
114
+ Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)
115
+ end
116
+
117
+ test 'raises configuration error if primary does not support buffering' do
118
+ i = create_output(:sync)
119
+ assert_raise Fluent::ConfigError do
120
+ i.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'output_secondary_test'})]))
121
+ end
122
+ end
123
+
124
+ test 'raises configuration error if <buffer>/<secondary> section is specified in <secondary> section' do
125
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
126
+ secconf1 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('buffer', 'time')])
127
+ secconf2 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('secondary', '')])
128
+ i = create_output()
129
+ assert_raise Fluent::ConfigError do
130
+ i.configure(config_element('ROOT','',{},[priconf,secconf1]))
131
+ end
132
+ assert_raise Fluent::ConfigError do
133
+ i.configure(config_element('ROOT','',{},[priconf,secconf2]))
134
+ end
135
+ end
136
+
137
+ test 'uses same plugin type with primary if @type is missing in secondary' do
138
+ bufconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
139
+ secconf = config_element('secondary','',{})
140
+ priconf = config_element('ROOT', '', {'@type' => 'output_secondary_test'}, [bufconf, secconf])
141
+ i = create_output()
142
+ assert_nothing_raised do
143
+ i.configure(priconf)
144
+ end
145
+ logs = i.log.out.logs
146
+ assert{ logs.empty? }
147
+ assert{ i.secondary.is_a? FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput }
148
+ end
149
+
150
+ test 'warns if secondary plugin is different type from primary one' do
151
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
152
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
153
+ i = create_output()
154
+ i.configure(config_element('ROOT','',{},[priconf,secconf]))
155
+ logs = i.log.out.logs
156
+ assert{ logs.any?{|l| l.include?("Use different plugin for secondary. Check the plugin works with primary like secondary_file") } }
157
+ end
158
+
159
+ test 'secondary plugin lifecycle is kicked by primary' do
160
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
161
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
162
+ i = create_output()
163
+ i.configure(config_element('ROOT','',{},[priconf,secconf]))
164
+ logs = i.log.out.logs
165
+ assert{ logs.any?{|l| l.include?("Use different plugin for secondary. Check the plugin works with primary like secondary_file") } }
166
+
167
+ assert i.secondary.configured?
168
+
169
+ assert !i.secondary.started?
170
+ i.start
171
+ assert i.secondary.started?
172
+
173
+ assert !i.secondary.after_started?
174
+ i.after_start
175
+ assert i.secondary.after_started?
176
+
177
+ assert !i.secondary.stopped?
178
+ i.stop
179
+ assert i.secondary.stopped?
180
+
181
+ assert !i.secondary.before_shutdown?
182
+ i.before_shutdown
183
+ assert i.secondary.before_shutdown?
184
+
185
+ assert !i.secondary.shutdown?
186
+ i.shutdown
187
+ assert i.secondary.shutdown?
188
+
189
+ assert !i.secondary.after_shutdown?
190
+ i.after_shutdown
191
+ assert i.secondary.after_shutdown?
192
+
193
+ assert !i.secondary.closed?
194
+ i.close
195
+ assert i.secondary.closed?
196
+
197
+ assert !i.secondary.terminated?
198
+ i.terminate
199
+ assert i.secondary.terminated?
200
+ end
201
+
202
+ test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do
203
+ written = []
204
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
205
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
206
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
207
+ @i.register(:prefer_buffered_processing){ true }
208
+ @i.register(:prefer_delayed_commit){ false }
209
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
210
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
211
+ @i.secondary.register(:prefer_delayed_commit){ false }
212
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
213
+ @i.start
214
+ @i.after_start
215
+
216
+ @i.interrupt_flushes
217
+
218
+ now = Time.parse('2016-04-13 18:33:30 -0700')
219
+ Timecop.freeze( now )
220
+
221
+ @i.emit_events("test.tag.1", dummy_event_stream())
222
+
223
+ now = Time.parse('2016-04-13 18:33:31 -0700')
224
+ Timecop.freeze( now )
225
+
226
+ @i.emit_events("test.tag.2", dummy_event_stream())
227
+
228
+ assert_equal 0, @i.write_count
229
+ assert_equal 0, @i.num_errors
230
+
231
+ @i.enqueue_thread_wait
232
+ @i.flush_thread_wakeup
233
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
234
+
235
+ assert{ @i.buffer.queue.size > 0 }
236
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
237
+
238
+ assert{ @i.write_count > 0 }
239
+ assert{ @i.num_errors > 0 }
240
+
241
+ prev_write_count = @i.write_count
242
+ prev_num_errors = @i.num_errors
243
+
244
+ first_failure = @i.retry.start
245
+
246
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
247
+ now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary
248
+ Timecop.freeze( now )
249
+
250
+ unless @i.retry.secondary?
251
+ @i.enqueue_thread_wait
252
+ @i.flush_thread_wakeup
253
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
254
+
255
+ prev_write_count = @i.write_count
256
+ prev_num_errors = @i.num_errors
257
+
258
+ # next step is on secondary
259
+ now = first_failure + 60 * 0.8 + 10
260
+ Timecop.freeze( now )
261
+ end
262
+
263
+ @i.enqueue_thread_wait
264
+ @i.flush_thread_wakeup
265
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
266
+
267
+ current_write_count = @i.write_count
268
+ current_num_errors = @i.num_errors
269
+ assert{ current_write_count > prev_write_count }
270
+ assert{ current_num_errors == prev_num_errors }
271
+
272
+ assert_nil @i.retry
273
+
274
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
275
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
276
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
277
+
278
+ logs = @i.log.out.logs
279
+ waiting(4){ sleep 0.1 until logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
280
+ assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
281
+ end
282
+
283
+ test 'secondary can do non-delayed commit even if primary do delayed commit' do
284
+ written = []
285
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
286
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
287
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
288
+ @i.register(:prefer_buffered_processing){ true }
289
+ @i.register(:prefer_delayed_commit){ true }
290
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
291
+ @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
292
+ @i.secondary.register(:prefer_delayed_commit){ false }
293
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
294
+ @i.start
295
+ @i.after_start
296
+
297
+ @i.interrupt_flushes
298
+
299
+ now = Time.parse('2016-04-13 18:33:30 -0700')
300
+ Timecop.freeze( now )
301
+
302
+ @i.emit_events("test.tag.1", dummy_event_stream())
303
+
304
+ now = Time.parse('2016-04-13 18:33:31 -0700')
305
+ Timecop.freeze( now )
306
+
307
+ @i.emit_events("test.tag.2", dummy_event_stream())
308
+
309
+ assert_equal 0, @i.write_count
310
+ assert_equal 0, @i.num_errors
311
+
312
+ @i.enqueue_thread_wait
313
+ @i.flush_thread_wakeup
314
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
315
+
316
+ assert{ @i.buffer.queue.size > 0 }
317
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
318
+
319
+ assert{ @i.write_count > 0 }
320
+ assert{ @i.num_errors > 0 }
321
+
322
+ prev_write_count = @i.write_count
323
+ prev_num_errors = @i.num_errors
324
+
325
+ first_failure = @i.retry.start
326
+
327
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
328
+ now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary
329
+ Timecop.freeze( now )
330
+
331
+ unless @i.retry.secondary?
332
+ @i.enqueue_thread_wait
333
+ @i.flush_thread_wakeup
334
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
335
+
336
+ prev_write_count = @i.write_count
337
+ prev_num_errors = @i.num_errors
338
+
339
+ # next step is on secondary
340
+ now = first_failure + 60 * 0.8 + 10
341
+ Timecop.freeze( now )
342
+ end
343
+
344
+ @i.enqueue_thread_wait
345
+ @i.flush_thread_wakeup
346
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
347
+
348
+ assert{ @i.write_count > prev_write_count }
349
+ assert{ @i.num_errors == prev_num_errors }
350
+
351
+ assert_nil @i.retry
352
+
353
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
354
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
355
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
356
+
357
+ logs = @i.log.out.logs
358
+ waiting(4){ sleep 0.1 until logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
359
+ assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
360
+ end
361
+
362
+ test 'secondary plugin can do delayed commit if primary do it' do
363
+ written = []
364
+ chunks = []
365
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
366
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
367
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
368
+ @i.register(:prefer_buffered_processing){ true }
369
+ @i.register(:prefer_delayed_commit){ true }
370
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
371
+ @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
372
+ @i.secondary.register(:prefer_delayed_commit){ true }
373
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
374
+ @i.start
375
+ @i.after_start
376
+
377
+ @i.interrupt_flushes
378
+
379
+ now = Time.parse('2016-04-13 18:33:30 -0700')
380
+ Timecop.freeze( now )
381
+
382
+ @i.emit_events("test.tag.1", dummy_event_stream())
383
+
384
+ now = Time.parse('2016-04-13 18:33:31 -0700')
385
+ Timecop.freeze( now )
386
+
387
+ @i.emit_events("test.tag.2", dummy_event_stream())
388
+
389
+ assert_equal 0, @i.write_count
390
+ assert_equal 0, @i.num_errors
391
+
392
+ @i.enqueue_thread_wait
393
+ @i.flush_thread_wakeup
394
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
395
+
396
+ assert{ @i.buffer.queue.size > 0 }
397
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
398
+
399
+ assert{ @i.write_count > 0 }
400
+ assert{ @i.num_errors > 0 }
401
+
402
+ prev_write_count = @i.write_count
403
+ prev_num_errors = @i.num_errors
404
+
405
+ first_failure = @i.retry.start
406
+
407
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
408
+ now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary
409
+ Timecop.freeze( now )
410
+
411
+ unless @i.retry.secondary?
412
+ @i.enqueue_thread_wait
413
+ @i.flush_thread_wakeup
414
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
415
+
416
+ prev_write_count = @i.write_count
417
+ prev_num_errors = @i.num_errors
418
+
419
+ # next step is on secondary
420
+ now = first_failure + 60 * 0.8 + 10
421
+ Timecop.freeze( now )
422
+ end
423
+
424
+ @i.enqueue_thread_wait
425
+ @i.flush_thread_wakeup
426
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
427
+
428
+ assert{ @i.write_count > prev_write_count }
429
+ assert{ @i.num_errors == prev_num_errors }
430
+
431
+ assert @i.retry
432
+
433
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
434
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
435
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
436
+
437
+ assert{ @i.buffer.dequeued.size > 0 }
438
+ assert{ chunks.size > 0 }
439
+ assert{ !chunks.first.empty? }
440
+
441
+ @i.secondary.commit_write(chunks[0].unique_id)
442
+
443
+ assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }
444
+ assert{ chunks.first.empty? }
445
+
446
+ assert_nil @i.retry
447
+
448
+ logs = @i.log.out.logs
449
+ waiting(4){ sleep 0.1 until logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
450
+ assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
451
+ end
452
+
453
+ test 'secondary plugin can do delayed commit even if primary does not do it' do
454
+ written = []
455
+ chunks = []
456
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
457
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
458
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
459
+ @i.register(:prefer_buffered_processing){ true }
460
+ @i.register(:prefer_delayed_commit){ false }
461
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
462
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
463
+ @i.secondary.register(:prefer_delayed_commit){ true }
464
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
465
+ @i.start
466
+ @i.after_start
467
+
468
+ @i.interrupt_flushes
469
+
470
+ now = Time.parse('2016-04-13 18:33:30 -0700')
471
+ Timecop.freeze( now )
472
+
473
+ @i.emit_events("test.tag.1", dummy_event_stream())
474
+
475
+ now = Time.parse('2016-04-13 18:33:31 -0700')
476
+ Timecop.freeze( now )
477
+
478
+ @i.emit_events("test.tag.2", dummy_event_stream())
479
+
480
+ assert_equal 0, @i.write_count
481
+ assert_equal 0, @i.num_errors
482
+
483
+ @i.enqueue_thread_wait
484
+ @i.flush_thread_wakeup
485
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
486
+
487
+ assert{ @i.buffer.queue.size > 0 }
488
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
489
+
490
+ assert{ @i.write_count > 0 }
491
+ assert{ @i.num_errors > 0 }
492
+
493
+ prev_write_count = @i.write_count
494
+ prev_num_errors = @i.num_errors
495
+
496
+ first_failure = @i.retry.start
497
+
498
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
499
+ now = first_failure + 60 * 0.8 + 1 # to step from primary to secondary
500
+ Timecop.freeze( now )
501
+
502
+ unless @i.retry.secondary?
503
+ @i.enqueue_thread_wait
504
+ @i.flush_thread_wakeup
505
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
506
+
507
+ prev_write_count = @i.write_count
508
+ prev_num_errors = @i.num_errors
509
+
510
+ # next step is on secondary
511
+ now = first_failure + 60 * 0.8 + 10
512
+ Timecop.freeze( now )
513
+ end
514
+
515
+ @i.enqueue_thread_wait
516
+ @i.flush_thread_wakeup
517
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
518
+
519
+ assert{ @i.write_count > prev_write_count }
520
+ assert{ @i.num_errors == prev_num_errors }
521
+
522
+ assert @i.retry
523
+
524
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
525
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
526
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
527
+
528
+ assert{ @i.buffer.dequeued.size > 0 }
529
+ assert{ chunks.size > 0 }
530
+ assert{ !chunks.first.empty? }
531
+
532
+ @i.secondary.commit_write(chunks[0].unique_id)
533
+
534
+ assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }
535
+ assert{ chunks.first.empty? }
536
+
537
+ assert_nil @i.retry
538
+
539
+ logs = @i.log.out.logs
540
+ waiting(4){ sleep 0.1 until logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
541
+ assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
542
+ end
543
+
544
+ test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
545
+ written = []
546
+ chunks = []
547
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})
548
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
549
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
550
+ @i.register(:prefer_buffered_processing){ true }
551
+ @i.register(:prefer_delayed_commit){ false }
552
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
553
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
554
+ @i.secondary.register(:prefer_delayed_commit){ true }
555
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
556
+ @i.secondary.register(:write){|chunk| raise "don't use this" }
557
+ @i.start
558
+ @i.after_start
559
+
560
+ @i.interrupt_flushes
561
+
562
+ now = Time.parse('2016-04-13 18:33:30 -0700')
563
+ Timecop.freeze( now )
564
+
565
+ @i.emit_events("test.tag.1", dummy_event_stream())
566
+ @i.emit_events("test.tag.2", dummy_event_stream())
567
+
568
+ now = Time.parse('2016-04-13 18:33:31 -0700')
569
+ Timecop.freeze( now )
570
+
571
+ assert_equal 0, @i.write_count
572
+ assert_equal 0, @i.num_errors
573
+
574
+ @i.enqueue_thread_wait
575
+ @i.flush_thread_wakeup
576
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
577
+
578
+ assert{ @i.buffer.queue.size == 2 }
579
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
580
+
581
+ assert{ @i.write_count > 0 }
582
+ assert{ @i.num_errors > 0 }
583
+
584
+ prev_write_count = @i.write_count
585
+ prev_num_errors = @i.num_errors
586
+
587
+ first_failure = @i.retry.start
588
+
589
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
590
+
591
+ now = first_failure + 60 * 0.8 + 1
592
+ Timecop.freeze( now )
593
+ @i.enqueue_thread_wait
594
+ @i.flush_thread_wakeup
595
+ now = first_failure + 60 * 0.8 + 2
596
+ Timecop.freeze( now )
597
+ @i.enqueue_thread_wait
598
+ @i.flush_thread_wakeup
599
+
600
+ waiting(4){ sleep 0.1 until chunks.size == 2 }
601
+
602
+ assert{ @i.write_count > prev_write_count }
603
+ assert{ @i.num_errors == prev_num_errors }
604
+
605
+ assert @i.retry
606
+
607
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
608
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
609
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
610
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[3]
611
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[4]
612
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[5]
613
+
614
+ assert{ @i.buffer.dequeued.size == 2 }
615
+ assert{ chunks.size == 2 }
616
+ assert{ !chunks[0].empty? }
617
+ assert{ !chunks[1].empty? }
618
+
619
+ 30.times do |i| # large enough
620
+ # In https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb
621
+ # @timeout_at is 2016-04-13 18:34:31, @next_time must be less than 2016-04-13 18:34:30
622
+ #
623
+ # first_failure + 60 * 0.8 + 2 # => 2016-04-13 18:34:21
624
+ # @next_time is not added by 1, but by randomize(@retry_wait) https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb#L196
625
+ # current_time(=Time.now) + randomize(@retry_wait) < @timeout_at
626
+ # (2016-04-13 18:34:21 + 6) + 3 < 2016-04-13 18:34:31
627
+ # So, current_time must be at most 6
628
+ now = first_failure + 60 * 0.8 + 2 + [i, 6].min
629
+ Timecop.freeze( now )
630
+ @i.flush_thread_wakeup
631
+
632
+ break if @i.buffer.dequeued.size == 0
633
+ end
634
+
635
+ assert @i.retry
636
+ logs = @i.log.out.logs
637
+ waiting(4){ sleep 0.1 until logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
638
+ assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
639
+ end
640
+
641
+ test 'retry_wait for secondary is same with one for primary' do
642
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
643
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
644
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
645
+ @i.register(:prefer_buffered_processing){ true }
646
+ @i.register(:prefer_delayed_commit){ false }
647
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
648
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
649
+ @i.secondary.register(:prefer_delayed_commit){ false }
650
+ @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
651
+ @i.start
652
+ @i.after_start
653
+
654
+ @i.interrupt_flushes
655
+
656
+ now = Time.parse('2016-04-13 18:33:30 -0700')
657
+ Timecop.freeze( now )
658
+
659
+ @i.emit_events("test.tag.1", dummy_event_stream())
660
+
661
+ now = Time.parse('2016-04-13 18:33:31 -0700')
662
+ Timecop.freeze( now )
663
+
664
+ @i.emit_events("test.tag.2", dummy_event_stream())
665
+
666
+ assert_equal 0, @i.write_count
667
+ assert_equal 0, @i.num_errors
668
+
669
+ @i.enqueue_thread_wait
670
+ @i.flush_thread_wakeup
671
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
672
+
673
+ assert{ @i.buffer.queue.size > 0 }
674
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
675
+
676
+ assert{ @i.write_count > 0 }
677
+ assert{ @i.num_errors > 0 }
678
+
679
+ prev_write_count = @i.write_count
680
+ prev_num_errors = @i.num_errors
681
+
682
+ first_failure = @i.retry.start
683
+
684
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
685
+
686
+ now = first_failure + 60 * 0.8 + 1
687
+
688
+ Timecop.freeze( now )
689
+ @i.enqueue_thread_wait
690
+ @i.flush_thread_wakeup
691
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
692
+
693
+ assert{ @i.write_count > prev_write_count }
694
+ assert{ @i.num_errors > prev_num_errors }
695
+
696
+ assert @i.retry
697
+
698
+ assert_equal 3, (@i.next_flush_time - Time.now)
699
+
700
+ logs = @i.log.out.logs
701
+ waiting(4){ sleep 0.1 until logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } }
702
+ assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } }
703
+ end
704
+ end
705
+
706
+ sub_test_case 'secondary plugin feature for buffered output with exponential backoff' do
707
+ setup do
708
+ Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)
709
+ Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)
710
+ end
711
+
712
+ test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do
713
+ written = []
714
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})
715
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
716
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
717
+ @i.register(:prefer_buffered_processing){ true }
718
+ @i.register(:prefer_delayed_commit){ false }
719
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
720
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
721
+ @i.secondary.register(:prefer_delayed_commit){ false }
722
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
723
+ @i.start
724
+ @i.after_start
725
+
726
+ @i.interrupt_flushes
727
+
728
+ now = Time.parse('2016-04-13 18:33:30 -0700')
729
+ Timecop.freeze( now )
730
+
731
+ @i.emit_events("test.tag.1", dummy_event_stream())
732
+
733
+ now = Time.parse('2016-04-13 18:33:31 -0700')
734
+ Timecop.freeze( now )
735
+
736
+ @i.emit_events("test.tag.2", dummy_event_stream())
737
+
738
+ assert_equal 0, @i.write_count
739
+ assert_equal 0, @i.num_errors
740
+
741
+ @i.enqueue_thread_wait
742
+ @i.flush_thread_wakeup
743
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
744
+
745
+ assert{ @i.buffer.queue.size > 0 }
746
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
747
+
748
+ assert{ @i.write_count > 0 }
749
+ assert{ @i.num_errors > 0 }
750
+
751
+ prev_write_count = @i.write_count
752
+ first_failure = @i.retry.start
753
+
754
+ 20.times do |i| # large enough
755
+ now = @i.next_flush_time
756
+ Timecop.freeze( now )
757
+ @i.enqueue_thread_wait
758
+ @i.flush_thread_wakeup
759
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
760
+
761
+ assert{ @i.write_count > prev_write_count }
762
+
763
+ break if @i.buffer.queue.size == 0
764
+
765
+ prev_write_count = @i.write_count
766
+ end
767
+
768
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
769
+
770
+ assert{ now >= first_failure + 60 * 0.8 }
771
+
772
+ assert_nil @i.retry
773
+
774
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
775
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
776
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
777
+
778
+ assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
779
+ end
780
+
781
+ test 'exponential backoff interval will be initialized when switched to secondary' do
782
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})
783
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
784
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
785
+ @i.register(:prefer_buffered_processing){ true }
786
+ @i.register(:prefer_delayed_commit){ false }
787
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
788
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
789
+ @i.secondary.register(:prefer_delayed_commit){ false }
790
+ @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
791
+ @i.start
792
+ @i.after_start
793
+
794
+ @i.interrupt_flushes
795
+
796
+ now = Time.parse('2016-04-13 18:33:30 -0700')
797
+ Timecop.freeze( now )
798
+
799
+ @i.emit_events("test.tag.1", dummy_event_stream())
800
+
801
+ now = Time.parse('2016-04-13 18:33:31 -0700')
802
+ Timecop.freeze( now )
803
+
804
+ @i.emit_events("test.tag.2", dummy_event_stream())
805
+
806
+ assert_equal 0, @i.write_count
807
+ assert_equal 0, @i.num_errors
808
+
809
+ @i.enqueue_thread_wait
810
+ @i.flush_thread_wakeup
811
+ waiting(4){ sleep 0.1 until @i.write_count > 0 && @i.num_errors > 0 }
812
+
813
+ assert{ @i.buffer.queue.size > 0 }
814
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
815
+
816
+ assert{ @i.write_count > 0 }
817
+ assert{ @i.num_errors > 0 }
818
+
819
+ prev_write_count = @i.write_count
820
+ prev_num_errors = @i.num_errors
821
+
822
+ first_failure = @i.retry.start
823
+
824
+ 20.times do |i| # large enough
825
+ now = @i.next_flush_time
826
+ # p({i: i, now: now, diff: (now - Time.now)})
827
+ # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0}
828
+ # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0}
829
+ # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0}
830
+ # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0}
831
+ # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0}
832
+ # {:i=>5, :now=>2016-04-13 18:34:19 -0700, :diff=>17.0}
833
+ Timecop.freeze( now )
834
+ @i.enqueue_thread_wait
835
+ @i.flush_thread_wakeup
836
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
837
+
838
+ assert{ @i.write_count > prev_write_count }
839
+ assert{ @i.num_errors > prev_num_errors }
840
+
841
+ prev_write_count = @i.write_count
842
+ prev_num_errors = @i.num_errors
843
+
844
+ break if @i.retry.secondary?
845
+
846
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
847
+ end
848
+
849
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
850
+
851
+ assert{ now >= first_failure + 60 * 0.8 }
852
+ assert @i.retry
853
+ logs = @i.log.out.logs
854
+ assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } }
855
+
856
+ assert{ (@i.next_flush_time - Time.now) <= 2 } # <= retry_wait (1s) * base (2) ** 1
857
+
858
+ 20.times do |i| # large enough again
859
+ now = @i.next_flush_time
860
+ # p({i: i, now: now, diff: (now - Time.now)})
861
+ # {:i=>0, :now=>2016-04-13 18:34:20 -0700, :diff=>1.0}
862
+ # {:i=>1, :now=>2016-04-13 18:34:24 -0700, :diff=>4.0}
863
+ # {:i=>2, :now=>2016-04-13 18:34:31 -0700, :diff=>7.0}
864
+
865
+ Timecop.freeze( now )
866
+ @i.enqueue_thread_wait
867
+ @i.flush_thread_wakeup
868
+ waiting(4){ sleep 0.1 until @i.write_count > prev_write_count }
869
+
870
+ assert{ @i.write_count > prev_write_count }
871
+ assert{ @i.num_errors > prev_num_errors }
872
+
873
+ break if @i.buffer.queue.size == 0
874
+ end
875
+
876
+ logs = @i.log.out.logs
877
+ assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
878
+
879
+ assert{ now >= first_failure + 60 }
880
+ end
881
+ end
882
+ end