fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,1556 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/env'
18
+ require 'fluent/error'
19
+ require 'fluent/plugin/base'
20
+ require 'fluent/plugin/buffer'
21
+ require 'fluent/plugin_helper/record_accessor'
22
+ require 'fluent/msgpack_factory'
23
+ require 'fluent/log'
24
+ require 'fluent/plugin_id'
25
+ require 'fluent/plugin_helper'
26
+ require 'fluent/timezone'
27
+ require 'fluent/unique_id'
28
+ require 'fluent/clock'
29
+ require 'fluent/ext_monitor_require'
30
+
31
+ require 'time'
32
+
33
+ module Fluent
34
+ module Plugin
35
+ class Output < Base
36
+ include PluginId
37
+ include PluginLoggerMixin
38
+ include PluginHelper::Mixin
39
+ include UniqueId::Mixin
40
+
41
+ helpers_internal :thread, :retry_state, :metrics
42
+
43
+ CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
44
+ CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
45
+ CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
46
+ CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
47
+
48
+ CHUNKING_FIELD_WARN_NUM = 4
49
+
50
+ config_param :time_as_integer, :bool, default: false
51
+ desc 'The threshold to show slow flush logs'
52
+ config_param :slow_flush_log_threshold, :float, default: 20.0
53
+
54
+ # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
55
+ config_section :buffer, param_name: :buffer_config, init: true, required: false, multi: false, final: true do
56
+ config_argument :chunk_keys, :array, value_type: :string, default: []
57
+ config_param :@type, :string, default: 'memory', alias: :type
58
+
59
+ config_param :timekey, :time, default: nil # range size to be used: `time.to_i / @timekey`
60
+ config_param :timekey_wait, :time, default: 600
61
+ # These are for #extract_placeholders
62
+ config_param :timekey_use_utc, :bool, default: false # default is localtime
63
+ config_param :timekey_zone, :string, default: Time.now.strftime('%z') # e.g., "-0700" or "Asia/Tokyo"
64
+
65
+ desc 'If true, plugin will try to flush buffer just before shutdown.'
66
+ config_param :flush_at_shutdown, :bool, default: nil # change default by buffer_plugin.persistent?
67
+
68
+ desc 'How to enqueue chunks to be flushed. "interval" flushes per flush_interval, "immediate" flushes just after event arrival.'
69
+ config_param :flush_mode, :enum, list: [:default, :lazy, :interval, :immediate], default: :default
70
+ config_param :flush_interval, :time, default: 60, desc: 'The interval between buffer chunk flushes.'
71
+
72
+ config_param :flush_thread_count, :integer, default: 1, desc: 'The number of threads to flush the buffer.'
73
+
74
+ config_param :flush_thread_interval, :float, default: 1.0, desc: 'Seconds to sleep between checks for buffer flushes in flush threads.'
75
+ config_param :flush_thread_burst_interval, :float, default: 1.0, desc: 'Seconds to sleep between flushes when many buffer chunks are queued.'
76
+
77
+ config_param :delayed_commit_timeout, :time, default: 60, desc: 'Seconds of timeout for buffer chunks to be committed by plugins later.'
78
+
79
+ config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :throw_exception, desc: 'The action when the size of buffer exceeds the limit.'
80
+
81
+ config_param :retry_forever, :bool, default: false, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry flushing forever.'
82
+ config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'
83
+ # 72hours == 17 times with exponential backoff (not to change default behavior)
84
+ config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
85
+
86
+ config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
87
+ # exponential backoff sequence will be initialized at the time of this threshold
88
+
89
+ desc 'How to wait next retry to flush buffer.'
90
+ config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
91
+ ### Periodic -> fixed :retry_wait
92
+ ### Exponential backoff: k is number of retry times
93
+ # c: constant factor, @retry_wait
94
+ # b: base factor, @retry_exponential_backoff_base
95
+ # k: times
96
+ # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
97
+ config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
98
+ config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
99
+ config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
100
+
101
+ config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
102
+ config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
103
+ end
104
+
105
+ config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
106
+ config_param :@type, :string, default: nil, alias: :type
107
+ config_section :buffer, required: false, multi: false do
108
+ # dummy to detect invalid specification for here
109
+ end
110
+ config_section :secondary, required: false, multi: false do
111
+ # dummy to detect invalid specification for here
112
+ end
113
+ end
114
+
115
+ def process(tag, es)
116
+ raise NotImplementedError, "BUG: output plugins MUST implement this method"
117
+ end
118
+
119
+ def write(chunk)
120
+ raise NotImplementedError, "BUG: output plugins MUST implement this method"
121
+ end
122
+
123
+ def try_write(chunk)
124
+ raise NotImplementedError, "BUG: output plugins MUST implement this method"
125
+ end
126
+
127
+ def format(tag, time, record)
128
+ # standard msgpack_event_stream chunk will be used if this method is not implemented in plugin subclass
129
+ raise NotImplementedError, "BUG: output plugins MUST implement this method"
130
+ end
131
+
132
+ def formatted_to_msgpack_binary?
133
+ # To indicate custom format method (#format) returns msgpack binary or not.
134
+ # If #format returns msgpack binary, override this method to return true.
135
+ false
136
+ end
137
+
138
+ # Compatibility for existing plugins
139
+ def formatted_to_msgpack_binary
140
+ formatted_to_msgpack_binary?
141
+ end
142
+
143
+ def prefer_buffered_processing
144
+ # override this method to return false only when all of these are true:
145
+ # * plugin has both implementation for buffered and non-buffered methods
146
+ # * plugin is expected to work as non-buffered plugin if no `<buffer>` sections specified
147
+ true
148
+ end
149
+
150
+ def prefer_delayed_commit
151
+ # override this method to decide which is used of `write` or `try_write` if both are implemented
152
+ true
153
+ end
154
+
155
+ def multi_workers_ready?
156
+ false
157
+ end
158
+
159
+ # Internal states
160
+ FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)
161
+ DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
162
+ def expired?
163
+ time + timeout < Time.now
164
+ end
165
+ end
166
+
167
+ attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
168
+
169
+ # for tests
170
+ attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
171
+ attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
172
+ # output_enqueue_thread_waiting: for test of output.rb itself
173
+ attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
174
+
175
+ def num_errors
176
+ @num_errors_metrics.get
177
+ end
178
+
179
+ def emit_count
180
+ @emit_count_metrics.get
181
+ end
182
+
183
+ def emit_size
184
+ @emit_size_metrics.get
185
+ end
186
+
187
+ def emit_records
188
+ @emit_records_metrics.get
189
+ end
190
+
191
+ def write_count
192
+ @write_count_metrics.get
193
+ end
194
+
195
+ def rollback_count
196
+ @rollback_count_metrics.get
197
+ end
198
+
199
+ def initialize
200
+ super
201
+ @counter_mutex = Mutex.new
202
+ @buffering = false
203
+ @delayed_commit = false
204
+ @as_secondary = false
205
+ @primary_instance = nil
206
+
207
+ # TODO: well organized counters
208
+ @num_errors_metrics = nil
209
+ @emit_count_metrics = nil
210
+ @emit_records_metrics = nil
211
+ @emit_size_metrics = nil
212
+ @write_count_metrics = nil
213
+ @rollback_count_metrics = nil
214
+ @flush_time_count_metrics = nil
215
+ @slow_flush_count_metrics = nil
216
+ @enable_size_metrics = false
217
+
218
+ # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
219
+ if implement?(:synchronous)
220
+ if implement?(:buffered) || implement?(:delayed_commit)
221
+ @buffering = nil # do #configure or #start to determine this for full-featured plugins
222
+ else
223
+ @buffering = false
224
+ end
225
+ else
226
+ @buffering = true
227
+ end
228
+ @custom_format = implement?(:custom_format)
229
+ @enable_msgpack_streamer = false # decided later
230
+
231
+ @buffer = nil
232
+ @secondary = nil
233
+ @retry = nil
234
+ @dequeued_chunks = nil
235
+ @dequeued_chunks_mutex = nil
236
+ @output_enqueue_thread = nil
237
+ @output_flush_threads = nil
238
+
239
+ @simple_chunking = nil
240
+ @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
241
+ @flush_mode = nil
242
+ @timekey_zone = nil
243
+
244
+ @retry_for_error_chunk = false
245
+ end
246
+
247
+ def acts_as_secondary(primary)
248
+ @as_secondary = true
249
+ @primary_instance = primary
250
+ @chunk_keys = @primary_instance.chunk_keys || []
251
+ @chunk_key_tag = @primary_instance.chunk_key_tag || false
252
+ if @primary_instance.chunk_key_time
253
+ @chunk_key_time = @primary_instance.chunk_key_time
254
+ @timekey_zone = @primary_instance.timekey_zone
255
+ @output_time_formatter_cache = {}
256
+ end
257
+ self.context_router = primary.context_router
258
+
259
+ singleton_class.module_eval do
260
+ define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
261
+ define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) }
262
+ end
263
+ end
264
+
265
+ def configure(conf)
266
+ unless implement?(:synchronous) || implement?(:buffered) || implement?(:delayed_commit)
267
+ raise "BUG: output plugin must implement some methods. see developer documents."
268
+ end
269
+
270
+ has_buffer_section = (conf.elements(name: 'buffer').size > 0)
271
+ has_flush_interval = conf.has_key?('flush_interval')
272
+
273
+ super
274
+
275
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
276
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
277
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
278
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
279
+ @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
280
+ @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
281
+ @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
282
+ @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
283
+
284
+ if has_buffer_section
285
+ unless implement?(:buffered) || implement?(:delayed_commit)
286
+ raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
287
+ end
288
+ @buffering = true
289
+ else # no buffer sections
290
+ if implement?(:synchronous)
291
+ if !implement?(:buffered) && !implement?(:delayed_commit)
292
+ if @as_secondary
293
+ raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't."
294
+ end
295
+ @buffering = false
296
+ else
297
+ if @as_secondary
298
+ # secondary plugin always works as buffered plugin without buffer instance
299
+ @buffering = true
300
+ else
301
+ # @buffering.nil? shows that enabling buffering or not will be decided in lazy way in #start
302
+ @buffering = nil
303
+ end
304
+ end
305
+ else # buffered or delayed_commit is supported by `unless` of first line in this method
306
+ @buffering = true
307
+ end
308
+ end
309
+ # Enable to update record size metrics or not
310
+ @enable_size_metrics = !!system_config.enable_size_metrics
311
+
312
+ if @as_secondary
313
+ if !@buffering && !@buffering.nil?
314
+ raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't"
315
+ end
316
+ end
317
+
318
+ if (@buffering || @buffering.nil?) && !@as_secondary
319
+ # When @buffering.nil?, @buffer_config was initialized with default value for all parameters.
320
+ # If so, this configuration MUST success.
321
+ @chunk_keys = @buffer_config.chunk_keys.dup
322
+ @chunk_key_time = !!@chunk_keys.delete('time')
323
+ @chunk_key_tag = !!@chunk_keys.delete('tag')
324
+ if @chunk_keys.any? { |key|
325
+ begin
326
+ k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)
327
+ if k.is_a?(String)
328
+ k !~ CHUNK_KEY_PATTERN
329
+ else
330
+ if key.start_with?('$[')
331
+ raise Fluent::ConfigError, "in chunk_keys: bracket notation is not allowed"
332
+ else
333
+ false
334
+ end
335
+ end
336
+ rescue => e
337
+ raise Fluent::ConfigError, "in chunk_keys: #{e.message}"
338
+ end
339
+ }
340
+ raise Fluent::ConfigError, "chunk_keys specification includes invalid char"
341
+ else
342
+ @chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]
343
+ end
344
+
345
+ if @chunk_key_time
346
+ raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
347
+ Fluent::Timezone.validate!(@buffer_config.timekey_zone)
348
+ @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
349
+ @timekey = @buffer_config.timekey
350
+ if @timekey <= 0
351
+ raise Fluent::ConfigError, "timekey should be greater than 0. current timekey: #{@timekey}"
352
+ end
353
+ @timekey_use_utc = @buffer_config.timekey_use_utc
354
+ @offset = Fluent::Timezone.utc_offset(@timekey_zone)
355
+ @calculate_offset = @offset.respond_to?(:call) ? @offset : nil
356
+ @output_time_formatter_cache = {}
357
+ end
358
+
359
+ if (@chunk_key_tag ? 1 : 0) + @chunk_keys.size >= CHUNKING_FIELD_WARN_NUM
360
+ log.warn "many chunk keys specified, and it may cause too many chunks on your system."
361
+ end
362
+
363
+ # no chunk keys or only tags (chunking can be done without iterating event stream)
364
+ @simple_chunking = !@chunk_key_time && @chunk_keys.empty?
365
+
366
+ @flush_mode = @buffer_config.flush_mode
367
+ if @flush_mode == :default
368
+ if has_flush_interval
369
+ log.info "'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour"
370
+ @flush_mode = :interval
371
+ else
372
+ @flush_mode = (@chunk_key_time ? :lazy : :interval)
373
+ end
374
+ end
375
+
376
+ buffer_type = @buffer_config[:@type]
377
+ buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
378
+ @buffer = Plugin.new_buffer(buffer_type, parent: self)
379
+ @buffer.configure(buffer_conf)
380
+ @buffer.enable_update_timekeys if @chunk_key_time
381
+
382
+ @flush_at_shutdown = @buffer_config.flush_at_shutdown
383
+ if @flush_at_shutdown.nil?
384
+ @flush_at_shutdown = if @buffer.persistent?
385
+ false
386
+ else
387
+ true # flush_at_shutdown is true in default for on-memory buffer
388
+ end
389
+ elsif !@flush_at_shutdown && !@buffer.persistent?
390
+ buf_type = Plugin.lookup_type_from_class(@buffer.class)
391
+ log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
392
+ log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
393
+ end
394
+
395
+ if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
396
+ if buffer_conf.has_key?('flush_mode')
397
+ raise Fluent::ConfigError, "'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{@flush_mode}'"
398
+ else
399
+ log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
400
+ end
401
+ end
402
+
403
+ if @buffer.queued_chunks_limit_size.nil?
404
+ @buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count
405
+ end
406
+ end
407
+
408
+ if @secondary_config
409
+ raise Fluent::ConfigError, "Invalid <secondary> section for non-buffered plugin" unless @buffering
410
+ raise Fluent::ConfigError, "<secondary> section cannot have <buffer> section" if @secondary_config.buffer
411
+ raise Fluent::ConfigError, "<secondary> section cannot have <secondary> section" if @secondary_config.secondary
412
+ if @buffer_config.retry_forever
413
+ log.warn "<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary"
414
+ end
415
+
416
+ secondary_type = @secondary_config[:@type]
417
+ unless secondary_type
418
+ secondary_type = conf['@type'] # primary plugin type
419
+ end
420
+ secondary_conf = conf.elements(name: 'secondary').first
421
+ @secondary = Plugin.new_output(secondary_type)
422
+ unless @secondary.respond_to?(:acts_as_secondary)
423
+ raise Fluent::ConfigError, "Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output"
424
+ end
425
+ @secondary.acts_as_secondary(self)
426
+ @secondary.configure(secondary_conf)
427
+ if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
428
+ log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
429
+ end
430
+ else
431
+ @secondary = nil
432
+ end
433
+
434
+ self
435
+ end
436
+
437
+ def start
438
+ super
439
+
440
+ if @buffering.nil?
441
+ @buffering = prefer_buffered_processing
442
+ if !@buffering && @buffer
443
+ @buffer.terminate # it's not started, so terminate will be enough
444
+ # At here, this plugin works as non-buffered plugin.
445
+ # Un-assign @buffer not to show buffering metrics (e.g., in_monitor_agent)
446
+ @buffer = nil
447
+ end
448
+ end
449
+
450
+ if @buffering
451
+ m = method(:emit_buffered)
452
+ singleton_class.module_eval do
453
+ define_method(:emit_events, m)
454
+ end
455
+
456
+ @custom_format = implement?(:custom_format)
457
+ @enable_msgpack_streamer = @custom_format ? formatted_to_msgpack_binary : true
458
+ @delayed_commit = if implement?(:buffered) && implement?(:delayed_commit)
459
+ prefer_delayed_commit
460
+ else
461
+ implement?(:delayed_commit)
462
+ end
463
+ @delayed_commit_timeout = @buffer_config.delayed_commit_timeout
464
+ else # !@buffering
465
+ m = method(:emit_sync)
466
+ singleton_class.module_eval do
467
+ define_method(:emit_events, m)
468
+ end
469
+ end
470
+
471
+ if @buffering && !@as_secondary
472
+ @retry = nil
473
+ @retry_mutex = Mutex.new
474
+
475
+ @buffer.start
476
+
477
+ @output_enqueue_thread = nil
478
+ @output_enqueue_thread_running = true
479
+
480
+ @output_flush_threads = []
481
+ @output_flush_threads_mutex = Mutex.new
482
+ @output_flush_threads_running = true
483
+
484
+ # mainly for test: detect enqueue works as code below:
485
+ # @output.interrupt_flushes
486
+ # # emits
487
+ # @output.enqueue_thread_wait
488
+ @output_flush_interrupted = false
489
+ @output_enqueue_thread_mutex = Mutex.new
490
+ @output_enqueue_thread_waiting = false
491
+
492
+ @dequeued_chunks = []
493
+ @dequeued_chunks_mutex = Mutex.new
494
+
495
+ @buffer_config.flush_thread_count.times do |i|
496
+ thread_title = "flush_thread_#{i}".to_sym
497
+ thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
498
+ thread = thread_create(thread_title) do
499
+ flush_thread_run(thread_state)
500
+ end
501
+ thread_state.thread = thread
502
+ @output_flush_threads_mutex.synchronize do
503
+ @output_flush_threads << thread_state
504
+ end
505
+ end
506
+ @output_flush_thread_current_position = 0
507
+
508
+ if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
509
+ @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
510
+ end
511
+ end
512
+ @secondary.start if @secondary
513
+ end
514
+
515
+ def after_start
516
+ super
517
+ @secondary.after_start if @secondary
518
+ end
519
+
520
+ def stop
521
+ @secondary.stop if @secondary
522
+ @buffer.stop if @buffering && @buffer
523
+
524
+ super
525
+ end
526
+
527
+ def before_shutdown
528
+ @secondary.before_shutdown if @secondary
529
+
530
+ if @buffering && @buffer
531
+ if @flush_at_shutdown
532
+ force_flush
533
+ end
534
+ @buffer.before_shutdown
535
+ # Need to ensure to stop enqueueing ... after #shutdown, we cannot write any data
536
+ @output_enqueue_thread_running = false
537
+ if @output_enqueue_thread && @output_enqueue_thread.alive?
538
+ @output_enqueue_thread.wakeup
539
+ @output_enqueue_thread.join
540
+ end
541
+ end
542
+
543
+ super
544
+ end
545
+
546
+ def shutdown
547
+ @secondary.shutdown if @secondary
548
+ @buffer.shutdown if @buffering && @buffer
549
+
550
+ super
551
+ end
552
+
553
+ def after_shutdown
554
+ try_rollback_all if @buffering && !@as_secondary # rollback regardless with @delayed_commit, because secondary may do it
555
+ @secondary.after_shutdown if @secondary
556
+
557
+ if @buffering && @buffer
558
+ @buffer.after_shutdown
559
+
560
+ @output_flush_threads_running = false
561
+ if @output_flush_threads && !@output_flush_threads.empty?
562
+ @output_flush_threads.each do |state|
563
+ # to wakeup thread and make it to stop by itself
564
+ state.mutex.synchronize {
565
+ if state.thread && state.thread.status
566
+ state.next_clock = 0
567
+ state.cond_var.signal
568
+ end
569
+ }
570
+ Thread.pass
571
+ state.thread.join
572
+ end
573
+ end
574
+ end
575
+
576
+ super
577
+ end
578
+
579
+ def close
580
+ @buffer.close if @buffering && @buffer
581
+ @secondary.close if @secondary
582
+
583
+ super
584
+ end
585
+
586
+ def terminate
587
+ @buffer.terminate if @buffering && @buffer
588
+ @secondary.terminate if @secondary
589
+
590
+ super
591
+ end
592
+
593
+ def support_in_v12_style?(feature)
594
+ # for plugins written in v0.12 styles
595
+ case feature
596
+ when :synchronous then false
597
+ when :buffered then false
598
+ when :delayed_commit then false
599
+ when :custom_format then false
600
+ else
601
+ raise ArgumentError, "unknown feature: #{feature}"
602
+ end
603
+ end
604
+
605
+ def implement?(feature)
606
+ methods_of_plugin = self.class.instance_methods(false)
607
+ case feature
608
+ when :synchronous then methods_of_plugin.include?(:process) || support_in_v12_style?(:synchronous)
609
+ when :buffered then methods_of_plugin.include?(:write) || support_in_v12_style?(:buffered)
610
+ when :delayed_commit then methods_of_plugin.include?(:try_write)
611
+ when :custom_format then methods_of_plugin.include?(:format) || support_in_v12_style?(:custom_format)
612
+ else
613
+ raise ArgumentError, "Unknown feature for output plugin: #{feature}"
614
+ end
615
+ end
616
+
617
+ def placeholder_validate!(name, str)
618
+ placeholder_validators(name, str).each do |v|
619
+ v.validate!
620
+ end
621
+ end
622
+
623
+ def placeholder_validators(name, str, time_key = (@chunk_key_time && @buffer_config.timekey), tag_key = @chunk_key_tag, chunk_keys = @chunk_keys)
624
+ validators = []
625
+
626
+ sec, title, example = get_placeholders_time(str)
627
+ if sec || time_key
628
+ validators << PlaceholderValidator.new(name, str, :time, {sec: sec, title: title, example: example, timekey: time_key})
629
+ end
630
+
631
+ parts = get_placeholders_tag(str)
632
+ if tag_key || !parts.empty?
633
+ validators << PlaceholderValidator.new(name, str, :tag, {parts: parts, tagkey: tag_key})
634
+ end
635
+
636
+ keys = get_placeholders_keys(str)
637
+ if chunk_keys && !chunk_keys.empty? || !keys.empty?
638
+ validators << PlaceholderValidator.new(name, str, :keys, {keys: keys, chunkkeys: chunk_keys})
639
+ end
640
+
641
+ validators
642
+ end
643
+
644
+ class PlaceholderValidator
645
+ attr_reader :name, :string, :type, :argument
646
+
647
+ def initialize(name, str, type, arg)
648
+ @name = name
649
+ @string = str
650
+ @type = type
651
+ raise ArgumentError, "invalid type:#{type}" if @type != :time && @type != :tag && @type != :keys
652
+ @argument = arg
653
+ end
654
+
655
+ def time?
656
+ @type == :time
657
+ end
658
+
659
+ def tag?
660
+ @type == :tag
661
+ end
662
+
663
+ def keys?
664
+ @type == :keys
665
+ end
666
+
667
+ def validate!
668
+ case @type
669
+ when :time then validate_time!
670
+ when :tag then validate_tag!
671
+ when :keys then validate_keys!
672
+ end
673
+ end
674
+
675
+ def validate_time!
676
+ sec = @argument[:sec]
677
+ title = @argument[:title]
678
+ example = @argument[:example]
679
+ timekey = @argument[:timekey]
680
+ if !sec && timekey
681
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
682
+ end
683
+ if sec && !timekey
684
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has timestamp placeholders, but chunk key 'time' is not configured"
685
+ end
686
+ if sec && timekey && timekey < sec
687
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
688
+ end
689
+ end
690
+
691
+ def validate_tag!
692
+ parts = @argument[:parts]
693
+ tagkey = @argument[:tagkey]
694
+ if tagkey && parts.empty?
695
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have tag placeholder"
696
+ end
697
+ if !tagkey && !parts.empty?
698
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has tag placeholders, but chunk key 'tag' is not configured"
699
+ end
700
+ end
701
+
702
+ def validate_keys!
703
+ keys = @argument[:keys]
704
+ chunk_keys = @argument[:chunkkeys]
705
+ if (chunk_keys - keys).size > 0
706
+ not_specified = (chunk_keys - keys).sort
707
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
708
+ end
709
+ if (keys - chunk_keys).size > 0
710
+ not_satisfied = (keys - chunk_keys).sort
711
+ raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
712
+ end
713
+ end
714
+ end
715
+
716
+ TIME_KEY_PLACEHOLDER_THRESHOLDS = [
717
+ [1, :second, '%S'],
718
+ [60, :minute, '%M'],
719
+ [3600, :hour, '%H'],
720
+ [86400, :day, '%d'],
721
+ ]
722
+ TIMESTAMP_CHECK_BASE_TIME = Time.parse("2016-01-01 00:00:00 UTC")
723
+ # it's not validated to use timekey larger than 1 day
724
+ def get_placeholders_time(str)
725
+ base_str = TIMESTAMP_CHECK_BASE_TIME.strftime(str)
726
+ TIME_KEY_PLACEHOLDER_THRESHOLDS.each do |triple|
727
+ sec = triple.first
728
+ return triple if (TIMESTAMP_CHECK_BASE_TIME + sec).strftime(str) != base_str
729
+ end
730
+ nil
731
+ end
732
+
733
+ # -1 means whole tag
734
+ def get_placeholders_tag(str)
735
+ # [["tag"],["tag[0]"]]
736
+ parts = []
737
+ str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
738
+ if ph == "tag"
739
+ parts << -1
740
+ elsif ph =~ /^tag\[(-?\d+)\]$/
741
+ parts << $1.to_i
742
+ end
743
+ end
744
+ parts.sort
745
+ end
746
+
747
+ def get_placeholders_keys(str)
748
+ str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map(&:first).reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
749
+ end
750
+
751
+ # TODO: optimize this code
752
+ def extract_placeholders(str, chunk)
753
+ metadata = if chunk.is_a?(Fluent::Plugin::Buffer::Chunk)
754
+ chunk_passed = true
755
+ chunk.metadata
756
+ else
757
+ chunk_passed = false
758
+ # For existing plugins. Old plugin passes Chunk.metadata instead of Chunk
759
+ chunk
760
+ end
761
+ if metadata.empty?
762
+ str.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
763
+ if chunk_passed
764
+ dump_unique_id_hex(chunk.unique_id)
765
+ else
766
+ log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
767
+ end
768
+ }
769
+ else
770
+ rvalue = str.dup
771
+ # strftime formatting
772
+ if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
773
+ @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
774
+ rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
775
+ end
776
+ # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
777
+ if @chunk_key_tag
778
+ if str.include?('${tag}')
779
+ rvalue = rvalue.gsub('${tag}', metadata.tag)
780
+ end
781
+ if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
782
+ hash = {}
783
+ tag_parts = metadata.tag.split('.')
784
+ tag_parts.each_with_index do |part, i|
785
+ hash["${tag[#{i}]}"] = part
786
+ hash["${tag[#{i-tag_parts.size}]}"] = part
787
+ end
788
+ rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
789
+ end
790
+ if rvalue =~ CHUNK_TAG_PLACEHOLDER_PATTERN
791
+ log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
792
+ end
793
+ end
794
+
795
+ # First we replace ${chunk_id} with chunk.unique_id (hexlified).
796
+ rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
797
+ if chunk_passed
798
+ dump_unique_id_hex(chunk.unique_id)
799
+ else
800
+ log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
801
+ end
802
+ }
803
+
804
+ # Then, replace other ${chunk_key}s.
805
+ if !@chunk_keys.empty? && metadata.variables
806
+ hash = {'${tag}' => '${tag}'} # not to erase this wrongly
807
+ @chunk_keys.each do |key|
808
+ hash["${#{key}}"] = metadata.variables[key.to_sym]
809
+ end
810
+
811
+ rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
812
+ hash.fetch(matched) do
813
+ log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
814
+ ''
815
+ end
816
+ end
817
+ end
818
+
819
+ if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
820
+ log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
821
+ end
822
+
823
+ rvalue
824
+ end
825
+ end
826
+
827
+ def emit_events(tag, es)
828
+ # actually this method will be overwritten by #configure
829
+ if @buffering
830
+ emit_buffered(tag, es)
831
+ else
832
+ emit_sync(tag, es)
833
+ end
834
+ end
835
+
836
+ def emit_sync(tag, es)
837
+ @emit_count_metrics.inc
838
+ begin
839
+ process(tag, es)
840
+ @emit_records_metrics.add(es.size)
841
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
842
+ rescue
843
+ @num_errors_metrics.inc
844
+ raise
845
+ end
846
+ end
847
+
848
+ def emit_buffered(tag, es)
849
+ @emit_count_metrics.inc
850
+ begin
851
+ execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
852
+ if !@retry && @buffer.queued?(nil, optimistic: true)
853
+ submit_flush_once
854
+ end
855
+ rescue
856
+ # TODO: separate number of errors into emit errors and write/flush errors
857
+ @num_errors_metrics.inc
858
+ raise
859
+ end
860
+ end
861
+
862
+ # TODO: optimize this code
863
+ def metadata(tag, time, record)
864
+ # this arguments are ordered in output plugin's rule
865
+ # Metadata 's argument order is different from this one (timekey, tag, variables)
866
+
867
+ raise ArgumentError, "tag must be a String: #{tag.class}" unless tag.nil? || tag.is_a?(String)
868
+ raise ArgumentError, "time must be a Fluent::EventTime (or Integer): #{time.class}" unless time.nil? || time.is_a?(Fluent::EventTime) || time.is_a?(Integer)
869
+ raise ArgumentError, "record must be a Hash: #{record.class}" unless record.nil? || record.is_a?(Hash)
870
+
871
+ if @chunk_keys.nil? && @chunk_key_time.nil? && @chunk_key_tag.nil?
872
+ # for tests
873
+ return Struct.new(:timekey, :tag, :variables).new
874
+ end
875
+
876
+ # timekey is int from epoch, and `timekey - timekey % 60` is assumed to mach with 0s of each minutes.
877
+ # it's wrong if timezone is configured as one which supports leap second, but it's very rare and
878
+ # we can ignore it (especially in production systems).
879
+ if @chunk_keys.empty?
880
+ if !@chunk_key_time && !@chunk_key_tag
881
+ @buffer.metadata()
882
+ elsif @chunk_key_time && @chunk_key_tag
883
+ timekey = calculate_timekey(time)
884
+ @buffer.metadata(timekey: timekey, tag: tag)
885
+ elsif @chunk_key_time
886
+ timekey = calculate_timekey(time)
887
+ @buffer.metadata(timekey: timekey)
888
+ else
889
+ @buffer.metadata(tag: tag)
890
+ end
891
+ else
892
+ timekey = if @chunk_key_time
893
+ calculate_timekey(time)
894
+ else
895
+ nil
896
+ end
897
+ pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
898
+ @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
899
+ end
900
+ end
901
+
902
+ def calculate_timekey(time)
903
+ time_int = time.to_i
904
+ if @timekey_use_utc
905
+ (time_int - (time_int % @timekey)).to_i
906
+ else
907
+ offset = @calculate_offset ? @calculate_offset.call(time) : @offset
908
+ (time_int - ((time_int + offset)% @timekey)).to_i
909
+ end
910
+ end
911
+
912
+ def chunk_for_test(tag, time, record)
913
+ require 'fluent/plugin/buffer/memory_chunk'
914
+
915
+ m = metadata(tag, time, record)
916
+ Fluent::Plugin::Buffer::MemoryChunk.new(m)
917
+ end
918
+
919
+ def execute_chunking(tag, es, enqueue: false)
920
+ if @simple_chunking
921
+ handle_stream_simple(tag, es, enqueue: enqueue)
922
+ elsif @custom_format
923
+ handle_stream_with_custom_format(tag, es, enqueue: enqueue)
924
+ else
925
+ handle_stream_with_standard_format(tag, es, enqueue: enqueue)
926
+ end
927
+ end
928
+
929
+ def write_guard(&block)
930
+ begin
931
+ block.call
932
+ rescue Fluent::Plugin::Buffer::BufferOverflowError
933
+ log.warn "failed to write data into buffer by buffer overflow", action: @buffer_config.overflow_action
934
+ case @buffer_config.overflow_action
935
+ when :throw_exception
936
+ raise
937
+ when :block
938
+ log.debug "buffer.write is now blocking"
939
+ until @buffer.storable?
940
+ if self.stopped?
941
+ log.error "breaking block behavior to shutdown Fluentd"
942
+ # to break infinite loop to exit Fluentd process
943
+ raise
944
+ end
945
+ log.trace "sleeping until buffer can store more data"
946
+ sleep 1
947
+ end
948
+ log.debug "retrying buffer.write after blocked operation"
949
+ retry
950
+ when :drop_oldest_chunk
951
+ begin
952
+ oldest = @buffer.dequeue_chunk
953
+ if oldest
954
+ log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
955
+ @buffer.purge_chunk(oldest.unique_id)
956
+ else
957
+ log.error "no queued chunks to be dropped for drop_oldest_chunk"
958
+ end
959
+ rescue
960
+ # ignore any errors
961
+ end
962
+ raise unless @buffer.storable?
963
+ retry
964
+ else
965
+ raise "BUG: unknown overflow_action '#{@buffer_config.overflow_action}'"
966
+ end
967
+ end
968
+ end
969
+
970
+ FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
971
+ FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
972
+ FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
973
+ FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
974
+
975
+ def generate_format_proc
976
+ if @buffer && @buffer.compress == :gzip
977
+ @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
978
+ else
979
+ @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
980
+ end
981
+ end
982
+
983
+ # metadata_and_data is a Hash of:
984
+ # (standard format) metadata => event stream
985
+ # (custom format) metadata => array of formatted event
986
+ # For standard format, formatting should be done for whole event stream, but
987
+ # "whole event stream" may be a split of "es" here when it's bigger than chunk_limit_size.
988
+ # `@buffer.write` will do this splitting.
989
+ # For custom format, formatting will be done here. Custom formatting always requires
990
+ # iteration of event stream, and it should be done just once even if total event stream size
991
+ # is bigger than chunk_limit_size because of performance.
992
+ def handle_stream_with_custom_format(tag, es, enqueue: false)
993
+ meta_and_data = {}
994
+ records = 0
995
+ es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
996
+ meta = metadata(tag, time, record)
997
+ meta_and_data[meta] ||= []
998
+ res = format(tag, time, record)
999
+ if res
1000
+ meta_and_data[meta] << res
1001
+ records += 1
1002
+ end
1003
+ end
1004
+ write_guard do
1005
+ @buffer.write(meta_and_data, enqueue: enqueue)
1006
+ end
1007
+ @emit_records_metrics.add(es.size)
1008
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1009
+ true
1010
+ end
1011
+
1012
+ def handle_stream_with_standard_format(tag, es, enqueue: false)
1013
+ format_proc = generate_format_proc
1014
+ meta_and_data = {}
1015
+ records = 0
1016
+ es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
1017
+ meta = metadata(tag, time, record)
1018
+ meta_and_data[meta] ||= MultiEventStream.new
1019
+ meta_and_data[meta].add(time, record)
1020
+ records += 1
1021
+ end
1022
+ write_guard do
1023
+ @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
1024
+ end
1025
+ @emit_records_metrics.add(es.size)
1026
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1027
+ true
1028
+ end
1029
+
1030
+ def handle_stream_simple(tag, es, enqueue: false)
1031
+ format_proc = nil
1032
+ meta = metadata((@chunk_key_tag ? tag : nil), nil, nil)
1033
+ records = es.size
1034
+ if @custom_format
1035
+ records = 0
1036
+ data = []
1037
+ es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
1038
+ res = format(tag, time, record)
1039
+ if res
1040
+ data << res
1041
+ records += 1
1042
+ end
1043
+ end
1044
+ else
1045
+ format_proc = generate_format_proc
1046
+ data = es
1047
+ end
1048
+ write_guard do
1049
+ @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
1050
+ end
1051
+ @emit_records_metrics.add(es.size)
1052
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1053
+ true
1054
+ end
1055
+
1056
+ def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
1057
+ log.on_trace { log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }
1058
+
1059
+ if delayed
1060
+ @dequeued_chunks_mutex.synchronize do
1061
+ @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
1062
+ end
1063
+ end
1064
+ @buffer.purge_chunk(chunk_id)
1065
+
1066
+ @retry_mutex.synchronize do
1067
+ if @retry # success to flush chunks in retries
1068
+ if secondary
1069
+ log.warn "retry succeeded by secondary.", chunk_id: dump_unique_id_hex(chunk_id)
1070
+ else
1071
+ log.warn "retry succeeded.", chunk_id: dump_unique_id_hex(chunk_id)
1072
+ end
1073
+ @retry = nil
1074
+ end
1075
+ end
1076
+ end
1077
+
1078
+ # update_retry parameter is for preventing busy loop by async write
1079
+ # We will remove this parameter by re-design retry_state management between threads.
1080
+ def rollback_write(chunk_id, update_retry: true)
1081
+ # This API is to rollback chunks explicitly from plugins.
1082
+ # 3rd party plugins can depend it on automatic rollback of #try_rollback_write
1083
+ @dequeued_chunks_mutex.synchronize do
1084
+ @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
1085
+ end
1086
+ # returns true if chunk was rollbacked as expected
1087
+ # false if chunk was already flushed and couldn't be rollbacked unexpectedly
1088
+ # in many cases, false can be just ignored
1089
+ if @buffer.takeback_chunk(chunk_id)
1090
+ @rollback_count_metrics.inc
1091
+ if update_retry
1092
+ primary = @as_secondary ? @primary_instance : self
1093
+ primary.update_retry_state(chunk_id, @as_secondary)
1094
+ end
1095
+ true
1096
+ else
1097
+ false
1098
+ end
1099
+ end
1100
+
1101
+ def try_rollback_write
1102
+ @dequeued_chunks_mutex.synchronize do
1103
+ while @dequeued_chunks.first && @dequeued_chunks.first.expired?
1104
+ info = @dequeued_chunks.shift
1105
+ if @buffer.takeback_chunk(info.chunk_id)
1106
+ @rollback_count_metrics.inc
1107
+ log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
1108
+ primary = @as_secondary ? @primary_instance : self
1109
+ primary.update_retry_state(info.chunk_id, @as_secondary)
1110
+ end
1111
+ end
1112
+ end
1113
+ end
1114
+
1115
+ def try_rollback_all
1116
+ return unless @dequeued_chunks
1117
+ @dequeued_chunks_mutex.synchronize do
1118
+ until @dequeued_chunks.empty?
1119
+ info = @dequeued_chunks.shift
1120
+ if @buffer.takeback_chunk(info.chunk_id)
1121
+ @rollback_count_metrics.inc
1122
+ log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
1123
+ primary = @as_secondary ? @primary_instance : self
1124
+ primary.update_retry_state(info.chunk_id, @as_secondary)
1125
+ end
1126
+ end
1127
+ end
1128
+ end
1129
+
1130
+ def next_flush_time
1131
+ if @buffer.queued?
1132
+ @retry_mutex.synchronize do
1133
+ @retry ? @retry.next_time : Time.now + @buffer_config.flush_thread_burst_interval
1134
+ end
1135
+ else
1136
+ Time.now + @buffer_config.flush_thread_interval
1137
+ end
1138
+ end
1139
+
1140
+ UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError]
1141
+
1142
+ def try_flush
1143
+ chunk = @buffer.dequeue_chunk
1144
+ return unless chunk
1145
+
1146
+ log.on_trace { log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id) }
1147
+
1148
+ output = self
1149
+ using_secondary = false
1150
+ if @retry_mutex.synchronize{ @retry && @retry.secondary? }
1151
+ output = @secondary
1152
+ using_secondary = true
1153
+ end
1154
+
1155
+ if @enable_msgpack_streamer
1156
+ chunk.extend ChunkMessagePackEventStreamer
1157
+ end
1158
+
1159
+ begin
1160
+ chunk_write_start = Fluent::Clock.now
1161
+
1162
+ if output.delayed_commit
1163
+ log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
1164
+ @write_count_metrics.inc
1165
+ @dequeued_chunks_mutex.synchronize do
1166
+ # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
1167
+ @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
1168
+ end
1169
+
1170
+ output.try_write(chunk)
1171
+ check_slow_flush(chunk_write_start)
1172
+ else # output plugin without delayed purge
1173
+ chunk_id = chunk.unique_id
1174
+ dump_chunk_id = dump_unique_id_hex(chunk_id)
1175
+ log.trace "adding write count", instance: self.object_id
1176
+ @write_count_metrics.inc
1177
+ log.trace "executing sync write", chunk: dump_chunk_id
1178
+
1179
+ output.write(chunk)
1180
+ check_slow_flush(chunk_write_start)
1181
+
1182
+ log.trace "write operation done, committing", chunk: dump_chunk_id
1183
+ commit_write(chunk_id, delayed: false, secondary: using_secondary)
1184
+ log.trace "done to commit a chunk", chunk: dump_chunk_id
1185
+ end
1186
+ rescue *UNRECOVERABLE_ERRORS => e
1187
+ if @secondary
1188
+ if using_secondary
1189
+ log.warn "got unrecoverable error in secondary.", error: e
1190
+ log.warn_backtrace
1191
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1192
+ else
1193
+ if (self.class == @secondary.class)
1194
+ log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
1195
+ log.warn_backtrace
1196
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1197
+ else
1198
+ # Call secondary output directly without retry update.
1199
+ # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
1200
+ if @secondary.delayed_commit
1201
+ log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
1202
+ log.warn_backtrace
1203
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1204
+ else
1205
+ log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
1206
+ log.warn_backtrace
1207
+ begin
1208
+ @secondary.write(chunk)
1209
+ commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
1210
+ rescue => e
1211
+ log.warn "got an error in secondary for unrecoverable error", error: e
1212
+ log.warn_backtrace
1213
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1214
+ end
1215
+ end
1216
+ end
1217
+ end
1218
+ else
1219
+ log.warn "got unrecoverable error in primary and no secondary", error: e
1220
+ log.warn_backtrace
1221
+ backup_chunk(chunk, using_secondary, output.delayed_commit)
1222
+ end
1223
+ rescue => e
1224
+ log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
1225
+ if output.delayed_commit
1226
+ @dequeued_chunks_mutex.synchronize do
1227
+ @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
1228
+ end
1229
+ end
1230
+
1231
+ if @buffer.takeback_chunk(chunk.unique_id)
1232
+ @rollback_count_metrics.inc
1233
+ end
1234
+
1235
+ update_retry_state(chunk.unique_id, using_secondary, e)
1236
+
1237
+ raise if @under_plugin_development && !@retry_for_error_chunk
1238
+ end
1239
+ end
1240
+
1241
+ def backup_chunk(chunk, using_secondary, delayed_commit)
1242
+ if @buffer_config.disable_chunk_backup
1243
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
1244
+ else
1245
+ unique_id = dump_unique_id_hex(chunk.unique_id)
1246
+ safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
1247
+ backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
1248
+ backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
1249
+ backup_dir = File.dirname(backup_file)
1250
+
1251
+ log.warn "bad chunk is moved to #{backup_file}"
1252
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
1253
+ File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
1254
+ chunk.write_to(f)
1255
+ }
1256
+ end
1257
+ commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
1258
+ end
1259
+
1260
+ def check_slow_flush(start)
1261
+ elapsed_time = Fluent::Clock.now - start
1262
+ elapsed_millsec = (elapsed_time * 1000).to_i
1263
+ @flush_time_count_metrics.add(elapsed_millsec)
1264
+ if elapsed_time > @slow_flush_log_threshold
1265
+ @slow_flush_count_metrics.inc
1266
+ log.warn "buffer flush took longer time than slow_flush_log_threshold:",
1267
+ elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
1268
+ end
1269
+ end
1270
+
1271
+ def update_retry_state(chunk_id, using_secondary, error = nil)
1272
+ @retry_mutex.synchronize do
1273
+ @num_errors_metrics.inc
1274
+ chunk_id_hex = dump_unique_id_hex(chunk_id)
1275
+
1276
+ unless @retry
1277
+ @retry = retry_state(@buffer_config.retry_randomize)
1278
+ if error
1279
+ log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1280
+ log.warn_backtrace error.backtrace
1281
+ end
1282
+ return
1283
+ end
1284
+
1285
+ # @retry exists
1286
+
1287
+ if @retry.limit?
1288
+ if error
1289
+ records = @buffer.queued_records
1290
+ msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
1291
+ log.error msg, retry_times: @retry.steps, records: records, error: error
1292
+ log.error_backtrace error.backtrace
1293
+ end
1294
+ @buffer.clear_queue!
1295
+ log.debug "buffer queue cleared"
1296
+ @retry = nil
1297
+ else
1298
+ # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
1299
+ # @retry.step is called almost as many times as the number of flush threads in a short time.
1300
+ if Time.now >= @retry.next_time
1301
+ @retry.step
1302
+ else
1303
+ @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
1304
+ end
1305
+ if error
1306
+ if using_secondary
1307
+ msg = "failed to flush the buffer with secondary output."
1308
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1309
+ log.warn_backtrace error.backtrace
1310
+ else
1311
+ msg = "failed to flush the buffer."
1312
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1313
+ log.warn_backtrace error.backtrace
1314
+ end
1315
+ end
1316
+ end
1317
+ end
1318
+ end
1319
+
1320
+ def retry_state(randomize)
1321
+ if @secondary
1322
+ retry_state_create(
1323
+ :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,
1324
+ forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,
1325
+ max_interval: @buffer_config.retry_max_interval,
1326
+ secondary: true, secondary_threshold: @buffer_config.retry_secondary_threshold,
1327
+ randomize: randomize
1328
+ )
1329
+ else
1330
+ retry_state_create(
1331
+ :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,
1332
+ forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,
1333
+ max_interval: @buffer_config.retry_max_interval,
1334
+ randomize: randomize
1335
+ )
1336
+ end
1337
+ end
1338
+
1339
+ def submit_flush_once
1340
+ # Without locks: it is rough but enough to select "next" writer selection
1341
+ @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
1342
+ state = @output_flush_threads[@output_flush_thread_current_position]
1343
+ state.mutex.synchronize {
1344
+ if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
1345
+ state.next_clock = 0
1346
+ state.cond_var.signal
1347
+ else
1348
+ log.warn "thread is already dead"
1349
+ end
1350
+ }
1351
+ Thread.pass
1352
+ end
1353
+
1354
+ def force_flush
1355
+ if @buffering
1356
+ @buffer.enqueue_all(true)
1357
+ submit_flush_all
1358
+ end
1359
+ end
1360
+
1361
+ def submit_flush_all
1362
+ while !@retry && @buffer.queued?
1363
+ submit_flush_once
1364
+ sleep @buffer_config.flush_thread_burst_interval
1365
+ end
1366
+ end
1367
+
1368
+ # only for tests of output plugin
1369
+ def interrupt_flushes
1370
+ @output_flush_interrupted = true
1371
+ end
1372
+
1373
+ # only for tests of output plugin
1374
+ def enqueue_thread_wait
1375
+ @output_enqueue_thread_mutex.synchronize do
1376
+ @output_flush_interrupted = false
1377
+ @output_enqueue_thread_waiting = true
1378
+ end
1379
+ require 'timeout'
1380
+ Timeout.timeout(10) do
1381
+ Thread.pass while @output_enqueue_thread_waiting
1382
+ end
1383
+ end
1384
+
1385
+ # only for tests of output plugin
1386
+ def flush_thread_wakeup
1387
+ @output_flush_threads.each do |state|
1388
+ state.mutex.synchronize {
1389
+ if state.thread && state.thread.status
1390
+ state.next_clock = 0
1391
+ state.cond_var.signal
1392
+ end
1393
+ }
1394
+ Thread.pass
1395
+ end
1396
+ end
1397
+
1398
+ def enqueue_thread_run
1399
+ value_for_interval = nil
1400
+ if @flush_mode == :interval
1401
+ value_for_interval = @buffer_config.flush_interval
1402
+ end
1403
+ if @chunk_key_time
1404
+ if !value_for_interval || @buffer_config.timekey < value_for_interval
1405
+ value_for_interval = [@buffer_config.timekey, @buffer_config.timekey_wait].min
1406
+ end
1407
+ end
1408
+ unless value_for_interval
1409
+ raise "BUG: both of flush_interval and timekey are disabled"
1410
+ end
1411
+ interval = value_for_interval / 11.0
1412
+ if interval < @buffer_config.flush_thread_interval
1413
+ interval = @buffer_config.flush_thread_interval
1414
+ end
1415
+
1416
+ while !self.after_started? && !self.stopped?
1417
+ sleep 0.5
1418
+ end
1419
+ log.debug "enqueue_thread actually running"
1420
+
1421
+ begin
1422
+ while @output_enqueue_thread_running
1423
+ now_int = Time.now.to_i
1424
+ if @output_flush_interrupted
1425
+ sleep interval
1426
+ next
1427
+ end
1428
+
1429
+ @output_enqueue_thread_mutex.lock
1430
+ begin
1431
+ if @flush_mode == :interval
1432
+ flush_interval = @buffer_config.flush_interval.to_i
1433
+ # This block should be done by integer values.
1434
+ # If both of flush_interval & flush_thread_interval are 1s, expected actual flush timing is 1.5s.
1435
+ # If we use integered values for this comparison, expected actual flush timing is 1.0s.
1436
+ @buffer.enqueue_all{ |metadata, chunk| chunk.raw_create_at + flush_interval <= now_int }
1437
+ end
1438
+
1439
+ if @chunk_key_time
1440
+ timekey_unit = @buffer_config.timekey
1441
+ timekey_wait = @buffer_config.timekey_wait
1442
+ current_timekey = now_int - now_int % timekey_unit
1443
+ @buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_timekey && metadata.timekey + timekey_unit + timekey_wait <= now_int }
1444
+ end
1445
+ rescue => e
1446
+ raise if @under_plugin_development
1447
+ log.error "unexpected error while checking flushed chunks. ignored.", error: e
1448
+ log.error_backtrace
1449
+ ensure
1450
+ @output_enqueue_thread_waiting = false
1451
+ @output_enqueue_thread_mutex.unlock
1452
+ end
1453
+ sleep interval
1454
+ end
1455
+ rescue => e
1456
+ # normal errors are rescued by inner begin-rescue clause.
1457
+ log.error "error on enqueue thread", error: e
1458
+ log.error_backtrace
1459
+ raise
1460
+ end
1461
+ end
1462
+
1463
+ def flush_thread_run(state)
1464
+ flush_thread_interval = @buffer_config.flush_thread_interval
1465
+
1466
+ state.next_clock = Fluent::Clock.now + flush_thread_interval
1467
+
1468
+ while !self.after_started? && !self.stopped?
1469
+ sleep 0.5
1470
+ end
1471
+ log.debug "flush_thread actually running"
1472
+
1473
+ state.mutex.lock
1474
+ begin
1475
+ # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
1476
+ while @output_flush_threads_running
1477
+ current_clock = Fluent::Clock.now
1478
+ next_retry_time = nil
1479
+
1480
+ @retry_mutex.synchronize do
1481
+ next_retry_time = @retry ? @retry.next_time : nil
1482
+ end
1483
+
1484
+ if state.next_clock > current_clock
1485
+ interval = state.next_clock - current_clock
1486
+ elsif next_retry_time && next_retry_time > Time.now
1487
+ interval = next_retry_time.to_f - Time.now.to_f
1488
+ else
1489
+ state.mutex.unlock
1490
+ begin
1491
+ try_flush
1492
+ # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
1493
+ interval = next_flush_time.to_f - Time.now.to_f
1494
+ # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
1495
+ # because @retry still exists (#commit_write is not called yet in #try_flush)
1496
+ # @retry should be cleared if delayed commit is enabled? Or any other solution?
1497
+ state.next_clock = Fluent::Clock.now + interval
1498
+ ensure
1499
+ state.mutex.lock
1500
+ end
1501
+ end
1502
+
1503
+ if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
1504
+ unless @output_flush_interrupted
1505
+ state.mutex.unlock
1506
+ begin
1507
+ try_rollback_write
1508
+ ensure
1509
+ state.mutex.lock
1510
+ end
1511
+ end
1512
+ end
1513
+
1514
+ state.cond_var.wait(state.mutex, interval) if interval > 0
1515
+ end
1516
+ rescue => e
1517
+ # normal errors are rescued by output plugins in #try_flush
1518
+ # so this rescue section is for critical & unrecoverable errors
1519
+ log.error "error on output thread", error: e
1520
+ log.error_backtrace
1521
+ raise
1522
+ ensure
1523
+ state.mutex.unlock
1524
+ end
1525
+ end
1526
+
1527
+ BUFFER_STATS_KEYS = {}
1528
+ Fluent::Plugin::Buffer::STATS_KEYS.each { |key|
1529
+ BUFFER_STATS_KEYS[key] = "buffer_#{key}"
1530
+ }
1531
+
1532
+ def statistics
1533
+ stats = {
1534
+ 'emit_records' => @emit_records_metrics.get,
1535
+ 'emit_size' => @emit_size_metrics.get,
1536
+ # Respect original name
1537
+ # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
1538
+ 'retry_count' => @num_errors_metrics.get,
1539
+ 'emit_count' => @emit_count_metrics.get,
1540
+ 'write_count' => @write_count_metrics.get,
1541
+ 'rollback_count' => @rollback_count_metrics.get,
1542
+ 'slow_flush_count' => @slow_flush_count_metrics.get,
1543
+ 'flush_time_count' => @flush_time_count_metrics.get,
1544
+ }
1545
+
1546
+ if @buffer && @buffer.respond_to?(:statistics)
1547
+ (@buffer.statistics['buffer'] || {}).each do |k, v|
1548
+ stats[BUFFER_STATS_KEYS[k]] = v
1549
+ end
1550
+ end
1551
+
1552
+ { 'output' => stats }
1553
+ end
1554
+ end
1555
+ end
1556
+ end