fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,1065 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/event'
5
+
6
+ require 'json'
7
+ require 'time'
8
+ require 'timeout'
9
+
10
+ module FluentPluginOutputTest
11
+ class DummyBareOutput < Fluent::Plugin::Output
12
+ def register(name, &block)
13
+ instance_variable_set("@#{name}", block)
14
+ end
15
+ end
16
+ class DummySyncOutput < DummyBareOutput
17
+ def initialize
18
+ super
19
+ @process = nil
20
+ end
21
+ def process(tag, es)
22
+ @process ? @process.call(tag, es) : nil
23
+ end
24
+ end
25
+ class DummyAsyncOutput < DummyBareOutput
26
+ def initialize
27
+ super
28
+ @format = nil
29
+ @write = nil
30
+ end
31
+ def format(tag, time, record)
32
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
33
+ end
34
+ def write(chunk)
35
+ @write ? @write.call(chunk) : nil
36
+ end
37
+ end
38
+ class DummyAsyncStandardOutput < DummyBareOutput
39
+ def initialize
40
+ super
41
+ @write = nil
42
+ end
43
+ def write(chunk)
44
+ @write ? @write.call(chunk) : nil
45
+ end
46
+ end
47
+ class DummyDelayedOutput < DummyBareOutput
48
+ def initialize
49
+ super
50
+ @format = nil
51
+ @try_write = nil
52
+ end
53
+ def format(tag, time, record)
54
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
55
+ end
56
+ def try_write(chunk)
57
+ @try_write ? @try_write.call(chunk) : nil
58
+ end
59
+ end
60
+ class DummyDelayedStandardOutput < DummyBareOutput
61
+ def initialize
62
+ super
63
+ @try_write = nil
64
+ end
65
+ def try_write(chunk)
66
+ @try_write ? @try_write.call(chunk) : nil
67
+ end
68
+ end
69
+ class DummyFullFeatureOutput < DummyBareOutput
70
+ def initialize
71
+ super
72
+ @prefer_buffered_processing = nil
73
+ @prefer_delayed_commit = nil
74
+ @process = nil
75
+ @format = nil
76
+ @write = nil
77
+ @try_write = nil
78
+ end
79
+ def prefer_buffered_processing
80
+ @prefer_buffered_processing ? @prefer_buffered_processing.call : false
81
+ end
82
+ def prefer_delayed_commit
83
+ @prefer_delayed_commit ? @prefer_delayed_commit.call : false
84
+ end
85
+ def process(tag, es)
86
+ @process ? @process.call(tag, es) : nil
87
+ end
88
+ def format(tag, time, record)
89
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
90
+ end
91
+ def write(chunk)
92
+ @write ? @write.call(chunk) : nil
93
+ end
94
+ def try_write(chunk)
95
+ @try_write ? @try_write.call(chunk) : nil
96
+ end
97
+ end
98
+ end
99
+
100
+ class OutputTest < Test::Unit::TestCase
101
+ class << self
102
+ def startup
103
+ $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))
104
+ require 'fluent/plugin/out_test'
105
+ end
106
+
107
+ def shutdown
108
+ $LOAD_PATH.shift
109
+ end
110
+ end
111
+
112
+ def create_output(type=:full)
113
+ case type
114
+ when :bare then FluentPluginOutputTest::DummyBareOutput.new
115
+ when :sync then FluentPluginOutputTest::DummySyncOutput.new
116
+ when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new
117
+ when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new
118
+ when :delayed then FluentPluginOutputTest::DummyDelayedOutput.new
119
+ when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new
120
+ when :full then FluentPluginOutputTest::DummyFullFeatureOutput.new
121
+ else
122
+ raise ArgumentError, "unknown type: #{type}"
123
+ end
124
+ end
125
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
126
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
127
+ end
128
+ def create_chunk(timekey: nil, tag: nil, variables: nil)
129
+ m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
130
+ Fluent::Plugin::Buffer::MemoryChunk.new(m)
131
+ end
132
+ def waiting(seconds)
133
+ begin
134
+ Timeout.timeout(seconds) do
135
+ yield
136
+ end
137
+ rescue Timeout::Error
138
+ STDERR.print(*@i.log.out.logs)
139
+ raise
140
+ end
141
+ end
142
+
143
+ sub_test_case 'basic output feature' do
144
+ setup do
145
+ @i = create_output(:full)
146
+ end
147
+
148
+ test 'are not available with multi workers configuration in default' do
149
+ assert_false @i.multi_workers_ready?
150
+ end
151
+
152
+ test '#implement? can return features for plugin instances' do
153
+ i1 = FluentPluginOutputTest::DummyBareOutput.new
154
+ assert !i1.implement?(:synchronous)
155
+ assert !i1.implement?(:buffered)
156
+ assert !i1.implement?(:delayed_commit)
157
+ assert !i1.implement?(:custom_format)
158
+
159
+ i2 = FluentPluginOutputTest::DummySyncOutput.new
160
+ assert i2.implement?(:synchronous)
161
+ assert !i2.implement?(:buffered)
162
+ assert !i2.implement?(:delayed_commit)
163
+ assert !i2.implement?(:custom_format)
164
+
165
+ i3 = FluentPluginOutputTest::DummyAsyncOutput.new
166
+ assert !i3.implement?(:synchronous)
167
+ assert i3.implement?(:buffered)
168
+ assert !i3.implement?(:delayed_commit)
169
+ assert i3.implement?(:custom_format)
170
+
171
+ i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new
172
+ assert !i4.implement?(:synchronous)
173
+ assert i4.implement?(:buffered)
174
+ assert !i4.implement?(:delayed_commit)
175
+ assert !i4.implement?(:custom_format)
176
+
177
+ i5 = FluentPluginOutputTest::DummyDelayedOutput.new
178
+ assert !i5.implement?(:synchronous)
179
+ assert !i5.implement?(:buffered)
180
+ assert i5.implement?(:delayed_commit)
181
+ assert i5.implement?(:custom_format)
182
+
183
+ i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new
184
+ assert !i6.implement?(:synchronous)
185
+ assert !i6.implement?(:buffered)
186
+ assert i6.implement?(:delayed_commit)
187
+ assert !i6.implement?(:custom_format)
188
+
189
+ i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new
190
+ assert i6.implement?(:synchronous)
191
+ assert i6.implement?(:buffered)
192
+ assert i6.implement?(:delayed_commit)
193
+ assert i6.implement?(:custom_format)
194
+ end
195
+
196
+ test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do
197
+ assert !@i.configured?
198
+ @i.configure(config_element())
199
+ assert @i.configured?
200
+ assert !@i.started?
201
+ @i.start
202
+ assert @i.started?
203
+ assert !@i.after_started?
204
+ @i.after_start
205
+ assert @i.after_started?
206
+ assert !@i.stopped?
207
+ @i.stop
208
+ assert @i.stopped?
209
+ assert !@i.before_shutdown?
210
+ @i.before_shutdown
211
+ assert @i.before_shutdown?
212
+ assert !@i.shutdown?
213
+ @i.shutdown
214
+ assert @i.shutdown?
215
+ assert !@i.after_shutdown?
216
+ @i.after_shutdown
217
+ assert @i.after_shutdown?
218
+ assert !@i.closed?
219
+ @i.close
220
+ assert @i.closed?
221
+ assert !@i.terminated?
222
+ @i.terminate
223
+ assert @i.terminated?
224
+ end
225
+
226
+ test 'can use metrics plugins and fallback methods' do
227
+ @i.configure(config_element())
228
+
229
+ %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
230
+ write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
231
+ assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
232
+ end
233
+
234
+ assert_equal 0, @i.num_errors
235
+ assert_equal 0, @i.emit_count
236
+ assert_equal 0, @i.emit_size
237
+ assert_equal 0, @i.emit_records
238
+ assert_equal 0, @i.write_count
239
+ assert_equal 0, @i.rollback_count
240
+ end
241
+
242
+ data(:new_api => :chunk,
243
+ :old_api => :metadata)
244
+ test '#extract_placeholders does nothing if chunk key is not specified' do |api|
245
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
246
+ assert !@i.chunk_key_time
247
+ assert !@i.chunk_key_tag
248
+ assert_equal [], @i.chunk_keys
249
+ tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
250
+ t = event_time('2016-04-11 20:30:00 +0900')
251
+ v = {key1: "value1", key2: "value2"}
252
+ c = if api == :chunk
253
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
254
+ else
255
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
256
+ end
257
+ assert_equal tmpl, @i.extract_placeholders(tmpl, c)
258
+ end
259
+
260
+ data(:new_api => :chunk,
261
+ :old_api => :metadata)
262
+ test '#extract_placeholders can extract time if time key and range are configured' do |api|
263
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
264
+ assert @i.chunk_key_time
265
+ assert !@i.chunk_key_tag
266
+ assert_equal [], @i.chunk_keys
267
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
268
+ t = event_time('2016-04-11 20:30:00 +0900')
269
+ v = {key1: "value1", key2: "value2"}
270
+ c = if api == :chunk
271
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
272
+ else
273
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
274
+ end
275
+ assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, c)
276
+ end
277
+
278
+ data(:new_api => :chunk,
279
+ :old_api => :metadata)
280
+ test '#extract_placeholders can extract tag and parts of tag if tag is configured' do |api|
281
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
282
+ assert !@i.chunk_key_time
283
+ assert @i.chunk_key_tag
284
+ assert_equal [], @i.chunk_keys
285
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
286
+ t = event_time('2016-04-11 20:30:00 +0900')
287
+ v = {key1: "value1", key2: "value2"}
288
+ c = if api == :chunk
289
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
290
+ else
291
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
292
+ end
293
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, c)
294
+ end
295
+
296
+ data(:new_api => :chunk,
297
+ :old_api => :metadata)
298
+ test '#extract_placeholders can extract variables if variables are configured' do |api|
299
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
300
+ assert !@i.chunk_key_time
301
+ assert !@i.chunk_key_tag
302
+ assert_equal ['key1','key2'], @i.chunk_keys
303
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
304
+ t = event_time('2016-04-11 20:30:00 +0900')
305
+ v = {key1: "value1", key2: "value2"}
306
+ c = if api == :chunk
307
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
308
+ else
309
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
310
+ end
311
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, c)
312
+ end
313
+
314
+ data(:new_api => :chunk,
315
+ :old_api => :metadata)
316
+ test '#extract_placeholders can extract nested variables if variables are configured with dot notation' do |api|
317
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key,$.nest.key', {})]))
318
+ assert !@i.chunk_key_time
319
+ assert !@i.chunk_key_tag
320
+ assert_equal ['key','$.nest.key'], @i.chunk_keys
321
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key}/${$.nest.key}/tail"
322
+ t = event_time('2016-04-11 20:30:00 +0900')
323
+ v = {:key => "value1", :"$.nest.key" => "value2"}
324
+ c = if api == :chunk
325
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
326
+ else
327
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
328
+ end
329
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, c)
330
+ end
331
+
332
+ data(:new_api => :chunk,
333
+ :old_api => :metadata)
334
+ test '#extract_placeholders can extract all chunk keys if configured' do |api|
335
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
336
+ assert @i.chunk_key_time
337
+ assert @i.chunk_key_tag
338
+ assert_equal ['key1','key2'], @i.chunk_keys
339
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
340
+ t = event_time('2016-04-11 20:30:00 +0900')
341
+ v = {key1: "value1", key2: "value2"}
342
+ c = if api == :chunk
343
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
344
+ else
345
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
346
+ end
347
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, c)
348
+ end
349
+
350
+ data(:new_api => :chunk,
351
+ :old_api => :metadata)
352
+ test '#extract_placeholders can extract negative index with tag' do |api|
353
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
354
+ assert @i.chunk_key_time
355
+ assert @i.chunk_key_tag
356
+ assert_equal ['key1','key2'], @i.chunk_keys
357
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[-1]}/${tag[-2]}/${key1}/${key2}/tail"
358
+ t = event_time('2016-04-11 20:30:00 +0900')
359
+ v = {key1: "value1", key2: "value2"}
360
+ c = if api == :chunk
361
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
362
+ else
363
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
364
+ end
365
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/output/test/value1/value2/tail", @i.extract_placeholders(tmpl, c)
366
+ end
367
+
368
+ data(:new_api => :chunk,
369
+ :old_api => :metadata)
370
+ test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do |api|
371
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
372
+ assert @i.chunk_key_time
373
+ assert @i.chunk_key_tag
374
+ assert_equal ['key1','key2'], @i.chunk_keys
375
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[-4]}/${key3}/${key4}/tail"
376
+ t = event_time('2016-04-11 20:30:00 +0900')
377
+ v = {key1: "value1", key2: "value2"}
378
+ c = if api == :chunk
379
+ create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
380
+ else
381
+ create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
382
+ end
383
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, c)
384
+ end
385
+
386
+ test '#extract_placeholders logs warn message if metadata is passed for ${chunk_id} placeholder' do
387
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
388
+ tmpl = "/mypath/${chunk_id}/tail"
389
+ t = event_time('2016-04-11 20:30:00 +0900')
390
+ v = {key1: "value1", key2: "value2"}
391
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
392
+ @i.extract_placeholders(tmpl, m)
393
+ logs = @i.log.out.logs
394
+ assert { logs.any? { |log| log.include?("${chunk_id} is not allowed in this plugin") } }
395
+ end
396
+
397
+ test '#extract_placeholders does not log for ${chunk_id} placeholder' do
398
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
399
+ tmpl = "/mypath/${chunk_id}/tail"
400
+ t = event_time('2016-04-11 20:30:00 +0900')
401
+ v = {key1: "value1", key2: "value2"}
402
+ c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
403
+ @i.log.out.logs.clear
404
+ @i.extract_placeholders(tmpl, c)
405
+ logs = @i.log.out.logs
406
+ assert { logs.none? { |log| log.include?("${chunk_id}") } }
407
+ end
408
+
409
+ test '#extract_placeholders does not log for ${chunk_id} placeholder (with @chunk_keys)' do
410
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))
411
+ tmpl = "/mypath/${chunk_id}/${key1}/tail"
412
+ t = event_time('2016-04-11 20:30:00 +0900')
413
+ v = {key1: "value1", key2: "value2"}
414
+ c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
415
+ @i.log.out.logs.clear
416
+ @i.extract_placeholders(tmpl, c)
417
+ logs = @i.log.out.logs
418
+ assert { logs.none? { |log| log.include?("${chunk_id}") } }
419
+ end
420
+
421
+ test '#extract_placeholders logs warn message with not replaced key' do
422
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
423
+ tmpl = "/mypath/${key1}/test"
424
+ t = event_time('2016-04-11 20:30:00 +0900')
425
+ v = { key1: "value1" }
426
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
427
+ @i.extract_placeholders(tmpl, m)
428
+ logs = @i.log.out.logs
429
+
430
+ assert { logs.any? { |log| log.include?("chunk key placeholder 'key1' not replaced. template:#{tmpl}") } }
431
+ end
432
+
433
+ test '#extract_placeholders logs warn message with not replaced key if variables exist and chunk_key is not empty' do
434
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))
435
+ tmpl = "/mypath/${key1}/${key2}/test"
436
+ t = event_time('2016-04-11 20:30:00 +0900')
437
+ v = { key1: "value1" }
438
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
439
+ @i.extract_placeholders(tmpl, m)
440
+ logs = @i.log.out.logs
441
+
442
+ assert { logs.any? { |log| log.include?("chunk key placeholder 'key2' not replaced. template:#{tmpl}") } }
443
+ end
444
+
445
+ sub_test_case '#placeholder_validators' do
446
+ test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
447
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
448
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
449
+ assert_equal 3, validators.size
450
+ assert_equal 1, validators.select(&:time?).size
451
+ assert_equal 1, validators.select(&:tag?).size
452
+ assert_equal 1, validators.select(&:keys?).size
453
+ end
454
+
455
+ test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
456
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
457
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
458
+ assert_equal 3, validators.size
459
+ assert_equal 1, validators.select(&:time?).size
460
+ assert_equal 1, validators.select(&:tag?).size
461
+ assert_equal 1, validators.select(&:keys?).size
462
+ end
463
+
464
+ test 'returns a validator for time if a template has timestamp placeholders' do
465
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
466
+ validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
467
+ assert_equal 1, validators.size
468
+ assert_equal 1, validators.select(&:time?).size
469
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
470
+ validators.first.validate!
471
+ end
472
+ end
473
+
474
+ test 'returns a validator for time if a plugin is configured with time key' do
475
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
476
+ validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
477
+ assert_equal 1, validators.size
478
+ assert_equal 1, validators.select(&:time?).size
479
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
480
+ validators.first.validate!
481
+ end
482
+ end
483
+
484
+ test 'returns a validator for tag if a template has tag placeholders' do
485
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
486
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
487
+ assert_equal 1, validators.size
488
+ assert_equal 1, validators.select(&:tag?).size
489
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
490
+ validators.first.validate!
491
+ end
492
+ end
493
+
494
+ test 'returns a validator for tag if a plugin is configured with tag key' do
495
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
496
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
497
+ assert_equal 1, validators.size
498
+ assert_equal 1, validators.select(&:tag?).size
499
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
500
+ validators.first.validate!
501
+ end
502
+ end
503
+
504
+ test 'returns a validator for variable keys if a template has variable placeholders' do
505
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
506
+ validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
507
+ assert_equal 1, validators.size
508
+ assert_equal 1, validators.select(&:keys?).size
509
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
510
+ validators.first.validate!
511
+ end
512
+ end
513
+
514
+ test 'returns a validator for variable keys if a plugin is configured with variable keys' do
515
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
516
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
517
+ assert_equal 1, validators.size
518
+ assert_equal 1, validators.select(&:keys?).size
519
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
520
+ validators.first.validate!
521
+ end
522
+ end
523
+ end
524
+
525
+ sub_test_case '#placeholder_validate!' do
526
+ test 'raises configuration error for a template when timestamp placeholders exist but time key is missing' do
527
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
528
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
529
+ @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
530
+ end
531
+ end
532
+
533
+ test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
534
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
535
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have timestamp placeholders for timekey 180") do
536
+ @i.placeholder_validate!(:path, "/my/path/file.log")
537
+ end
538
+ assert_nothing_raised do
539
+ @i.placeholder_validate!(:path, "/my/path/%Y%m%d/file.%H%M.log")
540
+ end
541
+ end
542
+
543
+ test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
544
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
545
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y%m%d_%H.log' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
546
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
547
+ end
548
+ assert_nothing_raised do
549
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H%M.log")
550
+ end
551
+ end
552
+
553
+ test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
554
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
555
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.${tag[2]}.log' has tag placeholders, but chunk key 'tag' is not configured") do
556
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
557
+ end
558
+ end
559
+
560
+ test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
561
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
562
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
563
+ @i.placeholder_validate!(:path, "/my/path/file.log")
564
+ end
565
+ assert_nothing_raised do
566
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
567
+ end
568
+ assert_nothing_raised do
569
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[-1]}.log")
570
+ end
571
+ end
572
+
573
+ test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
574
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
575
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.${username}.log' has placeholders, but chunk keys doesn't have keys service,username") do
576
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
577
+ end
578
+ end
579
+
580
+ test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
581
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
582
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys service,username") do
583
+ @i.placeholder_validate!(:path, "/my/path/file.log")
584
+ end
585
+ assert_nothing_raised do
586
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
587
+ end
588
+ end
589
+
590
+ test 'raise configuration error for a template and configuration with keys mismatch' do
591
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
592
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.${username}.log' doesn't have enough placeholders for keys service") do
593
+ @i.placeholder_validate!(:path, "/my/path/file.${username}.log")
594
+ end
595
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.log' doesn't have enough placeholders for keys username") do
596
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.log")
597
+ end
598
+ assert_nothing_raised do
599
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
600
+ end
601
+ end
602
+ end
603
+
604
+ test '#get_placeholders_time returns seconds,title and example placeholder for a template' do
605
+ s, t, e = @i.get_placeholders_time("/path/to/dir/yay")
606
+ assert_nil s
607
+ assert_nil t
608
+ assert_nil e
609
+
610
+ s, t, e = @i.get_placeholders_time("/path/to/%Y%m%d/yay")
611
+ assert_equal 86400, s
612
+ assert_equal :day, t
613
+ assert_equal '%d', e
614
+ s, t, e = @i.get_placeholders_time("my birthday! at %F")
615
+ assert_equal 86400, s
616
+ assert_equal :day, t
617
+ assert_equal '%d', e
618
+
619
+ s, t, e = @i.get_placeholders_time("myfile.%Y-%m-%d_%H.log")
620
+ assert_equal 3600, s
621
+ assert_equal :hour, t
622
+ assert_equal '%H', e
623
+
624
+ s, t, e = @i.get_placeholders_time("part-%Y%m%d-%H%M.ts")
625
+ assert_equal 60, s
626
+ assert_equal :minute, t
627
+ assert_equal '%M', e
628
+
629
+ s, t, e = @i.get_placeholders_time("my first data at %F %T %z")
630
+ assert_equal 1, s
631
+ assert_equal :second, t
632
+ assert_equal '%S', e
633
+ end
634
+
635
+ test '#get_placeholders_tag returns a list of tag part position for a template' do
636
+ assert_equal [], @i.get_placeholders_tag("db.table")
637
+ assert_equal [], @i.get_placeholders_tag("db.table_${non_tag}")
638
+ assert_equal [-1], @i.get_placeholders_tag("table_${tag}")
639
+ assert_equal [0, 1], @i.get_placeholders_tag("db_${tag[0]}.table_${tag[1]}")
640
+ assert_equal [-1, 0], @i.get_placeholders_tag("/treedir/${tag[0]}/${tag}")
641
+ end
642
+
643
+ test '#get_placeholders_keys returns a list of keys for a template' do
644
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/data/file.log")
645
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/${tag}/file.log")
646
+ assert_equal ['key1', 'key2'], @i.get_placeholders_keys("/path/to/${key2}/${tag}/file.${key1}.log")
647
+ assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
648
+ end
649
+
650
+ data('include space' => 'ke y',
651
+ 'bracket notation' => "$['key']",
652
+ 'invalid notation' => "$.ke y")
653
+ test 'configure checks invalid chunk keys' do |chunk_keys|
654
+ i = create_output(:buffered)
655
+ assert_raise Fluent::ConfigError do
656
+ i.configure(config_element('ROOT' , '', {}, [config_element('buffer', chunk_keys)]))
657
+ end
658
+ end
659
+
660
+ test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
661
+ tag = 'test.output'
662
+ time = event_time('2016-04-12 15:31:23 -0700')
663
+ timekey = event_time('2016-04-12 15:00:00 -0700')
664
+ record = {"key1" => "value1", "num1" => 1, "message" => "my message", "nest" => {"key" => "nested value"}}
665
+
666
+ i1 = create_output(:buffered)
667
+ i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
668
+ assert_equal create_metadata(), i1.metadata(tag, time, record)
669
+
670
+ i2 = create_output(:buffered)
671
+ i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))
672
+ assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)
673
+
674
+ i3 = create_output(:buffered)
675
+ i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
676
+ assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)
677
+
678
+ i4 = create_output(:buffered)
679
+ i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))
680
+ assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record)
681
+
682
+ i5 = create_output(:buffered)
683
+ i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))
684
+ assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record)
685
+
686
+ i6 = create_output(:buffered)
687
+ i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
688
+ assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)
689
+
690
+ i7 = create_output(:buffered)
691
+ i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
692
+ assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)
693
+
694
+ i8 = create_output(:buffered)
695
+ i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
696
+ assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
697
+
698
+ i9 = create_output(:buffered)
699
+ i9.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,$.nest.key', {})]))
700
+ assert_equal create_metadata(variables: {:key1 => "value1", :"$.nest.key" => 'nested value'}), i9.metadata(tag, time, record)
701
+ end
702
+
703
+ test '#emit calls #process via #emit_sync for non-buffered output' do
704
+ i = create_output(:sync)
705
+ process_called = false
706
+ i.register(:process){|tag, es| process_called = true }
707
+ i.configure(config_element())
708
+ i.start
709
+ i.after_start
710
+
711
+ t = event_time()
712
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
713
+
714
+ assert process_called
715
+
716
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
717
+ end
718
+
719
+ test '#emit calls #format for buffered output' do
720
+ i = create_output(:buffered)
721
+ format_called_times = 0
722
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
723
+ i.configure(config_element())
724
+ i.start
725
+ i.after_start
726
+
727
+ t = event_time()
728
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
729
+
730
+ assert_equal 2, format_called_times
731
+
732
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
733
+ end
734
+
735
+ test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do
736
+ i = create_output(:full)
737
+
738
+ process_called = false
739
+ format_called_times = 0
740
+ i.register(:process){|tag, es| process_called = true }
741
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
742
+
743
+ i.configure(config_element())
744
+ i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
745
+ i.start
746
+ i.after_start
747
+
748
+ assert !i.prefer_buffered_processing
749
+
750
+ t = event_time()
751
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
752
+
753
+ waiting(4){ Thread.pass until process_called }
754
+
755
+ assert process_called
756
+ assert_equal 0, format_called_times
757
+
758
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
759
+ end
760
+
761
+ test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do
762
+ i = create_output(:full)
763
+
764
+ process_called = false
765
+ format_called_times = 0
766
+ i.register(:process){|tag, es| process_called = true }
767
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
768
+
769
+ i.configure(config_element())
770
+ i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
771
+ i.start
772
+ i.after_start
773
+
774
+ assert i.prefer_buffered_processing
775
+
776
+ t = event_time()
777
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
778
+
779
+ assert !process_called
780
+ assert_equal 2, format_called_times
781
+
782
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
783
+ end
784
+
785
+ test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do
786
+ i = create_output(:buffered)
787
+ write_called = false
788
+ i.register(:write){ |chunk| write_called = true }
789
+
790
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
791
+ i.start
792
+ i.after_start
793
+
794
+ t = event_time()
795
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
796
+ i.force_flush
797
+
798
+ waiting(4){ Thread.pass until write_called }
799
+
800
+ assert write_called
801
+
802
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
803
+ end
804
+
805
+ test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
806
+ i = create_output(:delayed)
807
+ try_write_called = false
808
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
809
+
810
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
811
+ i.start
812
+ i.after_start
813
+
814
+ t = event_time()
815
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
816
+ i.force_flush
817
+
818
+ waiting(4){ Thread.pass until try_write_called }
819
+
820
+ assert try_write_called
821
+
822
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
823
+ end
824
+
825
+ test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
826
+ i = create_output(:full)
827
+ write_called = false
828
+ try_write_called = false
829
+ i.register(:write){ |chunk| write_called = true }
830
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
831
+
832
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
833
+ i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
834
+ i.start
835
+ i.after_start
836
+
837
+ assert !i.prefer_delayed_commit
838
+
839
+ t = event_time()
840
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
841
+ i.force_flush
842
+
843
+ waiting(4){ Thread.pass until write_called || try_write_called }
844
+
845
+ assert write_called
846
+ assert !try_write_called
847
+
848
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
849
+ end
850
+
851
+ test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
852
+ i = create_output(:full)
853
+ write_called = false
854
+ try_write_called = false
855
+ i.register(:write){ |chunk| write_called = true }
856
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
857
+
858
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
859
+ i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
860
+ i.start
861
+ i.after_start
862
+
863
+ assert i.prefer_delayed_commit
864
+
865
+ t = event_time()
866
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
867
+ i.force_flush
868
+
869
+ waiting(4){ Thread.pass until write_called || try_write_called }
870
+
871
+ assert !write_called
872
+ assert try_write_called
873
+
874
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
875
+ end
876
+
877
+ test 'flush_interval is ignored when flush_mode is not interval' do
878
+ mock(@i.log).warn("'flush_interval' is ignored because default 'flush_mode' is not 'interval': 'lazy'")
879
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'flush_interval' => 10})]))
880
+ end
881
+
882
+ data(:lazy => 'lazy', :immediate => 'immediate')
883
+ test 'flush_interval and non-interval flush_mode is exclusive ' do |mode|
884
+ assert_raise Fluent::ConfigError.new("'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{mode}'") do
885
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'flush_mode' => mode, 'flush_interval' => 10})]))
886
+ end
887
+ end
888
+
889
+ test 'flush_mode is set to interval when flush_interval with v0.12 configuration is given' do
890
+ mock(@i.log).info("'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour")
891
+ @i.configure(config_element('ROOT', '', {'flush_interval' => 60}, []))
892
+ assert_equal :interval, @i.instance_variable_get(:@flush_mode)
893
+ end
894
+
895
+ sub_test_case 'configure secondary' do
896
+ test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
897
+ o = create_output(:buffered)
898
+ mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
899
+ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput")
900
+
901
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
902
+ assert_not_nil o.instance_variable_get(:@secondary)
903
+ end
904
+
905
+ test "don't warn if primary type is the same as secondary type" do
906
+ o = Fluent::Plugin::TestOutput.new
907
+ mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
908
+ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" ).never
909
+
910
+ o.configure(config_element('ROOT','',{'name' => "cool2"},
911
+ [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
912
+ config_element('buffer','',{'@type'=>'memory'})]
913
+ ))
914
+ assert_not_nil o.instance_variable_get(:@secondary)
915
+ end
916
+
917
+ test "don't warn if primary type is different from secondary type and both don't have custom_format" do
918
+ o = create_output(:standard)
919
+ mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
920
+ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput").never
921
+
922
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
923
+ assert_not_nil o.instance_variable_get(:@secondary)
924
+ end
925
+
926
+ test "raise configuration error if secondary type specifies non buffered output" do
927
+ o = create_output(:standard)
928
+ assert_raise Fluent::ConfigError do
929
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'copy'})]))
930
+ end
931
+ end
932
+ end
933
+ end
934
+
935
+ test 'raises an error if timekey is less than equal 0' do
936
+ i = create_output(:delayed)
937
+ assert_raise Fluent::ConfigError.new("<buffer ...> argument includes 'time', but timekey is not configured") do
938
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => nil })]))
939
+ end
940
+
941
+ i = create_output(:delayed)
942
+ assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
943
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => 0 })]))
944
+ end
945
+
946
+ i = create_output(:delayed)
947
+ assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: -1.0') do
948
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => -1 })]))
949
+ end
950
+ end
951
+
952
+ sub_test_case 'sync output feature' do
953
+ setup do
954
+ @i = create_output(:sync)
955
+ end
956
+
957
+ test 'raises configuration error if <buffer> section is specified' do
958
+ assert_raise Fluent::ConfigError do
959
+ @i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
960
+ end
961
+ end
962
+
963
+ test 'raises configuration error if <secondary> section is specified' do
964
+ assert_raise Fluent::ConfigError do
965
+ @i.configure(config_element('ROOT','',{},[config_element('secondary','')]))
966
+ end
967
+ end
968
+
969
+ test '#process is called for each event streams' do
970
+ ary = []
971
+ @i.register(:process){|tag, es| ary << [tag, es] }
972
+ @i.configure(config_element())
973
+ @i.start
974
+ @i.after_start
975
+
976
+ t = event_time()
977
+ es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
978
+ 5.times do
979
+ @i.emit_events('tag', es)
980
+ end
981
+ assert_equal 5, ary.size
982
+
983
+ @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
984
+ end
985
+ end
986
+
987
+ sub_test_case '#generate_format_proc' do
988
+ test "when output doesn't have <buffer>" do
989
+ i = create_output(:sync)
990
+ i.configure(config_element('ROOT', '', {}, []))
991
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
992
+ end
993
+
994
+ test "when output doesn't have <buffer> and time_as_integer is true" do
995
+ i = create_output(:sync)
996
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}))
997
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
998
+ end
999
+
1000
+ test 'when output has <buffer> and compress is gzip' do
1001
+ i = create_output(:buffered)
1002
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))
1003
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM, i.generate_format_proc
1004
+ end
1005
+
1006
+ test 'when output has <buffer> and compress is gzip and time_as_integer is true' do
1007
+ i = create_output(:buffered)
1008
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))
1009
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
1010
+ end
1011
+
1012
+ test 'when output has <buffer> and compress is text' do
1013
+ i = create_output(:buffered)
1014
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))
1015
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
1016
+ end
1017
+ end
1018
+
1019
+ sub_test_case 'slow_flush_log_threshold' do
1020
+ def invoke_slow_flush_log_threshold_test(i)
1021
+ i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5},
1022
+ [config_element('buffer', '', {"flush_mode" => "immediate", "flush_thread_interval" => 30})]))
1023
+ i.start
1024
+ i.after_start
1025
+
1026
+ t = event_time()
1027
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
1028
+ i.force_flush
1029
+
1030
+ waiting(4) { Thread.pass until i.test_finished? }
1031
+
1032
+ yield
1033
+ ensure
1034
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
1035
+ end
1036
+
1037
+ test '#write flush took longer time than slow_flush_log_threshold' do
1038
+ i = create_output(:buffered)
1039
+ write_called = false
1040
+ i.register(:write) { |chunk| sleep 3 }
1041
+ i.define_singleton_method(:test_finished?) { write_called }
1042
+ i.define_singleton_method(:try_flush) { super(); write_called = true }
1043
+
1044
+ invoke_slow_flush_log_threshold_test(i) {
1045
+ assert write_called
1046
+ logs = i.log.out.logs
1047
+ assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
1048
+ }
1049
+ end
1050
+
1051
+ test '#try_write flush took longer time than slow_flush_log_threshold' do
1052
+ i = create_output(:delayed)
1053
+ try_write_called = false
1054
+ i.register(:try_write){ |chunk| sleep 3 }
1055
+ i.define_singleton_method(:test_finished?) { try_write_called }
1056
+ i.define_singleton_method(:try_flush) { super(); try_write_called = true }
1057
+
1058
+ invoke_slow_flush_log_threshold_test(i) {
1059
+ assert try_write_called
1060
+ logs = i.log.out.logs
1061
+ assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
1062
+ }
1063
+ end
1064
+ end
1065
+ end