fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,606 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_exec_filter'
4
+ require 'fileutils'
5
+
6
+ class ExecFilterOutputTest < Test::Unit::TestCase
7
+ def setup
8
+ Fluent::Test.setup
9
+ end
10
+
11
+ CONFIG = %[
12
+ command cat
13
+ num_children 3
14
+ <inject>
15
+ tag_key tag
16
+ time_key time_in
17
+ time_type string
18
+ time_format %Y-%m-%d %H:%M:%S
19
+ </inject>
20
+ <format>
21
+ keys ["time_in", "tag", "k1"]
22
+ </format>
23
+ <parse>
24
+ keys ["time_out", "tag", "k2"]
25
+ </parse>
26
+ <extract>
27
+ tag_key tag
28
+ time_key time_out
29
+ time_type string
30
+ time_format %Y-%m-%d %H:%M:%S
31
+ </extract>
32
+ ]
33
+
34
+ CONFIG_COMPAT = %[
35
+ command cat
36
+ in_keys time_in,tag,k1
37
+ out_keys time_out,tag,k2
38
+ tag_key tag
39
+ in_time_key time_in
40
+ out_time_key time_out
41
+ time_format %Y-%m-%d %H:%M:%S
42
+ localtime
43
+ num_children 3
44
+ ]
45
+
46
+ def create_driver(conf)
47
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)
48
+ end
49
+
50
+ SED_SUPPORT_UNBUFFERED_OPTION = ->(){
51
+ system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
52
+ $?.success?
53
+ }.call
54
+ SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''
55
+
56
+ data(
57
+ 'with sections' => CONFIG,
58
+ 'traditional' => CONFIG_COMPAT,
59
+ )
60
+ test 'configure' do |conf|
61
+ d = create_driver(conf)
62
+
63
+ assert_false d.instance.parser.estimate_current_event
64
+
65
+ assert_equal ["time_in","tag","k1"], d.instance.formatter.keys
66
+ assert_equal ["time_out","tag","k2"], d.instance.parser.keys
67
+ assert_equal "tag", d.instance.inject_config.tag_key
68
+ assert_equal "tag", d.instance.extract_config.tag_key
69
+ assert_equal "time_in", d.instance.inject_config.time_key
70
+ assert_equal "time_out", d.instance.extract_config.time_key
71
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
72
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
73
+ assert_equal true, d.instance.inject_config.localtime
74
+ assert_equal 3, d.instance.num_children
75
+
76
+ d = create_driver %[
77
+ command sed -l -e s/foo/bar/
78
+ in_keys time,k1
79
+ out_keys time,k2
80
+ tag xxx
81
+ time_key time
82
+ num_children 3
83
+ ]
84
+ assert_equal "sed -l -e s/foo/bar/", d.instance.command
85
+
86
+ d = create_driver(conf + %[
87
+ remove_prefix before
88
+ add_prefix after
89
+ ])
90
+ assert_equal "before", d.instance.remove_prefix
91
+ assert_equal "after" , d.instance.add_prefix
92
+ end
93
+
94
+ data(
95
+ 'with sections' => CONFIG,
96
+ 'traditional' => CONFIG_COMPAT,
97
+ )
98
+ test 'emit events with TSV format' do |conf|
99
+ d = create_driver(conf)
100
+ time = event_time("2011-01-02 13:14:15")
101
+
102
+ d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
103
+ # sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }
104
+ d.feed(time, {"k1"=>1})
105
+ d.feed(time, {"k1"=>2})
106
+ end
107
+
108
+ assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
109
+ assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
110
+
111
+ events = d.events
112
+ assert_equal 2, events.length
113
+ assert_equal_event_time time, events[0][1]
114
+ assert_equal ["test", time, {"k2"=>"1"}], events[0]
115
+ assert_equal_event_time time, events[1][1]
116
+ assert_equal ["test", time, {"k2"=>"2"}], events[1]
117
+ end
118
+
119
+ CONFIG_WITHOUT_TIME_FORMAT = %[
120
+ command cat
121
+ num_children 3
122
+ tag xxx
123
+ <inject>
124
+ time_key time
125
+ time_type unixtime
126
+ </inject>
127
+ <format>
128
+ keys time,k1
129
+ </format>
130
+ <parse>
131
+ keys time,k2
132
+ time_key time
133
+ time_type unixtime
134
+ </parse>
135
+ ]
136
+ CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[
137
+ command cat
138
+ in_keys time,k1
139
+ out_keys time,k2
140
+ tag xxx
141
+ time_key time
142
+ num_children 3
143
+ ]
144
+
145
+ data(
146
+ 'with sections' => CONFIG_WITHOUT_TIME_FORMAT,
147
+ 'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,
148
+ )
149
+ test 'emit events without time format configuration' do |conf|
150
+ d = create_driver(conf)
151
+ time = event_time("2011-01-02 13:14:15 +0900")
152
+
153
+ d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
154
+ d.feed(time, {"k1"=>1})
155
+ d.feed(time, {"k1"=>2})
156
+ end
157
+
158
+ assert_equal "1293941655\t1\n", d.formatted[0]
159
+ assert_equal "1293941655\t2\n", d.formatted[1]
160
+
161
+ events = d.events
162
+ assert_equal 2, events.length
163
+ assert_equal_event_time time, events[0][1]
164
+ assert_equal ["xxx", time, {"k2"=>"1"}], events[0]
165
+ assert_equal_event_time time, events[1][1]
166
+ assert_equal ["xxx", time, {"k2"=>"2"}], events[1]
167
+ end
168
+
169
+ CONFIG_TO_DO_GREP = %[
170
+ command grep --line-buffered -v poo
171
+ num_children 3
172
+ tag xxx
173
+ <inject>
174
+ time_key time
175
+ time_type unixtime
176
+ </inject>
177
+ <format>
178
+ keys time, val1
179
+ </format>
180
+ <parse>
181
+ keys time, val2
182
+ time_key time
183
+ time_type unixtime
184
+ </parse>
185
+ ]
186
+ CONFIG_TO_DO_GREP_COMPAT = %[
187
+ command grep --line-buffered -v poo
188
+ in_keys time,val1
189
+ out_keys time,val2
190
+ tag xxx
191
+ time_key time
192
+ num_children 3
193
+ ]
194
+
195
+ data(
196
+ 'with sections' => CONFIG_TO_DO_GREP,
197
+ 'traditional' => CONFIG_TO_DO_GREP_COMPAT,
198
+ )
199
+ test 'emit events through grep command' do |conf|
200
+ d = create_driver(conf)
201
+ time = event_time("2011-01-02 13:14:15 +0900")
202
+
203
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
204
+ d.feed(time, {"val1"=>"sed-ed value poo"})
205
+ d.feed(time, {"val1"=>"sed-ed value foo"})
206
+ end
207
+
208
+ assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
209
+ assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
210
+
211
+ events = d.events
212
+ assert_equal 1, events.length
213
+ assert_equal_event_time time, events[0][1]
214
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], events[0]
215
+ end
216
+
217
+ CONFIG_TO_DO_SED = %[
218
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
219
+ num_children 3
220
+ tag xxx
221
+ <inject>
222
+ time_key time
223
+ time_type unixtime
224
+ </inject>
225
+ <format>
226
+ keys time, val1
227
+ </format>
228
+ <parse>
229
+ keys time, val2
230
+ time_key time
231
+ time_type unixtime
232
+ </parse>
233
+ ]
234
+ CONFIG_TO_DO_SED_COMPAT = %[
235
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
236
+ in_keys time,val1
237
+ out_keys time,val2
238
+ tag xxx
239
+ time_key time
240
+ num_children 3
241
+ ]
242
+
243
+ data(
244
+ 'with sections' => CONFIG_TO_DO_SED,
245
+ 'traditional' => CONFIG_TO_DO_SED_COMPAT,
246
+ )
247
+ test 'emit events through sed command' do |conf|
248
+ d = create_driver(conf)
249
+ time = event_time("2011-01-02 13:14:15 +0900")
250
+
251
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
252
+ d.feed(time, {"val1"=>"sed-ed value poo"})
253
+ d.feed(time, {"val1"=>"sed-ed value foo"})
254
+ end
255
+
256
+ assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
257
+ assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
258
+
259
+ events = d.events
260
+ assert_equal 2, events.length
261
+ assert_equal_event_time time, events[0][1]
262
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], events[0]
263
+ assert_equal_event_time time, events[1][1]
264
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], events[1]
265
+ end
266
+
267
+ CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[
268
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
269
+ num_children 3
270
+ remove_prefix input
271
+ add_prefix output
272
+ <inject>
273
+ tag_key tag
274
+ time_key time
275
+ </inject>
276
+ <format>
277
+ keys tag, time, val1
278
+ </format>
279
+ <parse>
280
+ keys tag, time, val2
281
+ </parse>
282
+ <extract>
283
+ tag_key tag
284
+ time_key time
285
+ </extract>
286
+ ]
287
+ CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[
288
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
289
+ in_keys tag,time,val1
290
+ remove_prefix input
291
+ out_keys tag,time,val2
292
+ add_prefix output
293
+ tag_key tag
294
+ time_key time
295
+ num_children 3
296
+ ]
297
+
298
+ data(
299
+ 'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,
300
+ 'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,
301
+ )
302
+ test 'emit events with add/remove tag prefix' do |conf|
303
+ d = create_driver(conf)
304
+
305
+ time = event_time("2011-01-02 13:14:15 +0900")
306
+
307
+ d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do
308
+ d.feed(time, {"val1"=>"sed-ed value foo"})
309
+ d.feed(time, {"val1"=>"sed-ed value poo"})
310
+ end
311
+
312
+ assert_equal "test\t1293941655\tsed-ed value foo\n", d.formatted[0]
313
+ assert_equal "test\t1293941655\tsed-ed value poo\n", d.formatted[1]
314
+
315
+ events = d.events
316
+ assert_equal 2, events.length
317
+ assert_equal_event_time time, events[0][1]
318
+ assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], events[0]
319
+ assert_equal_event_time time, events[1][1]
320
+ assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], events[1]
321
+ end
322
+
323
+ CONFIG_JSON = %[
324
+ command cat
325
+ <format>
326
+ @type tsv
327
+ keys message
328
+ </format>
329
+ <parse>
330
+ @type json
331
+ stream_buffer_size 1
332
+ </parse>
333
+ <extract>
334
+ tag_key tag
335
+ time_key time
336
+ </extract>
337
+ ]
338
+ CONFIG_JSON_COMPAT = %[
339
+ command cat
340
+ in_keys message
341
+ out_format json
342
+ out_stream_buffer_size 1
343
+ time_key time
344
+ tag_key tag
345
+ ]
346
+
347
+ data(
348
+ 'with sections' => CONFIG_JSON,
349
+ 'traditional' => CONFIG_JSON_COMPAT,
350
+ )
351
+ test 'using json format' do |conf|
352
+ d = create_driver(conf)
353
+ time = event_time("2011-01-02 13:14:15 +0900")
354
+
355
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
356
+ i = d.instance
357
+ assert{ i.router }
358
+ d.feed(time, {"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]})
359
+ end
360
+
361
+ assert_equal '{"time":1293941655,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
362
+
363
+ events = d.events
364
+ assert_equal 1, events.length
365
+ assert_equal_event_time time, events[0][1]
366
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
367
+ end
368
+
369
+ CONFIG_JSON_WITH_FLOAT_TIME = %[
370
+ command cat
371
+ <format>
372
+ @type tsv
373
+ keys message
374
+ </format>
375
+ <parse>
376
+ @type json
377
+ stream_buffer_size 1
378
+ </parse>
379
+ <extract>
380
+ tag_key tag
381
+ time_key time
382
+ </extract>
383
+ ]
384
+ CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[
385
+ command cat
386
+ in_keys message
387
+ out_format json
388
+ out_stream_buffer_size 1
389
+ time_key time
390
+ tag_key tag
391
+ ]
392
+
393
+ data(
394
+ 'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,
395
+ 'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,
396
+ )
397
+ test 'using json format with float time' do |conf|
398
+ d = create_driver(conf)
399
+ time = event_time("2011-01-02 13:14:15.123 +0900")
400
+
401
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
402
+ d.feed(time + 10, {"message"=>%[{"time":#{time.sec}.#{time.nsec},"tag":"t1","k1":"v1"}]})
403
+ end
404
+
405
+ assert_equal '{"time":1293941655.123000000,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
406
+
407
+ events = d.events
408
+ assert_equal 1, events.length
409
+ assert_equal_event_time time, events[0][1]
410
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
411
+ end
412
+
413
+ CONFIG_JSON_WITH_TIME_FORMAT = %[
414
+ command cat
415
+ <format>
416
+ @type tsv
417
+ keys message
418
+ </format>
419
+ <parse>
420
+ @type json
421
+ stream_buffer_size 1
422
+ </parse>
423
+ <extract>
424
+ tag_key tag
425
+ time_key time
426
+ time_type string
427
+ time_format %d/%b/%Y %H:%M:%S.%N %z
428
+ </extract>
429
+ ]
430
+ CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[
431
+ command cat
432
+ in_keys message
433
+ out_format json
434
+ out_stream_buffer_size 1
435
+ time_key time
436
+ time_format %d/%b/%Y %H:%M:%S.%N %z
437
+ tag_key tag
438
+ ]
439
+
440
+ data(
441
+ 'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,
442
+ 'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,
443
+ )
444
+ test 'using json format with custom time format' do |conf|
445
+ d = create_driver(conf)
446
+ time_str = "28/Feb/2013 12:00:00.123456789 +0900"
447
+ time = event_time(time_str, format: "%d/%b/%Y %H:%M:%S.%N %z")
448
+
449
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
450
+ d.feed(time + 10, {"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]})
451
+ end
452
+
453
+ assert_equal '{"time":"28/Feb/2013 12:00:00.123456789 +0900","tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
454
+
455
+ events = d.events
456
+ assert_equal 1, events.length
457
+ assert_equal_event_time time, events[0][1]
458
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
459
+ end
460
+
461
+ CONFIG_ROUND_ROBIN = %[
462
+ command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
463
+ num_children 2
464
+ <inject>
465
+ tag_key tag
466
+ time_key time_in
467
+ time_type string
468
+ time_format %Y-%m-%d %H:%M:%S
469
+ </inject>
470
+ <format>
471
+ keys ["time_in", "tag", "k1"]
472
+ </format>
473
+ <parse>
474
+ keys ["time_out", "tag", "k2", "child_pid"]
475
+ </parse>
476
+ <extract>
477
+ tag_key tag
478
+ time_key time_out
479
+ time_type string
480
+ time_format %Y-%m-%d %H:%M:%S
481
+ </extract>
482
+ ]
483
+ CONFIG_ROUND_ROBIN_COMPAT = %[
484
+ command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
485
+ in_keys time_in,tag,k1
486
+ out_keys time_out,tag,k2,child_pid
487
+ tag_key tag
488
+ in_time_key time_in
489
+ out_time_key time_out
490
+ time_format %Y-%m-%d %H:%M:%S
491
+ localtime
492
+ num_children 2
493
+ ]
494
+
495
+ data(
496
+ 'with sections' => CONFIG_ROUND_ROBIN,
497
+ 'traditional' => CONFIG_ROUND_ROBIN_COMPAT,
498
+ )
499
+ test 'using child processes by round robin' do |conf|
500
+ d = create_driver(conf)
501
+ time = event_time('2011-01-02 13:14:15')
502
+
503
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true, shutdown: false){ d.feed(time, {"k1" => 0}) }
504
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 1}) }
505
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 2}) }
506
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 3}) }
507
+
508
+ assert_equal "2011-01-02 13:14:15\ttest\t0\n", d.formatted[0]
509
+ assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[1]
510
+ assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[2]
511
+ assert_equal "2011-01-02 13:14:15\ttest\t3\n", d.formatted[3]
512
+
513
+ events = d.events
514
+ assert_equal 4, events.length
515
+
516
+ pid_list = []
517
+ events.each do |event|
518
+ pid = event[2]['child_pid']
519
+ pid_list << pid unless pid_list.include?(pid)
520
+ end
521
+ assert_equal 2, pid_list.size, "the number of pids should be same with number of child processes: #{pid_list.inspect}"
522
+
523
+ assert_equal pid_list[0], events[0][2]['child_pid']
524
+ assert_equal pid_list[1], events[1][2]['child_pid']
525
+ assert_equal pid_list[0], events[2][2]['child_pid']
526
+ assert_equal pid_list[1], events[3][2]['child_pid']
527
+
528
+ ensure
529
+ d.run(start: false, shutdown: true)
530
+ end
531
+
532
+ # child process exits per 3 lines
533
+ CONFIG_RESPAWN = %[
534
+ command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
535
+ num_children 2
536
+ child_respawn -1
537
+ <inject>
538
+ tag_key tag
539
+ time_key time_in
540
+ time_type unixtime
541
+ </inject>
542
+ <format>
543
+ keys ["time_in", "tag", "k1"]
544
+ </format>
545
+ <parse>
546
+ keys ["time_out", "tag", "k2", "child_pid"]
547
+ </parse>
548
+ <extract>
549
+ tag_key tag
550
+ time_key time_out
551
+ time_type unixtime
552
+ </extract>
553
+ ]
554
+
555
+ CONFIG_RESPAWN_COMPAT = %[
556
+ command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
557
+ num_children 2
558
+ child_respawn -1
559
+ in_keys time_in,tag,k1
560
+ out_keys time_out,tag,k2,child_pid
561
+ tag_key tag
562
+ in_time_key time_in
563
+ out_time_key time_out
564
+ # time_format %Y-%m-%d %H:%M:%S
565
+ # localtime
566
+ ]
567
+
568
+ data(
569
+ 'with sections' => CONFIG_RESPAWN,
570
+ 'traditional' => CONFIG_RESPAWN_COMPAT,
571
+ )
572
+ test 'emit events via child processes which exits sometimes' do |conf|
573
+ d = create_driver(conf)
574
+ time = event_time("2011-01-02 13:14:15")
575
+ countup = 0
576
+
577
+ d.run(start: true, shutdown: false)
578
+ assert_equal 2, d.instance.instance_eval{ @_child_process_processes.size }
579
+
580
+ 2.times do
581
+ d.run(default_tag: 'test', expect_emits: 3, timeout: 3, force_flush_retry: true, start: false, shutdown: false) do
582
+ d.feed(time, { "k1" => countup }); countup += 1
583
+ d.feed(time, { "k1" => countup }); countup += 1
584
+ d.feed(time, { "k1" => countup }); countup += 1
585
+ end
586
+ end
587
+
588
+ events = d.events
589
+ assert_equal 6, events.length
590
+
591
+ pid_list = []
592
+ events.each do |event|
593
+ pid = event[2]['child_pid']
594
+ pid_list << pid unless pid_list.include?(pid)
595
+ end
596
+
597
+ # the number of pids should be same with number of child processes
598
+ assert_equal 2, pid_list.size
599
+ logs = d.instance.log.out.logs
600
+ assert_equal 2, logs.select { |l| l.include?('child process exits with error code') }.size
601
+ assert_equal 2, logs.select { |l| l.include?('respawning child process') }.size
602
+
603
+ ensure
604
+ d.run(start: false, shutdown: true)
605
+ end
606
+ end