fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,731 @@
1
+ require_relative '../helper'
2
+ require 'timecop'
3
+ require 'fluent/test/driver/filter'
4
+ require 'fluent/plugin/filter_parser'
5
+ require 'flexmock/test_unit'
6
+
7
+ class ParserFilterTest < Test::Unit::TestCase
8
+ include FlexMock::TestCase
9
+
10
+ def setup
11
+ Fluent::Test.setup
12
+ @tag = 'test'
13
+ @default_time = Time.parse('2010-05-04 03:02:01 UTC')
14
+ Timecop.freeze(@default_time)
15
+ end
16
+
17
+ def teardown
18
+ super
19
+ Timecop.return
20
+ end
21
+
22
+ def assert_equal_parsed_time(expected, actual)
23
+ if expected.is_a?(Integer)
24
+ assert_equal(expected, actual.to_i)
25
+ else
26
+ assert_equal_event_time(expected, actual)
27
+ end
28
+ end
29
+
30
+ ParserError = Fluent::Plugin::Parser::ParserError
31
+ CONFIG = %[
32
+ key_name message
33
+ reserve_data true
34
+ <parse>
35
+ @type regexp
36
+ expression /^(?<x>.)(?<y>.) (?<time>.+)$/
37
+ time_format %Y%m%d%H%M%S
38
+ </parse>
39
+ ]
40
+
41
+ def create_driver(conf=CONFIG)
42
+ Fluent::Test::Driver::Filter.new(Fluent::Plugin::ParserFilter).configure(conf)
43
+ end
44
+
45
+ def test_configure
46
+ assert_raise(Fluent::ConfigError) {
47
+ create_driver('')
48
+ }
49
+ assert_raise(Fluent::ConfigError) {
50
+ create_driver %[
51
+ key_name foo
52
+ <parse>
53
+ @type unknown_format_that_will_never_be_implemented
54
+ </parse>
55
+ ]
56
+ }
57
+ assert_nothing_raised {
58
+ create_driver %[
59
+ key_name foo
60
+ <parse>
61
+ @type regexp
62
+ expression /(?<x>.)/
63
+ </parse>
64
+ ]
65
+ }
66
+ assert_nothing_raised {
67
+ create_driver %[
68
+ key_name foo
69
+ <parse>
70
+ @type json
71
+ </parse>
72
+ ]
73
+ }
74
+ assert_nothing_raised {
75
+ create_driver %[
76
+ key_name foo
77
+ format json
78
+ ]
79
+ }
80
+ assert_nothing_raised {
81
+ create_driver %[
82
+ key_name foo
83
+ <parse>
84
+ @type ltsv
85
+ </parse>
86
+ ]
87
+ }
88
+ assert_nothing_raised {
89
+ create_driver %[
90
+ key_name message
91
+ <parse>
92
+ @type regexp
93
+ expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/
94
+ </parse>
95
+ ]
96
+ }
97
+ d = create_driver %[
98
+ key_name foo
99
+ <parse>
100
+ @type regexp
101
+ expression /(?<x>.)/
102
+ </parse>
103
+ ]
104
+ assert_false d.instance.reserve_data
105
+ end
106
+
107
+ # CONFIG = %[
108
+ # remove_prefix test
109
+ # add_prefix parsed
110
+ # key_name message
111
+ # format /^(?<x>.)(?<y>.) (?<time>.+)$/
112
+ # time_format %Y%m%d%H%M%S
113
+ # reserve_data true
114
+ # ]
115
+ def test_filter
116
+ d1 = create_driver(CONFIG)
117
+ time = event_time("2012-01-02 13:14:15")
118
+ d1.run(default_tag: @tag) do
119
+ d1.feed(time, {'message' => '12 20120402182059'})
120
+ d1.feed(time, {'message' => '34 20120402182100'})
121
+ d1.feed(time, {'message' => '56 20120402182100'})
122
+ d1.feed(time, {'message' => '78 20120402182101'})
123
+ d1.feed(time, {'message' => '90 20120402182100'})
124
+ end
125
+ filtered = d1.filtered
126
+ assert_equal 5, filtered.length
127
+
128
+ first = filtered[0]
129
+ assert_equal_event_time event_time("2012-04-02 18:20:59"), first[0]
130
+ assert_equal '1', first[1]['x']
131
+ assert_equal '2', first[1]['y']
132
+ assert_equal '12 20120402182059', first[1]['message']
133
+
134
+ second = filtered[1]
135
+ assert_equal_event_time event_time("2012-04-02 18:21:00"), second[0]
136
+ assert_equal '3', second[1]['x']
137
+ assert_equal '4', second[1]['y']
138
+
139
+ third = filtered[2]
140
+ assert_equal_event_time event_time("2012-04-02 18:21:00"), third[0]
141
+ assert_equal '5', third[1]['x']
142
+ assert_equal '6', third[1]['y']
143
+
144
+ fourth = filtered[3]
145
+ assert_equal_event_time event_time("2012-04-02 18:21:01"), fourth[0]
146
+ assert_equal '7', fourth[1]['x']
147
+ assert_equal '8', fourth[1]['y']
148
+
149
+ fifth = filtered[4]
150
+ assert_equal_event_time event_time("2012-04-02 18:21:00"), fifth[0]
151
+ assert_equal '9', fifth[1]['x']
152
+ assert_equal '0', fifth[1]['y']
153
+
154
+ d2 = create_driver(%[
155
+ key_name data
156
+ format /^(?<x>.)(?<y>.) (?<t>.+)$/
157
+ ])
158
+ time = Fluent::EventTime.from_time(@default_time) # EventTime emit test
159
+ d2.run(default_tag: @tag) do
160
+ d2.feed(time, {'data' => '12 20120402182059'})
161
+ d2.feed(time, {'data' => '34 20120402182100'})
162
+ end
163
+ filtered = d2.filtered
164
+ assert_equal 2, filtered.length
165
+
166
+ first = filtered[0]
167
+ assert_equal_event_time time, first[0]
168
+ assert_nil first[1]['data']
169
+ assert_equal '1', first[1]['x']
170
+ assert_equal '2', first[1]['y']
171
+ assert_equal '20120402182059', first[1]['t']
172
+
173
+ second = filtered[1]
174
+ assert_equal_event_time time, second[0]
175
+ assert_nil second[1]['data']
176
+ assert_equal '3', second[1]['x']
177
+ assert_equal '4', second[1]['y']
178
+ assert_equal '20120402182100', second[1]['t']
179
+
180
+ d3 = create_driver(%[
181
+ key_name data
182
+ <parse>
183
+ @type regexp
184
+ expression /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
185
+ </parse>
186
+ ])
187
+ time = Time.parse("2012-04-02 18:20:59").to_i
188
+ d3.run(default_tag: @tag) do
189
+ d3.feed(time, {'data' => '12 20120402182059'})
190
+ d3.feed(time, {'data' => '34 20120402182100'})
191
+ d3.feed(time, {'data' => 'xy 20120402182101'})
192
+ end
193
+ filtered = d3.filtered
194
+ assert_equal 2, filtered.length
195
+
196
+ d4 = create_driver(%[
197
+ key_name data
198
+ <parse>
199
+ @type json
200
+ </parse>
201
+ ])
202
+ time = Time.parse("2012-04-02 18:20:59").to_i
203
+ d4.run(default_tag: @tag) do
204
+ d4.feed(time, {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'})
205
+ d4.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})
206
+ end
207
+ filtered = d4.filtered
208
+ assert_equal 1, filtered.length
209
+
210
+ end
211
+
212
+ data(:keep_key_name => false,
213
+ :remove_key_name => true)
214
+ def test_filter_with_reserved_data(remove_key_name)
215
+ d1 = create_driver(%[
216
+ key_name data
217
+ reserve_data yes
218
+ remove_key_name_field #{remove_key_name}
219
+ <parse>
220
+ @type regexp
221
+ expression /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
222
+ </parse>
223
+ ])
224
+ time = event_time("2012-04-02 18:20:59")
225
+ d1.run(default_tag: @tag) do
226
+ d1.feed(time, {'data' => '12 20120402182059'})
227
+ d1.feed(time, {'data' => '34 20120402182100'})
228
+ d1.feed(time, {'data' => 'xy 20120402182101'})
229
+ end
230
+ filtered = d1.filtered
231
+ assert_equal 3, filtered.length
232
+
233
+ d2 = create_driver(%[
234
+ key_name data
235
+ reserve_data yes
236
+ remove_key_name_field #{remove_key_name}
237
+ <parse>
238
+ @type json
239
+ </parse>
240
+ ])
241
+ time = Fluent::EventTime.from_time(@default_time)
242
+ d2.run(default_tag: @tag) do
243
+ d2.feed(time, {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'})
244
+ d2.feed(time, {'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'})
245
+ end
246
+ filtered = d2.filtered
247
+ assert_equal 2, filtered.length
248
+
249
+ first = filtered[0]
250
+ assert_equal_event_time time, first[0]
251
+ if remove_key_name
252
+ assert_not_include first[1], 'data'
253
+ else
254
+ assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
255
+ end
256
+ assert_equal 'first', first[1]['xxx']
257
+ assert_equal 'second', first[1]['yyy']
258
+
259
+ second = filtered[1]
260
+ assert_equal_event_time time, second[0]
261
+ assert_equal 'foobar', second[1]['data']
262
+ assert_equal 'x', second[1]['xxx']
263
+ assert_equal 'y', second[1]['yyy']
264
+ end
265
+
266
+
267
+ CONFIG_LTSV = %[
268
+ key_name data
269
+ <parse>
270
+ @type ltsv
271
+ </parse>
272
+ ]
273
+ CONFIG_LTSV_WITH_TYPES = %[
274
+ key_name data
275
+ <parse>
276
+ @type ltsv
277
+ types i:integer,s:string,f:float,b:bool
278
+ </parse>
279
+ ]
280
+ data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
281
+ :int_time => lambda { |time| time.to_i })
282
+ def test_filter_ltsv(time_parse)
283
+ d = create_driver(CONFIG_LTSV)
284
+ time = time_parse.call(@default_time)
285
+ d.run(default_tag: @tag) do
286
+ d.feed(time, {'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'})
287
+ d.feed(time, {'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'})
288
+ end
289
+ filtered = d.filtered
290
+ assert_equal 2, filtered.length
291
+
292
+ first = filtered[0]
293
+ assert_equal_parsed_time time, first[0]
294
+ assert_nil first[1]['data']
295
+ assert_equal 'first', first[1]['xxx']
296
+ assert_equal 'second', first[1]['yyy']
297
+
298
+ second = filtered[1]
299
+ assert_equal_parsed_time time, second[0]
300
+ assert_nil first[1]['data']
301
+ assert_equal 'first', second[1]['xxx']
302
+ assert_equal 'second2', second[1]['yyy']
303
+
304
+ d = create_driver(CONFIG_LTSV + %[reserve_data yes])
305
+ time = @default_time.to_i
306
+ d.run(default_tag: @tag) do
307
+ d.feed(time, {'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'})
308
+ d.feed(time, {'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'})
309
+ end
310
+ filtered = d.filtered
311
+ assert_equal 2, filtered.length
312
+
313
+ first = filtered[0]
314
+ assert_equal_parsed_time time, first[0]
315
+ assert_equal "xxx:first\tyyy:second", first[1]['data']
316
+ assert_equal 'first', first[1]['xxx']
317
+ assert_equal 'second', first[1]['yyy']
318
+
319
+ second = filtered[1]
320
+ assert_equal_parsed_time time, second[0]
321
+ assert_equal "xxx:first\tyyy:second", first[1]['data']
322
+ assert_equal 'first', second[1]['xxx']
323
+ assert_equal 'second2', second[1]['yyy']
324
+
325
+ # convert types
326
+ #d = create_driver(CONFIG_LTSV + %[
327
+ d = create_driver(CONFIG_LTSV_WITH_TYPES)
328
+ time = @default_time.to_i
329
+ d.run do
330
+ d.feed(@tag, time, {'data' => "i:1\ts:2\tf:3\tb:true\tx:123"})
331
+ end
332
+ filtered = d.filtered
333
+ assert_equal 1, filtered.length
334
+
335
+ first = filtered[0]
336
+ assert_equal_parsed_time time, first[0]
337
+ assert_equal 1, first[1]['i']
338
+ assert_equal '2', first[1]['s']
339
+ assert_equal 3.0, first[1]['f']
340
+ assert_equal true, first[1]['b']
341
+ assert_equal '123', first[1]['x']
342
+ end
343
+
344
+ CONFIG_TSV = %[
345
+ key_name data
346
+ <parse>
347
+ @type tsv
348
+ keys key1,key2,key3
349
+ </parse>
350
+ ]
351
+ data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
352
+ :int_time => lambda { |time| time.to_i })
353
+ def test_filter_tsv(time_parse)
354
+ d = create_driver(CONFIG_TSV)
355
+ time = time_parse.call(@default_time)
356
+ d.run do
357
+ d.feed(@tag, time, {'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'})
358
+ end
359
+ filtered = d.filtered
360
+ assert_equal 1, filtered.length
361
+
362
+ first = filtered[0]
363
+ assert_equal_parsed_time time, first[0]
364
+ assert_nil first[1]['data']
365
+ assert_equal 'value1', first[1]['key1']
366
+ assert_equal 'value2', first[1]['key2']
367
+ assert_equal 'valueThree', first[1]['key3']
368
+ end
369
+
370
+ CONFIG_CSV = %[
371
+ key_name data
372
+ <parse>
373
+ @type csv
374
+ keys key1,key2,key3
375
+ </parse>
376
+ ]
377
+ CONFIG_CSV_COMPAT = %[
378
+ key_name data
379
+ format csv
380
+ keys key1,key2,key3
381
+ ]
382
+ data(new_conf: CONFIG_CSV,
383
+ compat_conf: CONFIG_CSV_COMPAT)
384
+ def test_filter_csv(conf)
385
+ d = create_driver(conf)
386
+ time = @default_time.to_i
387
+ d.run do
388
+ d.feed(@tag, time, {'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'})
389
+ end
390
+ filtered = d.filtered
391
+ assert_equal 1, filtered.length
392
+
393
+ first = filtered[0]
394
+ assert_equal time, first[0]
395
+ assert_nil first[1]['data']
396
+ assert_equal 'value1', first[1]['key1']
397
+ assert_equal 'value2', first[1]['key2']
398
+ assert_equal 'value"ThreeYes!', first[1]['key3']
399
+ end
400
+
401
+ def test_filter_with_nested_record
402
+ d = create_driver(%[
403
+ key_name $.data.log
404
+ <parse>
405
+ @type csv
406
+ keys key1,key2,key3
407
+ </parse>
408
+ ])
409
+ time = @default_time.to_i
410
+ d.run do
411
+ d.feed(@tag, time, {'data' => {'log' => 'value1,"value2","value""ThreeYes!"'}, 'xxx' => 'x', 'yyy' => 'y'})
412
+ end
413
+ filtered = d.filtered
414
+ assert_equal 1, filtered.length
415
+
416
+ first = filtered[0]
417
+ assert_equal time, first[0]
418
+ assert_nil first[1]['data']
419
+ assert_equal 'value1', first[1]['key1']
420
+ assert_equal 'value2', first[1]['key2']
421
+ assert_equal 'value"ThreeYes!', first[1]['key3']
422
+ end
423
+
424
+ CONFIG_HASH_VALUE_FIELD = %[
425
+ key_name data
426
+ hash_value_field parsed
427
+ <parse>
428
+ @type json
429
+ </parse>
430
+ ]
431
+ CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
432
+ key_name data
433
+ reserve_data yes
434
+ hash_value_field parsed
435
+ <parse>
436
+ @type json
437
+ </parse>
438
+ ]
439
+ CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
440
+ key_name data
441
+ hash_value_field parsed
442
+ inject_key_prefix data.
443
+ <parse>
444
+ @type json
445
+ </parse>
446
+ ]
447
+ data(:event_time => lambda { |time| Fluent::EventTime.from_time(time) },
448
+ :int_time => lambda { |time| time.to_i })
449
+ def test_filter_inject_hash_value_field(time_parse)
450
+ original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
451
+
452
+ d = create_driver(CONFIG_HASH_VALUE_FIELD)
453
+ time = time_parse.call(@default_time)
454
+ d.run do
455
+ d.feed(@tag, time, original)
456
+ end
457
+ filtered = d.filtered
458
+ assert_equal 1, filtered.length
459
+
460
+ first = filtered[0]
461
+ assert_equal_parsed_time time, first[0]
462
+
463
+ record = first[1]
464
+ assert_equal 1, record.keys.size
465
+ assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
466
+
467
+ d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA)
468
+ time = @default_time.to_i
469
+ d.run do
470
+ d.feed(@tag, time, original)
471
+ end
472
+ filtered = d.filtered
473
+ assert_equal 1, filtered.length
474
+
475
+ first = filtered[0]
476
+ assert_equal_parsed_time time, first[0]
477
+
478
+ record = first[1]
479
+ assert_equal 4, record.keys.size
480
+ assert_equal original['data'], record['data']
481
+ assert_equal original['xxx'], record['xxx']
482
+ assert_equal original['yyy'], record['yyy']
483
+ assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
484
+
485
+ d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX)
486
+ time = @default_time.to_i
487
+ d.run do
488
+ d.feed(@tag, time, original)
489
+ end
490
+ filtered = d.filtered
491
+ assert_equal 1, filtered.length
492
+
493
+ first = filtered[0]
494
+ assert_equal_parsed_time time, first[0]
495
+
496
+ record = first[1]
497
+ assert_equal 1, record.keys.size
498
+ assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
499
+ end
500
+
501
+ CONFIG_DONT_PARSE_TIME = %[
502
+ key_name data
503
+ reserve_time true
504
+ <parse>
505
+ @type json
506
+ keep_time_key true
507
+ </parse>
508
+ ]
509
+ CONFIG_DONT_PARSE_TIME_COMPAT = %[
510
+ key_name data
511
+ reserve_time true
512
+ format json
513
+ keep_time_key true
514
+ ]
515
+ data(new_conf: CONFIG_DONT_PARSE_TIME,
516
+ compat_conf: CONFIG_DONT_PARSE_TIME_COMPAT)
517
+ def test_time_should_be_reserved(conf)
518
+ t = Time.now.to_i
519
+ d = create_driver(conf)
520
+ d.run(default_tag: @tag) do
521
+ d.feed(t, {'data' => '{"time":1383190430, "f1":"v1"}'})
522
+ d.feed(t, {'data' => '{"time":"1383190430", "f1":"v1"}'})
523
+ d.feed(t, {'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'})
524
+ end
525
+ filtered = d.filtered
526
+ assert_equal 3, filtered.length
527
+
528
+ assert_equal 'v1', filtered[0][1]['f1']
529
+ assert_equal 1383190430, filtered[0][1]['time']
530
+ assert_equal t, filtered[0][0]
531
+
532
+ assert_equal 'v1', filtered[1][1]['f1']
533
+ assert_equal "1383190430", filtered[1][1]['time']
534
+ assert_equal t, filtered[1][0]
535
+
536
+ assert_equal 'v1', filtered[2][1]['f1']
537
+ assert_equal '2013-10-31 12:34:03 +0900', filtered[2][1]['time']
538
+ assert_equal t, filtered[2][0]
539
+ end
540
+
541
+ CONFIG_INVALID_TIME_VALUE = %[
542
+ key_name data
543
+ <parse>
544
+ @type json
545
+ </parse>
546
+ ] # 'time' is implicit @time_key
547
+ def test_filter_invalid_time_data
548
+ # should not raise errors
549
+ time = Time.now.to_i
550
+ d = create_driver(CONFIG_INVALID_TIME_VALUE)
551
+ assert_nothing_raised {
552
+ d.run(default_tag: @tag) do
553
+ d.feed(time, {'data' => '{"time":[], "f1":"v1"}'})
554
+ d.feed(time, {'data' => '{"time":"thisisnottime", "f1":"v1"}'})
555
+ end
556
+ }
557
+ filtered = d.filtered
558
+ assert_equal 1, filtered.length
559
+
560
+ assert_equal 0, filtered[0][0].to_i
561
+ assert_equal 'v1', filtered[0][1]['f1']
562
+ assert_equal nil, filtered[0][1]['time']
563
+ end
564
+
565
+ # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
566
+
567
+ CONFIG_NOT_REPLACE = %[
568
+ key_name data
569
+ <parse>
570
+ @type regexp
571
+ expression /^(?<message>.*)$/
572
+ </parse>
573
+ ]
574
+ CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
575
+ replace_invalid_sequence true
576
+ ]
577
+ def test_filter_invalid_byte
578
+ invalid_utf8 = "\xff".force_encoding('UTF-8')
579
+
580
+ d = create_driver(CONFIG_NOT_REPLACE)
581
+ d.run do
582
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
583
+ end
584
+ error_event = d.error_events.first
585
+ assert_equal "test", error_event[0]
586
+ assert_instance_of ArgumentError, error_event[3]
587
+
588
+ d = create_driver(CONFIG_INVALID_BYTE)
589
+ assert_nothing_raised {
590
+ d.run do
591
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
592
+ end
593
+ }
594
+ filtered = d.filtered
595
+ assert_equal 1, filtered.length
596
+ assert_nil filtered[0][1]['data']
597
+ assert_equal '?'.force_encoding('UTF-8'), filtered[0][1]['message']
598
+
599
+ d = create_driver(CONFIG_INVALID_BYTE + %[reserve_data yes])
600
+ assert_nothing_raised {
601
+ d.run do
602
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_utf8})
603
+ end
604
+ }
605
+ filtered = d.filtered
606
+ assert_equal 1, filtered.length
607
+ assert_equal invalid_utf8, filtered[0][1]['data']
608
+ assert_equal '?'.force_encoding('UTF-8'), filtered[0][1]['message']
609
+
610
+ invalid_ascii = "\xff".force_encoding('US-ASCII')
611
+ d = create_driver(CONFIG_INVALID_BYTE)
612
+ assert_nothing_raised {
613
+ d.run do
614
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => invalid_ascii})
615
+ end
616
+ }
617
+ filtered = d.filtered
618
+ assert_equal 1, filtered.length
619
+ assert_nil filtered[0][1]['data']
620
+ assert_equal '?'.force_encoding('US-ASCII'), filtered[0][1]['message']
621
+ end
622
+
623
+ CONFIG_NOT_IGNORE = %[
624
+ key_name data
625
+ hash_value_field parsed
626
+ <parse>
627
+ @type json
628
+ </parse>
629
+ ]
630
+ CONFIG_PASS_SAME_RECORD = CONFIG_NOT_IGNORE + %[
631
+ reserve_data true
632
+ ]
633
+ def test_filter_key_not_exist
634
+ d = create_driver(CONFIG_NOT_IGNORE)
635
+ flexmock(d.instance.router).should_receive(:emit_error_event).
636
+ with(String, Integer, Hash, ArgumentError.new("data does not exist")).once
637
+ assert_nothing_raised {
638
+ d.run do
639
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
640
+ end
641
+ }
642
+
643
+ d = create_driver(CONFIG_PASS_SAME_RECORD)
644
+ assert_nothing_raised {
645
+ d.run do
646
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
647
+ end
648
+ }
649
+ filtered = d.filtered
650
+ assert_equal 1, filtered.length
651
+ assert_nil filtered[0][1]['data']
652
+ assert_equal 'bar', filtered[0][1]['foo']
653
+ end
654
+
655
+ # emit_error_event test
656
+ INVALID_MESSAGE = 'foo bar'
657
+ VALID_MESSAGE = 'col1=foo col2=bar'
658
+
659
+ def test_call_emit_error_event_when_parser_error
660
+ d = create_driver(CONFIG_INVALID_TIME_VALUE)
661
+ flexmock(d.instance.router).should_receive(:emit_error_event).
662
+ with(String, Integer, Hash, ParserError).once
663
+ d.run do
664
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => '{"time":[], "f1":"v1"}'})
665
+ end
666
+ end
667
+
668
+ CONFIG_UNMATCHED_PATTERN_LOG = %[
669
+ key_name message
670
+ <parse>
671
+ @type regexp
672
+ expression /^col1=(?<col1>.+) col2=(?<col2>.+)$/
673
+ </parse>
674
+ ]
675
+
676
+ class UnmatchedPatternLogTest < self
677
+ setup do
678
+ @d = create_driver(CONFIG_UNMATCHED_PATTERN_LOG)
679
+ end
680
+
681
+ def test_call_emit_error_event_when_pattern_is_mismached
682
+ flexmock(@d.instance.router).should_receive(:emit_error_event).
683
+ with(String, Integer, Hash, ParserError.new("pattern not matched with data '#{INVALID_MESSAGE}'")).once
684
+ @d.run do
685
+ @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
686
+ end
687
+ end
688
+
689
+ def test_not_call_emit_error_event_when_pattern_is_mached
690
+ flexmock(@d.instance.router).should_receive(:emit_error_event).never
691
+ @d.run do
692
+ @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => VALID_MESSAGE})
693
+ end
694
+ end
695
+ end
696
+
697
+ class EmitInvalidRecordToErrorTest < self
698
+ def test_pattern_is_mismached_with_emit_invalid_record_to_error
699
+ d = create_driver(CONFIG_UNMATCHED_PATTERN_LOG + "emit_invalid_record_to_error false")
700
+ flexmock(d.instance.router).should_receive(:emit_error_event).never
701
+ assert_nothing_raised {
702
+ d.run do
703
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
704
+ end
705
+ }
706
+ assert_equal 0, d.filtered.length
707
+ end
708
+
709
+ def test_parser_error_with_emit_invalid_record_to_error
710
+ d = create_driver(CONFIG_INVALID_TIME_VALUE + "emit_invalid_record_to_error false")
711
+ flexmock(d.instance.router).should_receive(:emit_error_event).never
712
+ assert_nothing_raised {
713
+ d.run do
714
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => '{"time":[], "f1":"v1"}'})
715
+ end
716
+ }
717
+ assert_equal 0, d.filtered.length
718
+ end
719
+
720
+ def test_key_not_exist_with_emit_invalid_record_to_error
721
+ d = create_driver(CONFIG_NOT_IGNORE + "emit_invalid_record_to_error false")
722
+ flexmock(d.instance.router).should_receive(:emit_error_event).never
723
+ assert_nothing_raised {
724
+ d.run do
725
+ d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
726
+ end
727
+ }
728
+ assert_equal 0, d.filtered.length
729
+ end
730
+ end
731
+ end