fluentd 0.12.40 → 1.6.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (428) hide show
  1. checksums.yaml +5 -5
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  4. data/.github/ISSUE_TEMPLATE.md +17 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
  6. data/.gitignore +5 -0
  7. data/.gitlab/cicd-template.yaml +10 -0
  8. data/.gitlab-ci.yml +147 -0
  9. data/.travis.yml +56 -20
  10. data/ADOPTERS.md +5 -0
  11. data/CHANGELOG.md +1369 -0
  12. data/CONTRIBUTING.md +16 -5
  13. data/GOVERNANCE.md +55 -0
  14. data/Gemfile +5 -0
  15. data/GithubWorkflow.md +78 -0
  16. data/LICENSE +202 -0
  17. data/MAINTAINERS.md +7 -0
  18. data/README.md +23 -11
  19. data/Rakefile +48 -2
  20. data/Vagrantfile +17 -0
  21. data/appveyor.yml +37 -0
  22. data/bin/fluent-binlog-reader +7 -0
  23. data/bin/fluent-ca-generate +6 -0
  24. data/bin/fluent-plugin-config-format +5 -0
  25. data/bin/fluent-plugin-generate +5 -0
  26. data/bin/fluentd +3 -0
  27. data/code-of-conduct.md +3 -0
  28. data/example/copy_roundrobin.conf +39 -0
  29. data/example/counter.conf +18 -0
  30. data/example/in_dummy_blocks.conf +17 -0
  31. data/example/in_dummy_with_compression.conf +23 -0
  32. data/example/in_forward.conf +7 -0
  33. data/example/in_forward_client.conf +37 -0
  34. data/example/in_forward_shared_key.conf +15 -0
  35. data/example/in_forward_tls.conf +14 -0
  36. data/example/in_forward_users.conf +24 -0
  37. data/example/in_forward_workers.conf +21 -0
  38. data/example/in_http.conf +3 -1
  39. data/example/in_out_forward.conf +17 -0
  40. data/example/logevents.conf +25 -0
  41. data/example/multi_filters.conf +61 -0
  42. data/example/out_exec_filter.conf +42 -0
  43. data/example/out_forward.conf +13 -13
  44. data/example/out_forward_buf_file.conf +23 -0
  45. data/example/out_forward_client.conf +109 -0
  46. data/example/out_forward_heartbeat_none.conf +16 -0
  47. data/example/out_forward_shared_key.conf +36 -0
  48. data/example/out_forward_tls.conf +18 -0
  49. data/example/out_forward_users.conf +65 -0
  50. data/example/out_null.conf +36 -0
  51. data/example/secondary_file.conf +42 -0
  52. data/example/suppress_config_dump.conf +7 -0
  53. data/example/worker_section.conf +36 -0
  54. data/fluent.conf +29 -0
  55. data/fluentd.gemspec +21 -11
  56. data/lib/fluent/agent.rb +67 -90
  57. data/lib/fluent/clock.rb +62 -0
  58. data/lib/fluent/command/binlog_reader.rb +244 -0
  59. data/lib/fluent/command/ca_generate.rb +181 -0
  60. data/lib/fluent/command/cat.rb +42 -18
  61. data/lib/fluent/command/debug.rb +12 -10
  62. data/lib/fluent/command/fluentd.rb +153 -5
  63. data/lib/fluent/command/plugin_config_formatter.rb +292 -0
  64. data/lib/fluent/command/plugin_generator.rb +324 -0
  65. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  66. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  67. data/lib/fluent/compat/exec_util.rb +129 -0
  68. data/lib/fluent/compat/file_util.rb +54 -0
  69. data/lib/fluent/compat/filter.rb +68 -0
  70. data/lib/fluent/compat/formatter.rb +111 -0
  71. data/lib/fluent/compat/formatter_utils.rb +85 -0
  72. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  73. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  74. data/lib/fluent/compat/input.rb +49 -0
  75. data/lib/fluent/compat/output.rb +718 -0
  76. data/lib/fluent/compat/output_chain.rb +60 -0
  77. data/lib/fluent/compat/parser.rb +310 -0
  78. data/lib/fluent/compat/parser_utils.rb +40 -0
  79. data/lib/fluent/compat/propagate_default.rb +62 -0
  80. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  81. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  82. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  83. data/lib/fluent/compat/socket_util.rb +165 -0
  84. data/lib/fluent/compat/string_util.rb +34 -0
  85. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  86. data/lib/fluent/compat/type_converter.rb +90 -0
  87. data/lib/fluent/config/configure_proxy.rb +210 -62
  88. data/lib/fluent/config/dsl.rb +12 -5
  89. data/lib/fluent/config/element.rb +107 -9
  90. data/lib/fluent/config/literal_parser.rb +9 -3
  91. data/lib/fluent/config/parser.rb +4 -4
  92. data/lib/fluent/config/section.rb +51 -14
  93. data/lib/fluent/config/types.rb +28 -13
  94. data/lib/fluent/config/v1_parser.rb +3 -5
  95. data/lib/fluent/config.rb +23 -20
  96. data/lib/fluent/configurable.rb +79 -21
  97. data/lib/fluent/counter/base_socket.rb +46 -0
  98. data/lib/fluent/counter/client.rb +297 -0
  99. data/lib/fluent/counter/error.rb +86 -0
  100. data/lib/fluent/counter/mutex_hash.rb +163 -0
  101. data/lib/fluent/counter/server.rb +273 -0
  102. data/lib/fluent/counter/store.rb +205 -0
  103. data/lib/fluent/counter/validator.rb +145 -0
  104. data/lib/fluent/counter.rb +23 -0
  105. data/lib/fluent/daemon.rb +15 -0
  106. data/lib/fluent/engine.rb +102 -65
  107. data/lib/fluent/env.rb +7 -3
  108. data/lib/fluent/error.rb +30 -0
  109. data/lib/fluent/event.rb +197 -21
  110. data/lib/fluent/event_router.rb +93 -10
  111. data/lib/fluent/filter.rb +2 -50
  112. data/lib/fluent/formatter.rb +4 -293
  113. data/lib/fluent/input.rb +2 -32
  114. data/lib/fluent/label.rb +10 -2
  115. data/lib/fluent/load.rb +3 -3
  116. data/lib/fluent/log.rb +348 -81
  117. data/lib/fluent/match.rb +37 -36
  118. data/lib/fluent/mixin.rb +12 -176
  119. data/lib/fluent/msgpack_factory.rb +62 -0
  120. data/lib/fluent/output.rb +10 -612
  121. data/lib/fluent/output_chain.rb +23 -0
  122. data/lib/fluent/parser.rb +4 -800
  123. data/lib/fluent/plugin/bare_output.rb +63 -0
  124. data/lib/fluent/plugin/base.rb +192 -0
  125. data/lib/fluent/plugin/buf_file.rb +128 -174
  126. data/lib/fluent/plugin/buf_memory.rb +9 -92
  127. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  128. data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
  129. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  130. data/lib/fluent/plugin/buffer.rb +779 -0
  131. data/lib/fluent/plugin/compressable.rb +92 -0
  132. data/lib/fluent/plugin/exec_util.rb +3 -108
  133. data/lib/fluent/plugin/file_util.rb +4 -34
  134. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  135. data/lib/fluent/plugin/filter.rb +93 -0
  136. data/lib/fluent/plugin/filter_grep.rb +117 -34
  137. data/lib/fluent/plugin/filter_parser.rb +85 -62
  138. data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
  139. data/lib/fluent/plugin/filter_stdout.rb +15 -12
  140. data/lib/fluent/plugin/formatter.rb +50 -0
  141. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  142. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  143. data/lib/fluent/plugin/formatter_json.rb +55 -0
  144. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  145. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  146. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  147. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  148. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  149. data/lib/fluent/plugin/formatter_tsv.rb +38 -0
  150. data/lib/fluent/plugin/in_debug_agent.rb +17 -6
  151. data/lib/fluent/plugin/in_dummy.rb +47 -20
  152. data/lib/fluent/plugin/in_exec.rb +55 -123
  153. data/lib/fluent/plugin/in_forward.rb +299 -216
  154. data/lib/fluent/plugin/in_gc_stat.rb +14 -36
  155. data/lib/fluent/plugin/in_http.rb +204 -91
  156. data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
  157. data/lib/fluent/plugin/in_object_space.rb +13 -41
  158. data/lib/fluent/plugin/in_syslog.rb +112 -134
  159. data/lib/fluent/plugin/in_tail.rb +408 -745
  160. data/lib/fluent/plugin/in_tcp.rb +66 -9
  161. data/lib/fluent/plugin/in_udp.rb +60 -11
  162. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
  163. data/lib/fluent/plugin/input.rb +37 -0
  164. data/lib/fluent/plugin/multi_output.rb +158 -0
  165. data/lib/fluent/plugin/out_copy.rb +23 -35
  166. data/lib/fluent/plugin/out_exec.rb +67 -70
  167. data/lib/fluent/plugin/out_exec_filter.rb +204 -271
  168. data/lib/fluent/plugin/out_file.rb +267 -73
  169. data/lib/fluent/plugin/out_forward.rb +854 -325
  170. data/lib/fluent/plugin/out_null.rb +42 -9
  171. data/lib/fluent/plugin/out_relabel.rb +9 -5
  172. data/lib/fluent/plugin/out_roundrobin.rb +18 -37
  173. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  174. data/lib/fluent/plugin/out_stdout.rb +43 -10
  175. data/lib/fluent/plugin/out_stream.rb +7 -2
  176. data/lib/fluent/plugin/output.rb +1498 -0
  177. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  178. data/lib/fluent/plugin/parser.rb +191 -0
  179. data/lib/fluent/plugin/parser_apache.rb +28 -0
  180. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  181. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  182. data/lib/fluent/plugin/parser_csv.rb +39 -0
  183. data/lib/fluent/plugin/parser_json.rb +94 -0
  184. data/lib/fluent/plugin/parser_ltsv.rb +49 -0
  185. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  186. data/lib/fluent/plugin/parser_multiline.rb +106 -0
  187. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  188. data/lib/fluent/plugin/parser_none.rb +36 -0
  189. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  190. data/lib/fluent/plugin/parser_syslog.rb +142 -0
  191. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  192. data/lib/fluent/plugin/socket_util.rb +3 -143
  193. data/lib/fluent/plugin/storage.rb +84 -0
  194. data/lib/fluent/plugin/storage_local.rb +164 -0
  195. data/lib/fluent/plugin/string_util.rb +3 -15
  196. data/lib/fluent/plugin.rb +122 -121
  197. data/lib/fluent/plugin_helper/cert_option.rb +178 -0
  198. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  199. data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
  200. data/lib/fluent/plugin_helper/counter.rb +51 -0
  201. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  202. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  203. data/lib/fluent/plugin_helper/extract.rb +104 -0
  204. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  205. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  206. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  207. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  208. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  209. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  210. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  211. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  212. data/lib/fluent/plugin_helper/http_server.rb +76 -0
  213. data/lib/fluent/plugin_helper/inject.rb +151 -0
  214. data/lib/fluent/plugin_helper/parser.rb +147 -0
  215. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  216. data/lib/fluent/plugin_helper/retry_state.rb +205 -0
  217. data/lib/fluent/plugin_helper/server.rb +807 -0
  218. data/lib/fluent/plugin_helper/socket.rb +250 -0
  219. data/lib/fluent/plugin_helper/socket_option.rb +80 -0
  220. data/lib/fluent/plugin_helper/storage.rb +349 -0
  221. data/lib/fluent/plugin_helper/thread.rb +179 -0
  222. data/lib/fluent/plugin_helper/timer.rb +92 -0
  223. data/lib/fluent/plugin_helper.rb +73 -0
  224. data/lib/fluent/plugin_id.rb +80 -0
  225. data/lib/fluent/process.rb +3 -489
  226. data/lib/fluent/registry.rb +52 -10
  227. data/lib/fluent/root_agent.rb +204 -42
  228. data/lib/fluent/supervisor.rb +597 -359
  229. data/lib/fluent/system_config.rb +131 -42
  230. data/lib/fluent/test/base.rb +6 -54
  231. data/lib/fluent/test/driver/base.rb +224 -0
  232. data/lib/fluent/test/driver/base_owned.rb +70 -0
  233. data/lib/fluent/test/driver/base_owner.rb +135 -0
  234. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  235. data/lib/fluent/test/driver/filter.rb +57 -0
  236. data/lib/fluent/test/driver/formatter.rb +30 -0
  237. data/lib/fluent/test/driver/input.rb +31 -0
  238. data/lib/fluent/test/driver/multi_output.rb +53 -0
  239. data/lib/fluent/test/driver/output.rb +102 -0
  240. data/lib/fluent/test/driver/parser.rb +30 -0
  241. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  242. data/lib/fluent/test/filter_test.rb +0 -1
  243. data/lib/fluent/test/formatter_test.rb +4 -1
  244. data/lib/fluent/test/helpers.rb +58 -10
  245. data/lib/fluent/test/input_test.rb +27 -19
  246. data/lib/fluent/test/log.rb +79 -0
  247. data/lib/fluent/test/output_test.rb +28 -39
  248. data/lib/fluent/test/parser_test.rb +3 -1
  249. data/lib/fluent/test/startup_shutdown.rb +46 -0
  250. data/lib/fluent/test.rb +33 -1
  251. data/lib/fluent/time.rb +450 -1
  252. data/lib/fluent/timezone.rb +27 -3
  253. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  254. data/lib/fluent/version.rb +1 -1
  255. data/lib/fluent/winsvc.rb +85 -0
  256. data/templates/new_gem/Gemfile +3 -0
  257. data/templates/new_gem/README.md.erb +43 -0
  258. data/templates/new_gem/Rakefile +13 -0
  259. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  260. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  261. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  262. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  263. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  264. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  265. data/templates/new_gem/test/helper.rb.erb +8 -0
  266. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  267. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  268. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  269. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  270. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  271. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  272. data/templates/plugin_config_formatter/param.md.erb +34 -0
  273. data/templates/plugin_config_formatter/section.md.erb +12 -0
  274. data/test/command/test_binlog_reader.rb +346 -0
  275. data/test/command/test_ca_generate.rb +70 -0
  276. data/test/command/test_fluentd.rb +901 -0
  277. data/test/command/test_plugin_config_formatter.rb +276 -0
  278. data/test/command/test_plugin_generator.rb +92 -0
  279. data/test/compat/test_calls_super.rb +166 -0
  280. data/test/compat/test_parser.rb +92 -0
  281. data/test/config/test_config_parser.rb +126 -2
  282. data/test/config/test_configurable.rb +946 -187
  283. data/test/config/test_configure_proxy.rb +424 -74
  284. data/test/config/test_dsl.rb +11 -11
  285. data/test/config/test_element.rb +500 -0
  286. data/test/config/test_literal_parser.rb +8 -0
  287. data/test/config/test_plugin_configuration.rb +56 -0
  288. data/test/config/test_section.rb +79 -7
  289. data/test/config/test_system_config.rb +122 -35
  290. data/test/config/test_types.rb +38 -0
  291. data/test/counter/test_client.rb +559 -0
  292. data/test/counter/test_error.rb +44 -0
  293. data/test/counter/test_mutex_hash.rb +179 -0
  294. data/test/counter/test_server.rb +589 -0
  295. data/test/counter/test_store.rb +258 -0
  296. data/test/counter/test_validator.rb +137 -0
  297. data/test/helper.rb +89 -6
  298. data/test/helpers/fuzzy_assert.rb +89 -0
  299. data/test/plugin/test_bare_output.rb +118 -0
  300. data/test/plugin/test_base.rb +115 -0
  301. data/test/plugin/test_buf_file.rb +823 -460
  302. data/test/plugin/test_buf_memory.rb +32 -194
  303. data/test/plugin/test_buffer.rb +1233 -0
  304. data/test/plugin/test_buffer_chunk.rb +198 -0
  305. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  306. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  307. data/test/plugin/test_compressable.rb +84 -0
  308. data/test/plugin/test_filter.rb +357 -0
  309. data/test/plugin/test_filter_grep.rb +540 -29
  310. data/test/plugin/test_filter_parser.rb +439 -452
  311. data/test/plugin/test_filter_record_transformer.rb +123 -166
  312. data/test/plugin/test_filter_stdout.rb +160 -72
  313. data/test/plugin/test_formatter_csv.rb +111 -0
  314. data/test/plugin/test_formatter_hash.rb +35 -0
  315. data/test/plugin/test_formatter_json.rb +51 -0
  316. data/test/plugin/test_formatter_ltsv.rb +62 -0
  317. data/test/plugin/test_formatter_msgpack.rb +28 -0
  318. data/test/plugin/test_formatter_out_file.rb +95 -0
  319. data/test/plugin/test_formatter_single_value.rb +38 -0
  320. data/test/plugin/test_formatter_tsv.rb +68 -0
  321. data/test/plugin/test_in_debug_agent.rb +24 -1
  322. data/test/plugin/test_in_dummy.rb +111 -18
  323. data/test/plugin/test_in_exec.rb +200 -113
  324. data/test/plugin/test_in_forward.rb +990 -387
  325. data/test/plugin/test_in_gc_stat.rb +10 -8
  326. data/test/plugin/test_in_http.rb +600 -224
  327. data/test/plugin/test_in_monitor_agent.rb +690 -0
  328. data/test/plugin/test_in_object_space.rb +24 -8
  329. data/test/plugin/test_in_syslog.rb +154 -215
  330. data/test/plugin/test_in_tail.rb +1006 -707
  331. data/test/plugin/test_in_tcp.rb +125 -48
  332. data/test/plugin/test_in_udp.rb +204 -63
  333. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
  334. data/test/plugin/test_input.rb +126 -0
  335. data/test/plugin/test_metadata.rb +89 -0
  336. data/test/plugin/test_multi_output.rb +180 -0
  337. data/test/plugin/test_out_copy.rb +117 -112
  338. data/test/plugin/test_out_exec.rb +258 -53
  339. data/test/plugin/test_out_exec_filter.rb +538 -115
  340. data/test/plugin/test_out_file.rb +865 -178
  341. data/test/plugin/test_out_forward.rb +998 -210
  342. data/test/plugin/test_out_null.rb +105 -0
  343. data/test/plugin/test_out_relabel.rb +28 -0
  344. data/test/plugin/test_out_roundrobin.rb +36 -29
  345. data/test/plugin/test_out_secondary_file.rb +458 -0
  346. data/test/plugin/test_out_stdout.rb +135 -37
  347. data/test/plugin/test_out_stream.rb +18 -0
  348. data/test/plugin/test_output.rb +984 -0
  349. data/test/plugin/test_output_as_buffered.rb +2021 -0
  350. data/test/plugin/test_output_as_buffered_backup.rb +312 -0
  351. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  352. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  353. data/test/plugin/test_output_as_buffered_retries.rb +911 -0
  354. data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
  355. data/test/plugin/test_output_as_standard.rb +374 -0
  356. data/test/plugin/test_owned_by.rb +35 -0
  357. data/test/plugin/test_parser.rb +359 -0
  358. data/test/plugin/test_parser_apache.rb +42 -0
  359. data/test/plugin/test_parser_apache2.rb +47 -0
  360. data/test/plugin/test_parser_apache_error.rb +45 -0
  361. data/test/plugin/test_parser_csv.rb +103 -0
  362. data/test/plugin/test_parser_json.rb +138 -0
  363. data/test/plugin/test_parser_labeled_tsv.rb +145 -0
  364. data/test/plugin/test_parser_multiline.rb +100 -0
  365. data/test/plugin/test_parser_nginx.rb +88 -0
  366. data/test/plugin/test_parser_none.rb +52 -0
  367. data/test/plugin/test_parser_regexp.rb +289 -0
  368. data/test/plugin/test_parser_syslog.rb +441 -0
  369. data/test/plugin/test_parser_tsv.rb +122 -0
  370. data/test/plugin/test_storage.rb +167 -0
  371. data/test/plugin/test_storage_local.rb +335 -0
  372. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  373. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  374. data/test/plugin_helper/data/cert/cert.pem +19 -0
  375. data/test/plugin_helper/http_server/test_app.rb +65 -0
  376. data/test/plugin_helper/http_server/test_route.rb +32 -0
  377. data/test/plugin_helper/test_cert_option.rb +16 -0
  378. data/test/plugin_helper/test_child_process.rb +794 -0
  379. data/test/plugin_helper/test_compat_parameters.rb +353 -0
  380. data/test/plugin_helper/test_event_emitter.rb +51 -0
  381. data/test/plugin_helper/test_event_loop.rb +52 -0
  382. data/test/plugin_helper/test_extract.rb +194 -0
  383. data/test/plugin_helper/test_formatter.rb +255 -0
  384. data/test/plugin_helper/test_http_server_helper.rb +205 -0
  385. data/test/plugin_helper/test_inject.rb +519 -0
  386. data/test/plugin_helper/test_parser.rb +264 -0
  387. data/test/plugin_helper/test_record_accessor.rb +197 -0
  388. data/test/plugin_helper/test_retry_state.rb +442 -0
  389. data/test/plugin_helper/test_server.rb +1714 -0
  390. data/test/plugin_helper/test_storage.rb +542 -0
  391. data/test/plugin_helper/test_thread.rb +164 -0
  392. data/test/plugin_helper/test_timer.rb +132 -0
  393. data/test/scripts/exec_script.rb +0 -6
  394. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  395. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  396. data/test/scripts/fluent/plugin/out_test.rb +23 -15
  397. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  398. data/test/test_clock.rb +164 -0
  399. data/test/test_config.rb +16 -7
  400. data/test/test_configdsl.rb +2 -2
  401. data/test/test_event.rb +360 -13
  402. data/test/test_event_router.rb +108 -11
  403. data/test/test_event_time.rb +199 -0
  404. data/test/test_filter.rb +48 -6
  405. data/test/test_formatter.rb +11 -391
  406. data/test/test_input.rb +1 -1
  407. data/test/test_log.rb +591 -31
  408. data/test/test_mixin.rb +1 -1
  409. data/test/test_output.rb +121 -185
  410. data/test/test_plugin.rb +251 -0
  411. data/test/test_plugin_classes.rb +177 -10
  412. data/test/test_plugin_helper.rb +81 -0
  413. data/test/test_plugin_id.rb +101 -0
  414. data/test/test_process.rb +8 -42
  415. data/test/test_root_agent.rb +766 -21
  416. data/test/test_supervisor.rb +481 -0
  417. data/test/test_test_drivers.rb +135 -0
  418. data/test/test_time_formatter.rb +282 -0
  419. data/test/test_time_parser.rb +231 -0
  420. data/test/test_unique_id.rb +47 -0
  421. metadata +454 -60
  422. data/COPYING +0 -14
  423. data/ChangeLog +0 -666
  424. data/lib/fluent/buffer.rb +0 -365
  425. data/lib/fluent/plugin/in_status.rb +0 -76
  426. data/test/plugin/test_in_status.rb +0 -38
  427. data/test/test_buffer.rb +0 -624
  428. data/test/test_parser.rb +0 -1305
data/test/test_parser.rb DELETED
@@ -1,1305 +0,0 @@
1
- require_relative 'helper'
2
- require 'fluent/test'
3
- require 'fluent/parser'
4
- require 'fluent/plugin/string_util'
5
-
6
- module ParserTest
7
- include Fluent
8
-
9
- class BaseParserTest < ::Test::Unit::TestCase
10
- include ParserTest
11
-
12
- def create_parser
13
- parser = Parser.new
14
- parser.configure({})
15
- parser
16
- end
17
-
18
- def test_init
19
- assert_true create_parser.estimate_current_event
20
- end
21
-
22
- def test_parse
23
- assert_raise NotImplementedError do
24
- create_parser.parse('')
25
- end
26
- end
27
-
28
- def test_call
29
- assert_raise NotImplementedError do
30
- create_parser.call('')
31
- end
32
- end
33
- end
34
-
35
- class BaseParserTestWithTestDriver < ::Test::Unit::TestCase
36
- include ParserTest
37
-
38
- def create_driver(conf={})
39
- Fluent::Test::ParserTestDriver.new(Fluent::Parser).configure(conf)
40
- end
41
-
42
- def test_init
43
- d = create_driver
44
- assert_true d.instance.estimate_current_event
45
- end
46
-
47
- def test_configure_against_string_literal
48
- d = create_driver('keep_time_key true')
49
- assert_true d.instance.keep_time_key
50
- end
51
-
52
- def test_parse
53
- d = create_driver
54
- assert_raise NotImplementedError do
55
- d.parse('')
56
- end
57
- end
58
- end
59
-
60
- class TimeParserTest < ::Test::Unit::TestCase
61
- include ParserTest
62
-
63
- def test_call_with_parse
64
- parser = TextParser::TimeParser.new(nil)
65
-
66
- time = event_time('2013-09-18 12:00:00 +0900')
67
- assert_equal_event_time(time, parser.parse('2013-09-18 12:00:00 +0900'))
68
- end
69
-
70
- def test_parse_with_strptime
71
- parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
72
-
73
- time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
74
- assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
75
- end
76
-
77
- data('with sec' => '2017-01-01T12:00:00+09:00',
78
- 'with msec' => '2017-01-01T12:00:00.123+09:00')
79
- def test_parse_with_iso8601(data)
80
- parser = TextParser::TimeParser.new('%iso8601')
81
-
82
- time = event_time(data)
83
- assert_equal_event_time(time, parser.parse(data))
84
- end
85
-
86
- def test_parse_with_invalid_argument
87
- parser = TextParser::TimeParser.new(nil)
88
-
89
- [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
90
- assert_raise Fluent::ParserError do
91
- parser.parse(v)
92
- end
93
- }
94
- end
95
- end
96
-
97
- class RegexpParserTest < ::Test::Unit::TestCase
98
- include ParserTest
99
-
100
- def internal_test_case(parser)
101
- text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
102
- [parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
103
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
104
- assert_equal({
105
- 'user' => '-',
106
- 'flag' => true,
107
- 'code' => 200.0,
108
- 'size' => 777,
109
- 'date' => event_time('14/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'),
110
- 'host' => '192.168.0.1',
111
- 'path' => ['/', '/user']
112
- }, record)
113
- }
114
- end
115
-
116
- def test_parse_with_typed
117
- # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
118
- internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
119
- end
120
-
121
- def test_parse_with_configure
122
- # Specify conf by configure method instaed of intializer
123
- regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
124
- parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, regexp)
125
- parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
126
- internal_test_case(parser)
127
- assert_equal(regexp, parser.instance.patterns['format'])
128
- assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.instance.patterns['time_format'])
129
- end
130
-
131
- def test_parse_with_typed_and_name_separator
132
- internal_test_case(Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
133
- end
134
-
135
- def test_parse_with_time_key
136
- parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, /(?<logtime>[^\]]*)/)
137
- parser.configure(
138
- 'time_format'=>"%Y-%m-%d %H:%M:%S %z",
139
- 'time_key'=>'logtime',
140
- )
141
- text = '2013-02-28 12:00:00 +0900'
142
- parser.parse(text) do |time, record|
143
- assert_equal event_time(text), time
144
- end
145
- end
146
-
147
- def test_parse_without_time
148
- time_at_start = event_time()
149
- text = "tagomori_satoshi tagomoris 34\n"
150
-
151
- parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
152
- parser.configure('types'=>'name:string,user:string,age:bool')
153
-
154
- [parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
155
- assert time && time >= time_at_start, "parser puts current time without time input"
156
- assert_equal "tagomori_satoshi", record["name"]
157
- assert_equal "tagomoris", record["user"]
158
- assert_equal 34, record["age"]
159
- }
160
-
161
- parser2 = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
162
- parser2.configure('types'=>'name:string,user:string,age:integer')
163
- parser2.instance.estimate_current_event = false
164
- [parser2.parse(text), parser2.parse(text) { |time, record| return time, record}].each { |time, record|
165
- assert_equal "tagomori_satoshi", record["name"]
166
- assert_equal "tagomoris", record["user"]
167
- assert_equal 34, record["age"]
168
-
169
- assert_nil time, "parser returns nil if configured so"
170
- }
171
- end
172
-
173
- def test_parse_with_keep_time_key
174
- parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
175
- Regexp.new(%q!(?<time>.*)!),
176
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
177
- 'keep_time_key'=>'true',
178
- )
179
- text = '28/Feb/2013:12:00:00 +0900'
180
- parser.parse(text) do |time, record|
181
- assert_equal text, record['time']
182
- end
183
- end
184
-
185
- def test_parse_with_keep_time_key_with_typecast
186
- parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
187
- Regexp.new(%q!(?<time>.*)!),
188
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
189
- 'keep_time_key'=>'true',
190
- 'types'=>'time:time:%d/%b/%Y:%H:%M:%S %z',
191
- )
192
- text = '28/Feb/2013:12:00:00 +0900'
193
- parser.parse(text) do |time, record|
194
- assert_equal 1362020400, record['time']
195
- end
196
- end
197
- end
198
-
199
- class ApacheParserTest < ::Test::Unit::TestCase
200
- include ParserTest
201
-
202
- def setup
203
- @parser = TextParser::TEMPLATE_REGISTRY.lookup('apache').call
204
- end
205
-
206
- data('parse' => :parse, 'call' => :call)
207
- def test_call(method_name)
208
- m = @parser.method(method_name)
209
- m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
210
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
211
- assert_equal({
212
- 'user' => '-',
213
- 'method' => 'GET',
214
- 'code' => '200',
215
- 'size' => '777',
216
- 'host' => '192.168.0.1',
217
- 'path' => '/'
218
- }, record)
219
- }
220
- end
221
-
222
- def test_parse_with_keep_time_key
223
- parser = TextParser::ApacheParser.new
224
- parser.configure(
225
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
226
- 'keep_time_key'=>'true',
227
- )
228
- text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
229
- parser.parse(text) do |time, record|
230
- assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
231
- end
232
- end
233
- end
234
-
235
- class ApacheErrorParserTest < ::Test::Unit::TestCase
236
- include ParserTest
237
-
238
- def setup
239
- @parser = TextParser::TEMPLATE_REGISTRY.lookup('apache_error').call
240
- @expected = {
241
- 'level' => 'error',
242
- 'client' => '127.0.0.1',
243
- 'message' => 'client denied by server configuration'
244
- }
245
- end
246
-
247
- def test_parse
248
- @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
249
- assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
250
- assert_equal(@expected, record)
251
- }
252
- end
253
-
254
- def test_parse_with_pid
255
- @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
256
- assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
257
- assert_equal(@expected.merge('pid' => '1000'), record)
258
- }
259
- end
260
-
261
- def test_parse_without_client
262
- @parser.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
263
- assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
264
- assert_equal({
265
- 'level' => 'notice',
266
- 'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
267
- }, record)
268
- }
269
- end
270
- end
271
-
272
- class Apache2ParserTest < ::Test::Unit::TestCase
273
- include ParserTest
274
-
275
- def setup
276
- @parser = TextParser::ApacheParser.new
277
- @expected = {
278
- 'user' => nil,
279
- 'method' => 'GET',
280
- 'code' => 200,
281
- 'size' => 777,
282
- 'host' => '192.168.0.1',
283
- 'path' => '/',
284
- 'referer' => nil,
285
- 'agent' => 'Opera/12.0'
286
- }
287
- end
288
-
289
- def test_parse
290
- @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
291
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
292
- assert_equal(@expected, record)
293
- }
294
- assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
295
- assert_equal(TextParser::ApacheParser::TIME_FORMAT, @parser.patterns['time_format'])
296
- end
297
-
298
- def test_parse_without_http_version
299
- @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
300
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
301
- assert_equal(@expected, record)
302
- }
303
- end
304
-
305
- def test_parse_with_escape_sequence
306
- @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
307
- assert_equal('/\"', record['path'])
308
- assert_equal('referer \\\ \"', record['referer'])
309
- assert_equal('user agent \\\ \"', record['agent'])
310
- }
311
- end
312
- end
313
-
314
- class SyslogParserTest < ::Test::Unit::TestCase
315
- include ParserTest
316
-
317
- def setup
318
- @parser = Fluent::Test::ParserTestDriver.new('syslog')
319
- @expected = {
320
- 'host' => '192.168.0.1',
321
- 'ident' => 'fluentd',
322
- 'pid' => '11111',
323
- 'message' => '[error] Syslog test'
324
- }
325
- end
326
-
327
- def test_parse
328
- @parser.configure({})
329
- @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
330
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
331
- assert_equal(@expected, record)
332
- }
333
- assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
334
- assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
335
- end
336
-
337
- def test_parse_with_time_format
338
- @parser.configure('time_format' => '%b %d %M:%S:%H')
339
- @parser.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
340
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
341
- assert_equal(@expected, record)
342
- }
343
- assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
344
- end
345
-
346
- def test_parse_with_priority
347
- @parser.configure('with_priority' => true)
348
- @parser.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
349
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
350
- assert_equal(@expected.merge('pri' => 6), record)
351
- }
352
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
353
- assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
354
- end
355
-
356
- def test_parse_without_colon
357
- @parser.configure({})
358
- @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
359
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
360
- assert_equal(@expected, record)
361
- }
362
- assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
363
- assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
364
- end
365
-
366
- def test_parse_with_keep_time_key
367
- @parser.configure(
368
- 'time_format' => '%b %d %M:%S:%H',
369
- 'keep_time_key'=>'true',
370
- )
371
- text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
372
- @parser.parse(text) do |time, record|
373
- assert_equal "Feb 28 00:00:12", record['time']
374
- end
375
- end
376
-
377
- def test_parse_various_characters_for_tag
378
- ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
379
- @parser.configure({})
380
- @parser.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
381
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
382
- assert_equal(@expected.merge('ident' => ident), record)
383
- }
384
- end
385
-
386
- def test_parse_various_characters_for_tag_with_priority
387
- ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
388
- @parser.configure({'with_priority' => true})
389
- @parser.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
390
- assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
391
- assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
392
- }
393
- end
394
-
395
- class TestRFC5424Regexp < self
396
- def test_parse_with_rfc5424_message
397
- @parser.configure(
398
- 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
399
- 'message_format' => 'rfc5424',
400
- 'with_priority' => true,
401
- )
402
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
403
- @parser.parse(text) do |time, record|
404
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
405
- assert_equal "-", record["pid"]
406
- assert_equal "-", record["msgid"]
407
- assert_equal "-", record["extradata"]
408
- assert_equal "Hi, from Fluentd!", record["message"]
409
- end
410
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
411
- @parser.instance.patterns['format'])
412
-
413
- end
414
-
415
- def test_parse_with_rfc5424_message_and_without_priority
416
- @parser.configure(
417
- 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
418
- 'message_format' => 'rfc5424',
419
- )
420
- text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
421
- @parser.instance.parse(text) do |time, record|
422
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
423
- assert_equal "-", record["pid"]
424
- assert_equal "-", record["msgid"]
425
- assert_equal "-", record["extradata"]
426
- assert_equal "Hi, from Fluentd!", record["message"]
427
- end
428
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424,
429
- @parser.instance.patterns['format'])
430
- end
431
-
432
- def test_parse_with_rfc5424_message_without_time_format
433
- @parser.configure(
434
- 'message_format' => 'rfc5424',
435
- 'with_priority' => true,
436
- )
437
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
438
- @parser.instance.parse(text) do |time, record|
439
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
440
- assert_equal "-", record["pid"]
441
- assert_equal "-", record["msgid"]
442
- assert_equal "-", record["extradata"]
443
- assert_equal "Hi, from Fluentd!", record["message"]
444
- end
445
- end
446
-
447
- def test_parse_with_rfc5424_structured_message
448
- @parser.configure(
449
- 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
450
- 'message_format' => 'rfc5424',
451
- 'with_priority' => true,
452
- )
453
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
454
- @parser.parse(text) do |time, record|
455
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
456
- assert_equal "11111", record["pid"]
457
- assert_equal "ID24224", record["msgid"]
458
- assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
459
- record["extradata"]
460
- assert_equal "Hi, from Fluentd!", record["message"]
461
- end
462
- end
463
- end
464
-
465
- class TestAutoRegexp < self
466
- def test_auto_with_legacy_syslog_message
467
- @parser.configure(
468
- 'time_format' => '%b %d %M:%S:%H',
469
- 'mseeage_format' => 'auto',
470
- )
471
- text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
472
- @parser.parse(text) do |time, record|
473
- assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
474
- assert_equal(@expected, record)
475
- end
476
- end
477
-
478
- def test_auto_with_legacy_syslog_priority_message
479
- @parser.configure(
480
- 'time_format' => '%b %d %M:%S:%H',
481
- 'with_priority' => true,
482
- 'mseeage_format' => 'auto',
483
- )
484
- text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
485
- @parser.parse(text) do |time, record|
486
- assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
487
- assert_equal(@expected.merge('pri' => 6), record)
488
- end
489
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
490
- @parser.instance.patterns['format'])
491
- end
492
-
493
- def test_parse_with_rfc5424_message
494
- @parser.configure(
495
- 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
496
- 'message_format' => 'auto',
497
- 'with_priority' => true,
498
- )
499
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
500
- @parser.parse(text) do |time, record|
501
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
502
- assert_equal "-", record["pid"]
503
- assert_equal "-", record["msgid"]
504
- assert_equal "-", record["extradata"]
505
- assert_equal "Hi, from Fluentd!", record["message"]
506
- end
507
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
508
- @parser.instance.patterns['format'])
509
- end
510
-
511
- def test_parse_with_rfc5424_structured_message
512
- @parser.configure(
513
- 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
514
- 'message_format' => 'auto',
515
- 'with_priority' => true,
516
- )
517
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
518
- @parser.parse(text) do |time, record|
519
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
520
- assert_equal "11111", record["pid"]
521
- assert_equal "ID24224", record["msgid"]
522
- assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
523
- record["extradata"]
524
- assert_equal "Hi, from Fluentd!", record["message"]
525
- end
526
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
527
- @parser.instance.patterns['format'])
528
- end
529
-
530
- def test_parse_with_both_message_type
531
- @parser.configure(
532
- 'time_format' => '%b %d %M:%S:%H',
533
- 'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
534
- 'message_format' => 'auto',
535
- 'with_priority' => true,
536
- )
537
- text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
538
- @parser.parse(text) do |time, record|
539
- assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
540
- assert_equal(@expected.merge('pri' => 1), record)
541
- end
542
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
543
- @parser.instance.patterns['format'])
544
-
545
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
546
- @parser.parse(text) do |time, record|
547
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
548
- assert_equal "11111", record["pid"]
549
- assert_equal "ID24224", record["msgid"]
550
- assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
551
- record["extradata"]
552
- assert_equal "Hi, from Fluentd!", record["message"]
553
- end
554
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
555
- @parser.instance.patterns['format'])
556
-
557
- text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
558
- @parser.parse(text) do |time, record|
559
- assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
560
- assert_equal(@expected.merge('pri' => 1), record)
561
- end
562
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
563
- @parser.instance.patterns['format'])
564
-
565
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
566
- @parser.parse(text) do |time, record|
567
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
568
- assert_equal "-", record["pid"]
569
- assert_equal "-", record["msgid"]
570
- assert_equal "-", record["extradata"]
571
- assert_equal "Hi, from Fluentd!", record["message"]
572
- end
573
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
574
- @parser.instance.patterns['format'])
575
-
576
- end
577
-
578
- def test_parse_with_both_message_type_and_priority
579
- @parser.configure(
580
- 'time_format' => '%b %d %M:%S:%H',
581
- 'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
582
- 'with_priority' => true,
583
- 'message_format' => 'auto',
584
- )
585
- text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
586
- @parser.parse(text) do |time, record|
587
- assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
588
- assert_equal(@expected.merge('pri' => 6), record)
589
- end
590
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
591
- @parser.instance.patterns['format'])
592
-
593
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
594
- @parser.parse(text) do |time, record|
595
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
596
- assert_equal "11111", record["pid"]
597
- assert_equal "ID24224", record["msgid"]
598
- assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
599
- record["extradata"]
600
- assert_equal "Hi, from Fluentd!", record["message"]
601
- end
602
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
603
- @parser.instance.patterns['format'])
604
-
605
- text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
606
- @parser.parse(text) do |time, record|
607
- assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
608
- assert_equal(@expected.merge('pri' => 16), record)
609
- end
610
- assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
611
- @parser.instance.patterns['format'])
612
-
613
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
614
- @parser.parse(text) do |time, record|
615
- assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
616
- assert_equal "-", record["pid"]
617
- assert_equal "-", record["msgid"]
618
- assert_equal "-", record["extradata"]
619
- assert_equal "Hi, from Fluentd!", record["message"]
620
- end
621
- assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
622
- @parser.instance.patterns['format'])
623
-
624
- end
625
- end
626
- end
627
-
628
- class JsonParserTest < ::Test::Unit::TestCase
629
- include ParserTest
630
-
631
- def setup
632
- @parser = TextParser::JSONParser.new
633
- end
634
-
635
- data('oj' => 'oj', 'yajl' => 'yajl')
636
- def test_parse(data)
637
- @parser.configure('json_parser' => data)
638
- @parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
639
- assert_equal_event_time(event_time('2013-02-28 12:00:00 +0900'), time)
640
- assert_equal({
641
- 'host' => '192.168.0.1',
642
- 'size' => 777,
643
- 'method' => 'PUT',
644
- }, record)
645
- }
646
- end
647
-
648
- data('oj' => 'oj', 'yajl' => 'yajl')
649
- def test_parse_with_large_float(data)
650
- @parser.configure('json_parser' => data)
651
- @parser.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
652
- assert_equal(Float, record['num'].class)
653
- }
654
- end
655
-
656
- data('oj' => 'oj', 'yajl' => 'yajl')
657
- def test_parse_without_time(data)
658
- time_at_start = Time.now.to_i
659
-
660
- @parser.configure('json_parser' => data)
661
- @parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
662
- assert time && time >= time_at_start, "parser puts current time without time input"
663
- assert_equal({
664
- 'host' => '192.168.0.1',
665
- 'size' => 777,
666
- 'method' => 'PUT',
667
- }, record)
668
- }
669
-
670
- parser = TextParser::JSONParser.new
671
- parser.estimate_current_event = false
672
- parser.configure('json_parser' => data)
673
- parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
674
- assert_equal({
675
- 'host' => '192.168.0.1',
676
- 'size' => 777,
677
- 'method' => 'PUT',
678
- }, record)
679
- assert_nil time, "parser return nil w/o time and if specified so"
680
- }
681
- end
682
-
683
- data('oj' => 'oj', 'yajl' => 'yajl')
684
- def test_parse_with_colon_string(data)
685
- @parser.configure('json_parser' => data)
686
- @parser.parse('{"time":1362020400,"log":":message"}') { |time, record|
687
- assert_equal(record['log'], ':message')
688
- }
689
- end
690
-
691
- data('oj' => 'oj', 'yajl' => 'yajl')
692
- def test_parse_with_invalid_time(data)
693
- @parser.configure('json_parser' => data)
694
- assert_raise Fluent::ParserError do
695
- @parser.parse('{"time":[],"k":"v"}') { |time, record| }
696
- end
697
- end
698
-
699
- data('oj' => 'oj', 'yajl' => 'yajl')
700
- def test_parse_with_keep_time_key(data)
701
- parser = TextParser::JSONParser.new
702
- parser.configure(
703
- 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
704
- 'keep_time_key' => 'true',
705
- 'json_parser' => data
706
- )
707
- text = "28/Feb/2013:12:00:00 +0900"
708
- parser.parse("{\"time\":\"#{text}\"}") do |time, record|
709
- assert_equal text, record['time']
710
- end
711
- end
712
- end
713
-
714
- class NginxParserTest < ::Test::Unit::TestCase
715
- include ParserTest
716
-
717
- def setup
718
- @parser = TextParser::TEMPLATE_REGISTRY.lookup('nginx').call
719
- @expected = {
720
- 'remote' => '127.0.0.1',
721
- 'host' => '192.168.0.1',
722
- 'user' => '-',
723
- 'method' => 'GET',
724
- 'path' => '/',
725
- 'code' => '200',
726
- 'size' => '777',
727
- 'referer' => '-',
728
- 'agent' => 'Opera/12.0'
729
- }
730
- end
731
-
732
- def test_parse
733
- @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
734
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
735
- assert_equal(@expected, record)
736
- }
737
- end
738
-
739
- def test_parse_with_empty_included_path
740
- @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
741
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
742
- assert_equal(@expected.merge('path' => '/a[ ]b'), record)
743
- }
744
- end
745
-
746
- def test_parse_without_http_version
747
- @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
748
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
749
- assert_equal(@expected, record)
750
- }
751
- end
752
- end
753
-
754
- class TSVParserTest < ::Test::Unit::TestCase
755
- include ParserTest
756
-
757
- data('array param' => '["a","b"]', 'string param' => 'a,b')
758
- def test_config_params(param)
759
- parser = TextParser::TSVParser.new
760
-
761
- assert_equal "\t", parser.delimiter
762
-
763
- parser.configure(
764
- 'keys' => param,
765
- 'delimiter' => ',',
766
- )
767
-
768
- assert_equal ['a', 'b'], parser.keys
769
- assert_equal ",", parser.delimiter
770
- end
771
-
772
- data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
773
- def test_parse(param)
774
- parser = TextParser::TSVParser.new
775
- parser.configure('keys' => param, 'time_key' => 'time')
776
- parser.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
777
- assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
778
- assert_equal({
779
- 'a' => '192.168.0.1',
780
- 'b' => '111',
781
- }, record)
782
- }
783
- end
784
-
785
- def test_parse_with_time
786
- time_at_start = Time.now.to_i
787
-
788
- parser = TextParser::TSVParser.new
789
- parser.configure('keys' => 'a,b')
790
- parser.parse("192.168.0.1\t111") { |time, record|
791
- assert time && time >= time_at_start, "parser puts current time without time input"
792
- assert_equal({
793
- 'a' => '192.168.0.1',
794
- 'b' => '111',
795
- }, record)
796
- }
797
-
798
- parser = TextParser::TSVParser.new
799
- parser.estimate_current_event = false
800
- parser.configure('keys' => 'a,b', 'time_key' => 'time')
801
- parser.parse("192.168.0.1\t111") { |time, record|
802
- assert_equal({
803
- 'a' => '192.168.0.1',
804
- 'b' => '111',
805
- }, record)
806
- assert_nil time, "parser returns nil w/o time and if configured so"
807
- }
808
- end
809
-
810
- data(
811
- 'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
812
- 'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
813
- 'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
814
- '2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
815
- 'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
816
- 'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
817
- def test_black_column(data)
818
- line, expected = data
819
-
820
- parser = TextParser::TSVParser.new
821
- parser.configure('keys' => '1,2,3')
822
- parser.parse(line) { |time, record|
823
- assert_equal(expected, record)
824
- }
825
- end
826
-
827
- def test_parse_with_keep_time_key
828
- parser = TextParser::TSVParser.new
829
- parser.configure(
830
- 'keys'=>'time',
831
- 'time_key'=>'time',
832
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
833
- 'keep_time_key'=>'true',
834
- )
835
- text = '28/Feb/2013:12:00:00 +0900'
836
- parser.parse(text) do |time, record|
837
- assert_equal text, record['time']
838
- end
839
- end
840
-
841
- data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
842
- def test_parse_with_null_value_pattern(param)
843
- parser = TextParser::TSVParser.new
844
- parser.configure(
845
- 'keys'=>param,
846
- 'null_value_pattern'=>'^(-|null|NULL)$'
847
- )
848
- parser.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
849
- assert_nil record['a']
850
- assert_nil record['b']
851
- assert_nil record['c']
852
- assert_equal record['d'], ''
853
- assert_equal record['e'], '--'
854
- assert_equal record['f'], 'nuLL'
855
- end
856
- end
857
-
858
- data('array param' => '["a","b"]', 'string param' => 'a,b')
859
- def test_parse_with_null_empty_string(param)
860
- parser = TextParser::TSVParser.new
861
- parser.configure(
862
- 'keys'=>param,
863
- 'null_empty_string'=>true
864
- )
865
- parser.parse("\t ") do |time, record|
866
- assert_nil record['a']
867
- assert_equal record['b'], ' '
868
- end
869
- end
870
- end
871
-
872
- class CSVParserTest < ::Test::Unit::TestCase
873
- include ParserTest
874
-
875
- data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
876
- def test_parse(param)
877
- parser = TextParser::CSVParser.new
878
- parser.configure('keys' => param, 'time_key' => 'time')
879
- parser.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
880
- assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
881
- assert_equal({
882
- 'c' => '192.168.0.1',
883
- 'd' => '111',
884
- }, record)
885
- }
886
- end
887
-
888
- data('array param' => '["c","d"]', 'string param' => 'c,d')
889
- def test_parse_without_time(param)
890
- time_at_start = Time.now.to_i
891
-
892
- parser = TextParser::CSVParser.new
893
- parser.configure('keys' => param)
894
- parser.parse("192.168.0.1,111") { |time, record|
895
- assert time && time >= time_at_start, "parser puts current time without time input"
896
- assert_equal({
897
- 'c' => '192.168.0.1',
898
- 'd' => '111',
899
- }, record)
900
- }
901
-
902
- parser = TextParser::CSVParser.new
903
- parser.estimate_current_event = false
904
- parser.configure('keys' => param, 'time_key' => 'time')
905
- parser.parse("192.168.0.1,111") { |time, record|
906
- assert_equal({
907
- 'c' => '192.168.0.1',
908
- 'd' => '111',
909
- }, record)
910
- assert_nil time, "parser returns nil w/o time and if configured so"
911
- }
912
- end
913
-
914
- def test_parse_with_keep_time_key
915
- parser = TextParser::CSVParser.new
916
- parser.configure(
917
- 'keys'=>'time',
918
- 'time_key'=>'time',
919
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
920
- 'keep_time_key'=>'true',
921
- )
922
- text = '28/Feb/2013:12:00:00 +0900'
923
- parser.parse(text) do |time, record|
924
- assert_equal text, record['time']
925
- end
926
- end
927
-
928
- data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
929
- def test_parse_with_null_value_pattern(param)
930
- parser = TextParser::CSVParser.new
931
- parser.configure(
932
- 'keys'=>param,
933
- 'null_value_pattern'=>'^(-|null|NULL)$'
934
- )
935
- parser.parse("-,null,NULL,,--,nuLL") do |time, record|
936
- assert_nil record['a']
937
- assert_nil record['b']
938
- assert_nil record['c']
939
- assert_nil record['d']
940
- assert_equal record['e'], '--'
941
- assert_equal record['f'], 'nuLL'
942
- end
943
- end
944
-
945
- data('array param' => '["a","b"]', 'string param' => 'a,b')
946
- def test_parse_with_null_empty_string(param)
947
- parser = TextParser::CSVParser.new
948
- parser.configure(
949
- 'keys'=>param,
950
- 'null_empty_string'=>true
951
- )
952
- parser.parse(", ") do |time, record|
953
- assert_nil record['a']
954
- assert_equal record['b'], ' '
955
- end
956
- end
957
- end
958
-
959
- class LabeledTSVParserTest < ::Test::Unit::TestCase
960
- include ParserTest
961
-
962
- def test_config_params
963
- parser = TextParser::LabeledTSVParser.new
964
-
965
- assert_equal "\t", parser.delimiter
966
- assert_equal ":", parser.label_delimiter
967
-
968
- parser.configure(
969
- 'delimiter' => ',',
970
- 'label_delimiter' => '=',
971
- )
972
-
973
- assert_equal ",", parser.delimiter
974
- assert_equal "=", parser.label_delimiter
975
- end
976
-
977
- def test_parse
978
- parser = TextParser::LabeledTSVParser.new
979
- parser.configure({})
980
- parser.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
981
- assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
982
- assert_equal({
983
- 'host' => '192.168.0.1',
984
- 'req_id' => '111',
985
- }, record)
986
- }
987
- end
988
-
989
- def test_parse_with_customized_delimiter
990
- parser = TextParser::LabeledTSVParser.new
991
- parser.configure(
992
- 'delimiter' => ',',
993
- 'label_delimiter' => '=',
994
- )
995
- parser.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
996
- assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
997
- assert_equal({
998
- 'host' => '192.168.0.1',
999
- 'req_id' => '111',
1000
- }, record)
1001
- }
1002
- end
1003
-
1004
- def test_parse_with_customized_time_format
1005
- parser = TextParser::LabeledTSVParser.new
1006
- parser.configure(
1007
- 'time_key' => 'mytime',
1008
- 'time_format' => '%d/%b/%Y:%H:%M:%S %z',
1009
- )
1010
- parser.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
1011
- assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
1012
- assert_equal({
1013
- 'host' => '192.168.0.1',
1014
- 'req_id' => '111',
1015
- }, record)
1016
- }
1017
- end
1018
-
1019
- def test_parse_without_time
1020
- time_at_start = Time.now.to_i
1021
-
1022
- parser = TextParser::LabeledTSVParser.new
1023
- parser.configure({})
1024
- parser.parse("host:192.168.0.1\treq_id:111") { |time, record|
1025
- assert time && time >= time_at_start, "parser puts current time without time input"
1026
- assert_equal({
1027
- 'host' => '192.168.0.1',
1028
- 'req_id' => '111',
1029
- }, record)
1030
- }
1031
-
1032
- parser = TextParser::LabeledTSVParser.new
1033
- parser.estimate_current_event = false
1034
- parser.configure({})
1035
- parser.parse("host:192.168.0.1\treq_id:111") { |time, record|
1036
- assert_equal({
1037
- 'host' => '192.168.0.1',
1038
- 'req_id' => '111',
1039
- }, record)
1040
- assert_nil time, "parser returns nil w/o time and if configured so"
1041
- }
1042
- end
1043
-
1044
- def test_parse_with_keep_time_key
1045
- parser = TextParser::LabeledTSVParser.new
1046
- parser.configure(
1047
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
1048
- 'keep_time_key'=>'true',
1049
- )
1050
- text = '28/Feb/2013:12:00:00 +0900'
1051
- parser.parse("time:#{text}") do |time, record|
1052
- assert_equal text, record['time']
1053
- end
1054
- end
1055
-
1056
- def test_parse_with_null_value_pattern
1057
- parser = TextParser::LabeledTSVParser.new
1058
- parser.configure(
1059
- 'null_value_pattern'=>'^(-|null|NULL)$'
1060
- )
1061
- parser.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
1062
- assert_nil record['a']
1063
- assert_nil record['b']
1064
- assert_nil record['c']
1065
- assert_equal record['d'], ''
1066
- assert_equal record['e'], '--'
1067
- assert_equal record['f'], 'nuLL'
1068
- end
1069
- end
1070
-
1071
- def test_parse_with_null_empty_string
1072
- parser = TextParser::LabeledTSVParser.new
1073
- parser.configure(
1074
- 'null_empty_string'=>true
1075
- )
1076
- parser.parse("a:\tb: ") do |time, record|
1077
- assert_nil record['a']
1078
- assert_equal record['b'], ' '
1079
- end
1080
- end
1081
- end
1082
-
1083
- class NoneParserTest < ::Test::Unit::TestCase
1084
- include ParserTest
1085
-
1086
- def test_config_params
1087
- parser = TextParser::NoneParser.new
1088
- parser.configure({})
1089
- assert_equal "message", parser.message_key
1090
-
1091
- parser.configure('message_key' => 'foobar')
1092
- assert_equal "foobar", parser.message_key
1093
- end
1094
-
1095
- def test_parse
1096
- parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
1097
- parser.configure({})
1098
- parser.parse('log message!') { |time, record|
1099
- assert_equal({'message' => 'log message!'}, record)
1100
- }
1101
- end
1102
-
1103
- def test_parse_with_message_key
1104
- parser = TextParser::NoneParser.new
1105
- parser.configure('message_key' => 'foobar')
1106
- parser.parse('log message!') { |time, record|
1107
- assert_equal({'foobar' => 'log message!'}, record)
1108
- }
1109
- end
1110
-
1111
- def test_parse_without_default_time
1112
- time_at_start = Time.now.to_i
1113
-
1114
- parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
1115
- parser.configure({})
1116
- parser.parse('log message!') { |time, record|
1117
- assert time && time >= time_at_start, "parser puts current time without time input"
1118
- assert_equal({'message' => 'log message!'}, record)
1119
- }
1120
-
1121
- parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
1122
- parser.estimate_current_event = false
1123
- parser.configure({})
1124
- parser.parse('log message!') { |time, record|
1125
- assert_equal({'message' => 'log message!'}, record)
1126
- assert_nil time, "parser returns nil w/o time if configured so"
1127
- }
1128
- end
1129
- end
1130
-
1131
- class MultilineParserTest < ::Test::Unit::TestCase
1132
- include ParserTest
1133
-
1134
- def create_parser(conf)
1135
- parser = TextParser::TEMPLATE_REGISTRY.lookup('multiline').call
1136
- parser.configure(conf)
1137
- parser
1138
- end
1139
-
1140
- def test_configure_with_invalid_params
1141
- [{'format100' => '/(?<msg>.*)/'}, {'format1' => '/(?<msg>.*)/', 'format3' => '/(?<msg>.*)/'}, 'format1' => '/(?<msg>.*)'].each { |config|
1142
- assert_raise(ConfigError) {
1143
- create_parser(config)
1144
- }
1145
- }
1146
- end
1147
-
1148
- def test_parse
1149
- parser = create_parser('format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/')
1150
- parser.parse(<<EOS.chomp) { |time, record|
1151
- 2013-3-03 14:27:33 [main] ERROR Main - Exception
1152
- javax.management.RuntimeErrorException: null
1153
- \tat Main.main(Main.java:16) ~[bin/:na]
1154
- EOS
1155
-
1156
- assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
1157
- assert_equal({
1158
- "thread" => "main",
1159
- "level" => "ERROR",
1160
- "message" => " Main - Exception\njavax.management.RuntimeErrorException: null\n\tat Main.main(Main.java:16) ~[bin/:na]"
1161
- }, record)
1162
- }
1163
- end
1164
-
1165
- def test_parse_with_firstline
1166
- parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/')
1167
- parser.parse(<<EOS.chomp) { |time, record|
1168
- ----
1169
- time=2013-3-03 14:27:33
1170
- message=test1
1171
- EOS
1172
-
1173
- assert(parser.firstline?('----'))
1174
- assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
1175
- assert_equal({"message" => "test1"}, record)
1176
- }
1177
- end
1178
-
1179
- def test_parse_with_multiple_formats
1180
- parser = create_parser('format_firstline' => '/^Started/',
1181
- 'format1' => '/Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/',
1182
- 'format2' => '/Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/',
1183
- 'format3' => '/( Parameters: (?<parameters>[^ ]+)\n)?/',
1184
- 'format4' => '/ Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/',
1185
- 'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/'
1186
- )
1187
- parser.parse(<<EOS.chomp) { |time, record|
1188
- Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
1189
- Processing by UsersController#show as HTML
1190
- Parameters: {"user_id"=>"123"}
1191
- Rendered users/show.html.erb within layouts/application (0.3ms)
1192
- Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
1193
- EOS
1194
-
1195
- assert(parser.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
1196
- assert_equal_event_time(event_time('2013-06-14 12:00:11 +0900'), time)
1197
- assert_equal({
1198
- "method" => "GET",
1199
- "path" => "/users/123/",
1200
- "host" => "127.0.0.1",
1201
- "controller" => "UsersController",
1202
- "controller_method" => "show",
1203
- "format" => "HTML",
1204
- "parameters" => "{\"user_id\"=>\"123\"}",
1205
- "template" => "users/show.html.erb",
1206
- "layout" => "layouts/application",
1207
- "code" => "200",
1208
- "runtime" => "4",
1209
- "view_runtime" => "3.2",
1210
- "ar_runtime" => "0.0"
1211
- }, record)
1212
- }
1213
- end
1214
-
1215
- def test_parse_with_keep_time_key
1216
- parser = TextParser::MultilineParser.new
1217
- parser.configure(
1218
- 'format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})/',
1219
- 'keep_time_key' => 'true'
1220
- )
1221
- text = '2013-3-03 14:27:33'
1222
- parser.parse(text) { |time, record|
1223
- assert_equal text, record['time']
1224
- }
1225
- end
1226
- end
1227
-
1228
- class TextParserTest < ::Test::Unit::TestCase
1229
- include ParserTest
1230
-
1231
- class MultiEventTestParser < ::Fluent::Parser
1232
- include Fluent::Configurable
1233
-
1234
- def parse(text)
1235
- 2.times { |i|
1236
- record = {}
1237
- record['message'] = text
1238
- record['number'] = i
1239
- yield Fluent::Engine.now, record
1240
- }
1241
- end
1242
- end
1243
-
1244
- TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
1245
-
1246
- def test_lookup_unknown_format
1247
- assert_raise ConfigError do
1248
- TextParser::TEMPLATE_REGISTRY.lookup('unknown')
1249
- end
1250
- end
1251
-
1252
- data('register_formatter' => 'known', 'register_template' => 'known_old')
1253
- def test_lookup_known_parser(data)
1254
- $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))
1255
- assert_nothing_raised ConfigError do
1256
- TextParser::TEMPLATE_REGISTRY.lookup(data)
1257
- end
1258
- $LOAD_PATH.shift
1259
- end
1260
-
1261
- def test_parse_with_return
1262
- parser = TextParser.new
1263
- parser.configure('format' => 'none')
1264
- time, record = parser.parse('log message!')
1265
- assert_equal({'message' => 'log message!'}, record)
1266
- end
1267
-
1268
- def test_parse_with_block
1269
- parser = TextParser.new
1270
- parser.configure('format' => 'none')
1271
- parser.parse('log message!') { |time, record|
1272
- assert_equal({'message' => 'log message!'}, record)
1273
- }
1274
- end
1275
-
1276
- def test_multi_event_parser
1277
- parser = TextParser.new
1278
- parser.configure('format' => 'multi_event_test')
1279
- i = 0
1280
- parser.parse('log message!') { |time, record|
1281
- assert_equal('log message!', record['message'])
1282
- assert_equal(i, record['number'])
1283
- i += 1
1284
- }
1285
- end
1286
-
1287
- def test_setting_estimate_current_event_value
1288
- p1 = TextParser.new
1289
- assert_nil p1.estimate_current_event
1290
- assert_nil p1.parser
1291
-
1292
- p1.configure('format' => 'none')
1293
- assert_equal true, p1.parser.estimate_current_event
1294
-
1295
- p2 = TextParser.new
1296
- assert_nil p2.estimate_current_event
1297
- assert_nil p2.parser
1298
-
1299
- p2.estimate_current_event = false
1300
-
1301
- p2.configure('format' => 'none')
1302
- assert_equal false, p2.parser.estimate_current_event
1303
- end
1304
- end
1305
- end