fluentd 0.12.40 → 1.6.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (428) hide show
  1. checksums.yaml +5 -5
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  4. data/.github/ISSUE_TEMPLATE.md +17 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
  6. data/.gitignore +5 -0
  7. data/.gitlab/cicd-template.yaml +10 -0
  8. data/.gitlab-ci.yml +147 -0
  9. data/.travis.yml +56 -20
  10. data/ADOPTERS.md +5 -0
  11. data/CHANGELOG.md +1369 -0
  12. data/CONTRIBUTING.md +16 -5
  13. data/GOVERNANCE.md +55 -0
  14. data/Gemfile +5 -0
  15. data/GithubWorkflow.md +78 -0
  16. data/LICENSE +202 -0
  17. data/MAINTAINERS.md +7 -0
  18. data/README.md +23 -11
  19. data/Rakefile +48 -2
  20. data/Vagrantfile +17 -0
  21. data/appveyor.yml +37 -0
  22. data/bin/fluent-binlog-reader +7 -0
  23. data/bin/fluent-ca-generate +6 -0
  24. data/bin/fluent-plugin-config-format +5 -0
  25. data/bin/fluent-plugin-generate +5 -0
  26. data/bin/fluentd +3 -0
  27. data/code-of-conduct.md +3 -0
  28. data/example/copy_roundrobin.conf +39 -0
  29. data/example/counter.conf +18 -0
  30. data/example/in_dummy_blocks.conf +17 -0
  31. data/example/in_dummy_with_compression.conf +23 -0
  32. data/example/in_forward.conf +7 -0
  33. data/example/in_forward_client.conf +37 -0
  34. data/example/in_forward_shared_key.conf +15 -0
  35. data/example/in_forward_tls.conf +14 -0
  36. data/example/in_forward_users.conf +24 -0
  37. data/example/in_forward_workers.conf +21 -0
  38. data/example/in_http.conf +3 -1
  39. data/example/in_out_forward.conf +17 -0
  40. data/example/logevents.conf +25 -0
  41. data/example/multi_filters.conf +61 -0
  42. data/example/out_exec_filter.conf +42 -0
  43. data/example/out_forward.conf +13 -13
  44. data/example/out_forward_buf_file.conf +23 -0
  45. data/example/out_forward_client.conf +109 -0
  46. data/example/out_forward_heartbeat_none.conf +16 -0
  47. data/example/out_forward_shared_key.conf +36 -0
  48. data/example/out_forward_tls.conf +18 -0
  49. data/example/out_forward_users.conf +65 -0
  50. data/example/out_null.conf +36 -0
  51. data/example/secondary_file.conf +42 -0
  52. data/example/suppress_config_dump.conf +7 -0
  53. data/example/worker_section.conf +36 -0
  54. data/fluent.conf +29 -0
  55. data/fluentd.gemspec +21 -11
  56. data/lib/fluent/agent.rb +67 -90
  57. data/lib/fluent/clock.rb +62 -0
  58. data/lib/fluent/command/binlog_reader.rb +244 -0
  59. data/lib/fluent/command/ca_generate.rb +181 -0
  60. data/lib/fluent/command/cat.rb +42 -18
  61. data/lib/fluent/command/debug.rb +12 -10
  62. data/lib/fluent/command/fluentd.rb +153 -5
  63. data/lib/fluent/command/plugin_config_formatter.rb +292 -0
  64. data/lib/fluent/command/plugin_generator.rb +324 -0
  65. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  66. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  67. data/lib/fluent/compat/exec_util.rb +129 -0
  68. data/lib/fluent/compat/file_util.rb +54 -0
  69. data/lib/fluent/compat/filter.rb +68 -0
  70. data/lib/fluent/compat/formatter.rb +111 -0
  71. data/lib/fluent/compat/formatter_utils.rb +85 -0
  72. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  73. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  74. data/lib/fluent/compat/input.rb +49 -0
  75. data/lib/fluent/compat/output.rb +718 -0
  76. data/lib/fluent/compat/output_chain.rb +60 -0
  77. data/lib/fluent/compat/parser.rb +310 -0
  78. data/lib/fluent/compat/parser_utils.rb +40 -0
  79. data/lib/fluent/compat/propagate_default.rb +62 -0
  80. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  81. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  82. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  83. data/lib/fluent/compat/socket_util.rb +165 -0
  84. data/lib/fluent/compat/string_util.rb +34 -0
  85. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  86. data/lib/fluent/compat/type_converter.rb +90 -0
  87. data/lib/fluent/config/configure_proxy.rb +210 -62
  88. data/lib/fluent/config/dsl.rb +12 -5
  89. data/lib/fluent/config/element.rb +107 -9
  90. data/lib/fluent/config/literal_parser.rb +9 -3
  91. data/lib/fluent/config/parser.rb +4 -4
  92. data/lib/fluent/config/section.rb +51 -14
  93. data/lib/fluent/config/types.rb +28 -13
  94. data/lib/fluent/config/v1_parser.rb +3 -5
  95. data/lib/fluent/config.rb +23 -20
  96. data/lib/fluent/configurable.rb +79 -21
  97. data/lib/fluent/counter/base_socket.rb +46 -0
  98. data/lib/fluent/counter/client.rb +297 -0
  99. data/lib/fluent/counter/error.rb +86 -0
  100. data/lib/fluent/counter/mutex_hash.rb +163 -0
  101. data/lib/fluent/counter/server.rb +273 -0
  102. data/lib/fluent/counter/store.rb +205 -0
  103. data/lib/fluent/counter/validator.rb +145 -0
  104. data/lib/fluent/counter.rb +23 -0
  105. data/lib/fluent/daemon.rb +15 -0
  106. data/lib/fluent/engine.rb +102 -65
  107. data/lib/fluent/env.rb +7 -3
  108. data/lib/fluent/error.rb +30 -0
  109. data/lib/fluent/event.rb +197 -21
  110. data/lib/fluent/event_router.rb +93 -10
  111. data/lib/fluent/filter.rb +2 -50
  112. data/lib/fluent/formatter.rb +4 -293
  113. data/lib/fluent/input.rb +2 -32
  114. data/lib/fluent/label.rb +10 -2
  115. data/lib/fluent/load.rb +3 -3
  116. data/lib/fluent/log.rb +348 -81
  117. data/lib/fluent/match.rb +37 -36
  118. data/lib/fluent/mixin.rb +12 -176
  119. data/lib/fluent/msgpack_factory.rb +62 -0
  120. data/lib/fluent/output.rb +10 -612
  121. data/lib/fluent/output_chain.rb +23 -0
  122. data/lib/fluent/parser.rb +4 -800
  123. data/lib/fluent/plugin/bare_output.rb +63 -0
  124. data/lib/fluent/plugin/base.rb +192 -0
  125. data/lib/fluent/plugin/buf_file.rb +128 -174
  126. data/lib/fluent/plugin/buf_memory.rb +9 -92
  127. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  128. data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
  129. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  130. data/lib/fluent/plugin/buffer.rb +779 -0
  131. data/lib/fluent/plugin/compressable.rb +92 -0
  132. data/lib/fluent/plugin/exec_util.rb +3 -108
  133. data/lib/fluent/plugin/file_util.rb +4 -34
  134. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  135. data/lib/fluent/plugin/filter.rb +93 -0
  136. data/lib/fluent/plugin/filter_grep.rb +117 -34
  137. data/lib/fluent/plugin/filter_parser.rb +85 -62
  138. data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
  139. data/lib/fluent/plugin/filter_stdout.rb +15 -12
  140. data/lib/fluent/plugin/formatter.rb +50 -0
  141. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  142. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  143. data/lib/fluent/plugin/formatter_json.rb +55 -0
  144. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  145. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  146. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  147. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  148. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  149. data/lib/fluent/plugin/formatter_tsv.rb +38 -0
  150. data/lib/fluent/plugin/in_debug_agent.rb +17 -6
  151. data/lib/fluent/plugin/in_dummy.rb +47 -20
  152. data/lib/fluent/plugin/in_exec.rb +55 -123
  153. data/lib/fluent/plugin/in_forward.rb +299 -216
  154. data/lib/fluent/plugin/in_gc_stat.rb +14 -36
  155. data/lib/fluent/plugin/in_http.rb +204 -91
  156. data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
  157. data/lib/fluent/plugin/in_object_space.rb +13 -41
  158. data/lib/fluent/plugin/in_syslog.rb +112 -134
  159. data/lib/fluent/plugin/in_tail.rb +408 -745
  160. data/lib/fluent/plugin/in_tcp.rb +66 -9
  161. data/lib/fluent/plugin/in_udp.rb +60 -11
  162. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
  163. data/lib/fluent/plugin/input.rb +37 -0
  164. data/lib/fluent/plugin/multi_output.rb +158 -0
  165. data/lib/fluent/plugin/out_copy.rb +23 -35
  166. data/lib/fluent/plugin/out_exec.rb +67 -70
  167. data/lib/fluent/plugin/out_exec_filter.rb +204 -271
  168. data/lib/fluent/plugin/out_file.rb +267 -73
  169. data/lib/fluent/plugin/out_forward.rb +854 -325
  170. data/lib/fluent/plugin/out_null.rb +42 -9
  171. data/lib/fluent/plugin/out_relabel.rb +9 -5
  172. data/lib/fluent/plugin/out_roundrobin.rb +18 -37
  173. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  174. data/lib/fluent/plugin/out_stdout.rb +43 -10
  175. data/lib/fluent/plugin/out_stream.rb +7 -2
  176. data/lib/fluent/plugin/output.rb +1498 -0
  177. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  178. data/lib/fluent/plugin/parser.rb +191 -0
  179. data/lib/fluent/plugin/parser_apache.rb +28 -0
  180. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  181. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  182. data/lib/fluent/plugin/parser_csv.rb +39 -0
  183. data/lib/fluent/plugin/parser_json.rb +94 -0
  184. data/lib/fluent/plugin/parser_ltsv.rb +49 -0
  185. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  186. data/lib/fluent/plugin/parser_multiline.rb +106 -0
  187. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  188. data/lib/fluent/plugin/parser_none.rb +36 -0
  189. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  190. data/lib/fluent/plugin/parser_syslog.rb +142 -0
  191. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  192. data/lib/fluent/plugin/socket_util.rb +3 -143
  193. data/lib/fluent/plugin/storage.rb +84 -0
  194. data/lib/fluent/plugin/storage_local.rb +164 -0
  195. data/lib/fluent/plugin/string_util.rb +3 -15
  196. data/lib/fluent/plugin.rb +122 -121
  197. data/lib/fluent/plugin_helper/cert_option.rb +178 -0
  198. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  199. data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
  200. data/lib/fluent/plugin_helper/counter.rb +51 -0
  201. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  202. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  203. data/lib/fluent/plugin_helper/extract.rb +104 -0
  204. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  205. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  206. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  207. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  208. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  209. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  210. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  211. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  212. data/lib/fluent/plugin_helper/http_server.rb +76 -0
  213. data/lib/fluent/plugin_helper/inject.rb +151 -0
  214. data/lib/fluent/plugin_helper/parser.rb +147 -0
  215. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  216. data/lib/fluent/plugin_helper/retry_state.rb +205 -0
  217. data/lib/fluent/plugin_helper/server.rb +807 -0
  218. data/lib/fluent/plugin_helper/socket.rb +250 -0
  219. data/lib/fluent/plugin_helper/socket_option.rb +80 -0
  220. data/lib/fluent/plugin_helper/storage.rb +349 -0
  221. data/lib/fluent/plugin_helper/thread.rb +179 -0
  222. data/lib/fluent/plugin_helper/timer.rb +92 -0
  223. data/lib/fluent/plugin_helper.rb +73 -0
  224. data/lib/fluent/plugin_id.rb +80 -0
  225. data/lib/fluent/process.rb +3 -489
  226. data/lib/fluent/registry.rb +52 -10
  227. data/lib/fluent/root_agent.rb +204 -42
  228. data/lib/fluent/supervisor.rb +597 -359
  229. data/lib/fluent/system_config.rb +131 -42
  230. data/lib/fluent/test/base.rb +6 -54
  231. data/lib/fluent/test/driver/base.rb +224 -0
  232. data/lib/fluent/test/driver/base_owned.rb +70 -0
  233. data/lib/fluent/test/driver/base_owner.rb +135 -0
  234. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  235. data/lib/fluent/test/driver/filter.rb +57 -0
  236. data/lib/fluent/test/driver/formatter.rb +30 -0
  237. data/lib/fluent/test/driver/input.rb +31 -0
  238. data/lib/fluent/test/driver/multi_output.rb +53 -0
  239. data/lib/fluent/test/driver/output.rb +102 -0
  240. data/lib/fluent/test/driver/parser.rb +30 -0
  241. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  242. data/lib/fluent/test/filter_test.rb +0 -1
  243. data/lib/fluent/test/formatter_test.rb +4 -1
  244. data/lib/fluent/test/helpers.rb +58 -10
  245. data/lib/fluent/test/input_test.rb +27 -19
  246. data/lib/fluent/test/log.rb +79 -0
  247. data/lib/fluent/test/output_test.rb +28 -39
  248. data/lib/fluent/test/parser_test.rb +3 -1
  249. data/lib/fluent/test/startup_shutdown.rb +46 -0
  250. data/lib/fluent/test.rb +33 -1
  251. data/lib/fluent/time.rb +450 -1
  252. data/lib/fluent/timezone.rb +27 -3
  253. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  254. data/lib/fluent/version.rb +1 -1
  255. data/lib/fluent/winsvc.rb +85 -0
  256. data/templates/new_gem/Gemfile +3 -0
  257. data/templates/new_gem/README.md.erb +43 -0
  258. data/templates/new_gem/Rakefile +13 -0
  259. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  260. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  261. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  262. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  263. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  264. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  265. data/templates/new_gem/test/helper.rb.erb +8 -0
  266. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  267. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  268. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  269. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  270. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  271. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  272. data/templates/plugin_config_formatter/param.md.erb +34 -0
  273. data/templates/plugin_config_formatter/section.md.erb +12 -0
  274. data/test/command/test_binlog_reader.rb +346 -0
  275. data/test/command/test_ca_generate.rb +70 -0
  276. data/test/command/test_fluentd.rb +901 -0
  277. data/test/command/test_plugin_config_formatter.rb +276 -0
  278. data/test/command/test_plugin_generator.rb +92 -0
  279. data/test/compat/test_calls_super.rb +166 -0
  280. data/test/compat/test_parser.rb +92 -0
  281. data/test/config/test_config_parser.rb +126 -2
  282. data/test/config/test_configurable.rb +946 -187
  283. data/test/config/test_configure_proxy.rb +424 -74
  284. data/test/config/test_dsl.rb +11 -11
  285. data/test/config/test_element.rb +500 -0
  286. data/test/config/test_literal_parser.rb +8 -0
  287. data/test/config/test_plugin_configuration.rb +56 -0
  288. data/test/config/test_section.rb +79 -7
  289. data/test/config/test_system_config.rb +122 -35
  290. data/test/config/test_types.rb +38 -0
  291. data/test/counter/test_client.rb +559 -0
  292. data/test/counter/test_error.rb +44 -0
  293. data/test/counter/test_mutex_hash.rb +179 -0
  294. data/test/counter/test_server.rb +589 -0
  295. data/test/counter/test_store.rb +258 -0
  296. data/test/counter/test_validator.rb +137 -0
  297. data/test/helper.rb +89 -6
  298. data/test/helpers/fuzzy_assert.rb +89 -0
  299. data/test/plugin/test_bare_output.rb +118 -0
  300. data/test/plugin/test_base.rb +115 -0
  301. data/test/plugin/test_buf_file.rb +823 -460
  302. data/test/plugin/test_buf_memory.rb +32 -194
  303. data/test/plugin/test_buffer.rb +1233 -0
  304. data/test/plugin/test_buffer_chunk.rb +198 -0
  305. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  306. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  307. data/test/plugin/test_compressable.rb +84 -0
  308. data/test/plugin/test_filter.rb +357 -0
  309. data/test/plugin/test_filter_grep.rb +540 -29
  310. data/test/plugin/test_filter_parser.rb +439 -452
  311. data/test/plugin/test_filter_record_transformer.rb +123 -166
  312. data/test/plugin/test_filter_stdout.rb +160 -72
  313. data/test/plugin/test_formatter_csv.rb +111 -0
  314. data/test/plugin/test_formatter_hash.rb +35 -0
  315. data/test/plugin/test_formatter_json.rb +51 -0
  316. data/test/plugin/test_formatter_ltsv.rb +62 -0
  317. data/test/plugin/test_formatter_msgpack.rb +28 -0
  318. data/test/plugin/test_formatter_out_file.rb +95 -0
  319. data/test/plugin/test_formatter_single_value.rb +38 -0
  320. data/test/plugin/test_formatter_tsv.rb +68 -0
  321. data/test/plugin/test_in_debug_agent.rb +24 -1
  322. data/test/plugin/test_in_dummy.rb +111 -18
  323. data/test/plugin/test_in_exec.rb +200 -113
  324. data/test/plugin/test_in_forward.rb +990 -387
  325. data/test/plugin/test_in_gc_stat.rb +10 -8
  326. data/test/plugin/test_in_http.rb +600 -224
  327. data/test/plugin/test_in_monitor_agent.rb +690 -0
  328. data/test/plugin/test_in_object_space.rb +24 -8
  329. data/test/plugin/test_in_syslog.rb +154 -215
  330. data/test/plugin/test_in_tail.rb +1006 -707
  331. data/test/plugin/test_in_tcp.rb +125 -48
  332. data/test/plugin/test_in_udp.rb +204 -63
  333. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
  334. data/test/plugin/test_input.rb +126 -0
  335. data/test/plugin/test_metadata.rb +89 -0
  336. data/test/plugin/test_multi_output.rb +180 -0
  337. data/test/plugin/test_out_copy.rb +117 -112
  338. data/test/plugin/test_out_exec.rb +258 -53
  339. data/test/plugin/test_out_exec_filter.rb +538 -115
  340. data/test/plugin/test_out_file.rb +865 -178
  341. data/test/plugin/test_out_forward.rb +998 -210
  342. data/test/plugin/test_out_null.rb +105 -0
  343. data/test/plugin/test_out_relabel.rb +28 -0
  344. data/test/plugin/test_out_roundrobin.rb +36 -29
  345. data/test/plugin/test_out_secondary_file.rb +458 -0
  346. data/test/plugin/test_out_stdout.rb +135 -37
  347. data/test/plugin/test_out_stream.rb +18 -0
  348. data/test/plugin/test_output.rb +984 -0
  349. data/test/plugin/test_output_as_buffered.rb +2021 -0
  350. data/test/plugin/test_output_as_buffered_backup.rb +312 -0
  351. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  352. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  353. data/test/plugin/test_output_as_buffered_retries.rb +911 -0
  354. data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
  355. data/test/plugin/test_output_as_standard.rb +374 -0
  356. data/test/plugin/test_owned_by.rb +35 -0
  357. data/test/plugin/test_parser.rb +359 -0
  358. data/test/plugin/test_parser_apache.rb +42 -0
  359. data/test/plugin/test_parser_apache2.rb +47 -0
  360. data/test/plugin/test_parser_apache_error.rb +45 -0
  361. data/test/plugin/test_parser_csv.rb +103 -0
  362. data/test/plugin/test_parser_json.rb +138 -0
  363. data/test/plugin/test_parser_labeled_tsv.rb +145 -0
  364. data/test/plugin/test_parser_multiline.rb +100 -0
  365. data/test/plugin/test_parser_nginx.rb +88 -0
  366. data/test/plugin/test_parser_none.rb +52 -0
  367. data/test/plugin/test_parser_regexp.rb +289 -0
  368. data/test/plugin/test_parser_syslog.rb +441 -0
  369. data/test/plugin/test_parser_tsv.rb +122 -0
  370. data/test/plugin/test_storage.rb +167 -0
  371. data/test/plugin/test_storage_local.rb +335 -0
  372. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  373. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  374. data/test/plugin_helper/data/cert/cert.pem +19 -0
  375. data/test/plugin_helper/http_server/test_app.rb +65 -0
  376. data/test/plugin_helper/http_server/test_route.rb +32 -0
  377. data/test/plugin_helper/test_cert_option.rb +16 -0
  378. data/test/plugin_helper/test_child_process.rb +794 -0
  379. data/test/plugin_helper/test_compat_parameters.rb +353 -0
  380. data/test/plugin_helper/test_event_emitter.rb +51 -0
  381. data/test/plugin_helper/test_event_loop.rb +52 -0
  382. data/test/plugin_helper/test_extract.rb +194 -0
  383. data/test/plugin_helper/test_formatter.rb +255 -0
  384. data/test/plugin_helper/test_http_server_helper.rb +205 -0
  385. data/test/plugin_helper/test_inject.rb +519 -0
  386. data/test/plugin_helper/test_parser.rb +264 -0
  387. data/test/plugin_helper/test_record_accessor.rb +197 -0
  388. data/test/plugin_helper/test_retry_state.rb +442 -0
  389. data/test/plugin_helper/test_server.rb +1714 -0
  390. data/test/plugin_helper/test_storage.rb +542 -0
  391. data/test/plugin_helper/test_thread.rb +164 -0
  392. data/test/plugin_helper/test_timer.rb +132 -0
  393. data/test/scripts/exec_script.rb +0 -6
  394. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  395. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  396. data/test/scripts/fluent/plugin/out_test.rb +23 -15
  397. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  398. data/test/test_clock.rb +164 -0
  399. data/test/test_config.rb +16 -7
  400. data/test/test_configdsl.rb +2 -2
  401. data/test/test_event.rb +360 -13
  402. data/test/test_event_router.rb +108 -11
  403. data/test/test_event_time.rb +199 -0
  404. data/test/test_filter.rb +48 -6
  405. data/test/test_formatter.rb +11 -391
  406. data/test/test_input.rb +1 -1
  407. data/test/test_log.rb +591 -31
  408. data/test/test_mixin.rb +1 -1
  409. data/test/test_output.rb +121 -185
  410. data/test/test_plugin.rb +251 -0
  411. data/test/test_plugin_classes.rb +177 -10
  412. data/test/test_plugin_helper.rb +81 -0
  413. data/test/test_plugin_id.rb +101 -0
  414. data/test/test_process.rb +8 -42
  415. data/test/test_root_agent.rb +766 -21
  416. data/test/test_supervisor.rb +481 -0
  417. data/test/test_test_drivers.rb +135 -0
  418. data/test/test_time_formatter.rb +282 -0
  419. data/test/test_time_parser.rb +231 -0
  420. data/test/test_unique_id.rb +47 -0
  421. metadata +454 -60
  422. data/COPYING +0 -14
  423. data/ChangeLog +0 -666
  424. data/lib/fluent/buffer.rb +0 -365
  425. data/lib/fluent/plugin/in_status.rb +0 -76
  426. data/test/plugin/test_in_status.rb +0 -38
  427. data/test/test_buffer.rb +0 -624
  428. data/test/test_parser.rb +0 -1305
@@ -0,0 +1,126 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/input'
3
+ require 'flexmock/test_unit'
4
+
5
+ module FluentPluginInputTest
6
+ class DummyPlugin < Fluent::Plugin::Input
7
+ end
8
+ end
9
+
10
+ class InputTest < Test::Unit::TestCase
11
+ setup do
12
+ Fluent::Test.setup
13
+ @p = FluentPluginInputTest::DummyPlugin.new
14
+ end
15
+
16
+ test 'has healthy lifecycle' do
17
+ assert !@p.configured?
18
+ @p.configure(config_element())
19
+ assert @p.configured?
20
+
21
+ assert !@p.started?
22
+ @p.start
23
+ assert @p.start
24
+
25
+ assert !@p.stopped?
26
+ @p.stop
27
+ assert @p.stopped?
28
+
29
+ assert !@p.before_shutdown?
30
+ @p.before_shutdown
31
+ assert @p.before_shutdown?
32
+
33
+ assert !@p.shutdown?
34
+ @p.shutdown
35
+ assert @p.shutdown?
36
+
37
+ assert !@p.after_shutdown?
38
+ @p.after_shutdown
39
+ assert @p.after_shutdown?
40
+
41
+ assert !@p.closed?
42
+ @p.close
43
+ assert @p.closed?
44
+
45
+ assert !@p.terminated?
46
+ @p.terminate
47
+ assert @p.terminated?
48
+ end
49
+
50
+ test 'has plugin_id automatically generated' do
51
+ assert @p.respond_to?(:plugin_id_configured?)
52
+ assert @p.respond_to?(:plugin_id)
53
+
54
+ @p.configure(config_element())
55
+
56
+ assert !@p.plugin_id_configured?
57
+ assert @p.plugin_id
58
+ assert{ @p.plugin_id != 'mytest' }
59
+ end
60
+
61
+ test 'has plugin_id manually configured' do
62
+ @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
63
+ assert @p.plugin_id_configured?
64
+ assert_equal 'mytest', @p.plugin_id
65
+ end
66
+
67
+ test 'has plugin logger' do
68
+ assert @p.respond_to?(:log)
69
+ assert @p.log
70
+
71
+ # default logger
72
+ original_logger = @p.log
73
+
74
+ @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
75
+
76
+ assert{ @p.log.object_id != original_logger.object_id }
77
+ assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
78
+ end
79
+
80
+ test 'can load plugin helpers' do
81
+ assert_nothing_raised do
82
+ class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input
83
+ helpers :storage
84
+ end
85
+ end
86
+ end
87
+
88
+ test 'are not available with multi workers configuration in default' do
89
+ assert_false @p.multi_workers_ready?
90
+ end
91
+
92
+ test 'has router and can emit into it' do
93
+ assert @p.has_router?
94
+
95
+ @p.configure(config_element())
96
+ assert @p.router
97
+
98
+ DummyRouter = Struct.new(:emits) do
99
+ def emit(tag, es)
100
+ self.emits << [tag, es]
101
+ end
102
+ end
103
+ @p.router = DummyRouter.new([])
104
+ @p.router.emit('mytag', [])
105
+ @p.router.emit('mytag.testing', ['it is not es, but no problem for tests'])
106
+
107
+ assert_equal ['mytag', []], @p.router.emits[0]
108
+ assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1]
109
+ end
110
+
111
+ test 'has router for specified label if configured' do
112
+ @p.configure(config_element())
113
+ original_router = @p.router
114
+
115
+ router_mock = flexmock('mytest')
116
+ router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock'])
117
+ label_mock = flexmock('mylabel')
118
+ label_mock.should_receive(:event_router).once.and_return(router_mock)
119
+ Fluent::Engine.root_agent.labels['@mytest'] = label_mock
120
+
121
+ @p.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
122
+ assert{ @p.router.object_id != original_router.object_id }
123
+
124
+ @p.router.emit('mytag.testing', ['for mock'])
125
+ end
126
+ end
@@ -0,0 +1,89 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buffer'
3
+
4
+ class BufferMetadataTest < Test::Unit::TestCase
5
+
6
+ def meta(timekey=nil, tag=nil, variables=nil)
7
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
8
+ end
9
+
10
+ setup do
11
+ Fluent::Test.setup
12
+ end
13
+
14
+ sub_test_case 'about metadata' do
15
+ test 'comparison of variables should be stable' do
16
+ m = meta(nil, nil, nil)
17
+ # different sets of keys
18
+ assert_equal(-1, m.cmp_variables({}, {a: 1}))
19
+ assert_equal(1, m.cmp_variables({a: 1}, {}))
20
+ assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))
21
+ assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))
22
+ assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))
23
+ assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))
24
+ # same set of keys
25
+ assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))
26
+ assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))
27
+ assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))
28
+ assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys
29
+ assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))
30
+ assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))
31
+ end
32
+
33
+ test 'comparison of metadata should be stable' do
34
+ n = Time.now.to_i
35
+
36
+ assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))
37
+ assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))
38
+ assert_equal(0, meta(nil, "t1", nil) <=> meta(nil, "t1", nil))
39
+ assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))
40
+ assert_equal(0, meta(nil, nil, {a: "1"}) <=> meta(nil, nil, {a: "1"}))
41
+ assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))
42
+ assert_equal(0, meta(n, "t1", {}) <=> meta(n, "t1", {}))
43
+ assert_equal(0, meta(n, "t1", {a: "x", b: 10}) <=> meta(n, "t1", {a: "x", b: 10}))
44
+
45
+ # timekey is 1st comparison key
46
+ assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))
47
+ assert_equal(1, meta(n + 1, "a", nil) <=> meta(n - 1, "b", nil))
48
+ assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))
49
+
50
+ # tag is 2nd
51
+ assert_equal(-1, meta(nil, "a", {}) <=> meta(nil, "b", {}))
52
+ assert_equal(-1, meta(n, "a", {}) <=> meta(n, "b", {}))
53
+ assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {}))
54
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {}))
55
+ assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {a: 1}))
56
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 2}))
57
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 10, b: 1}))
58
+
59
+ # variables is the last
60
+ assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))
61
+ assert_equal(-1, meta(n, "t", {}) <=> meta(n, "t", {a: 1}))
62
+ assert_equal(1, meta(n, "t", {a: 1}) <=> meta(n, "t", {}))
63
+ assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 2}))
64
+ assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 1, b: 1}))
65
+ assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))
66
+ assert_equal(1, meta(n, "t", {b: 1}) <=> meta(n, "t", {a: 1}))
67
+ end
68
+
69
+ test 'metadata can be sorted' do
70
+ n = Time.now.to_i
71
+ m0 = meta(nil, nil, nil)
72
+ m1 = meta(n - 1, nil, nil)
73
+ m2 = meta(n - 1, "a", nil)
74
+ m3 = meta(n - 1, "a", {a: 1})
75
+ m4 = meta(n - 1, "a", {a: 100})
76
+ m5 = meta(n - 1, "a", {a: 100, b: 1})
77
+ m6 = meta(n - 1, "aa", nil)
78
+ m7 = meta(n - 1, "aa", {a: 1})
79
+ m8 = meta(n - 1, "b", nil)
80
+ m9 = meta(n, nil, nil)
81
+ m10 = meta(n + 1, nil, {a: 1})
82
+ expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze
83
+ ary = expected.dup
84
+ 100.times do
85
+ assert_equal expected, ary.shuffle.sort
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,180 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/multi_output'
3
+ require 'fluent/event'
4
+
5
+ require 'json'
6
+ require 'time'
7
+ require 'timeout'
8
+
9
+ module FluentPluginMultiOutputTest
10
+ class DummyMultiOutput < Fluent::Plugin::MultiOutput
11
+ attr_reader :events
12
+ def initialize
13
+ super
14
+ @events = []
15
+ end
16
+ def configure(conf)
17
+ super
18
+ end
19
+ def process(tag, es)
20
+ es.each do |time, record|
21
+ @events << [tag, time, record]
22
+ end
23
+ end
24
+ end
25
+ class DummyCompatMultiOutput < Fluent::Plugin::MultiOutput
26
+ def initialize
27
+ super
28
+ @compat = true
29
+ end
30
+ def configure(conf)
31
+ super
32
+ end
33
+ def process(tag, es)
34
+ # ...
35
+ end
36
+ end
37
+
38
+ class Dummy1Output < Fluent::Plugin::Output
39
+ Fluent::Plugin.register_output('dummy_test_multi_output_1', self)
40
+ attr_reader :configured
41
+ def configure(conf)
42
+ super
43
+ @configured = true
44
+ end
45
+ def process(tag, es)
46
+ end
47
+ end
48
+ class Dummy2Output < Fluent::Plugin::Output
49
+ Fluent::Plugin.register_output('dummy_test_multi_output_2', self)
50
+ attr_reader :configured
51
+ def configure(conf)
52
+ super
53
+ @configured = true
54
+ end
55
+ def process(tag, es)
56
+ end
57
+ end
58
+ class Dummy3Output < Fluent::Plugin::Output
59
+ Fluent::Plugin.register_output('dummy_test_multi_output_3', self)
60
+ attr_reader :configured
61
+ def configure(conf)
62
+ super
63
+ @configured = true
64
+ end
65
+ def process(tag, es)
66
+ end
67
+ end
68
+ class Dummy4Output < Fluent::Plugin::Output
69
+ Fluent::Plugin.register_output('dummy_test_multi_output_4', self)
70
+ attr_reader :configured
71
+ def configure(conf)
72
+ super
73
+ @configured = true
74
+ end
75
+ def process(tag, es)
76
+ end
77
+ end
78
+ end
79
+
80
+ class MultiOutputTest < Test::Unit::TestCase
81
+ def create_output(type=:multi)
82
+ case type
83
+ when :compat_multi
84
+ FluentPluginMultiOutputTest::DummyCompatMultiOutput.new
85
+ else
86
+ FluentPluginMultiOutputTest::DummyMultiOutput.new
87
+ end
88
+ end
89
+
90
+ sub_test_case 'basic multi output plugin' do
91
+ setup do
92
+ Fluent::Test.setup
93
+ @i = create_output()
94
+ end
95
+
96
+ teardown do
97
+ @i.log.out.reset
98
+ end
99
+
100
+ test '#configure raises error if <store> sections are missing' do
101
+ conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' }, [])
102
+ assert_raise Fluent::ConfigError do
103
+ @i.configure(conf)
104
+ end
105
+ end
106
+
107
+ test '#configure initialize child plugins and call these #configure' do
108
+ assert_equal [], @i.outputs
109
+
110
+ conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
111
+ [
112
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
113
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
114
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
115
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
116
+ ]
117
+ )
118
+ @i.configure(conf)
119
+
120
+ assert_equal 4, @i.outputs.size
121
+
122
+ assert @i.outputs[0].is_a? FluentPluginMultiOutputTest::Dummy1Output
123
+ assert @i.outputs[0].configured
124
+
125
+ assert @i.outputs[1].is_a? FluentPluginMultiOutputTest::Dummy2Output
126
+ assert @i.outputs[1].configured
127
+
128
+ assert @i.outputs[2].is_a? FluentPluginMultiOutputTest::Dummy3Output
129
+ assert @i.outputs[2].configured
130
+
131
+ assert @i.outputs[3].is_a? FluentPluginMultiOutputTest::Dummy4Output
132
+ assert @i.outputs[3].configured
133
+ end
134
+
135
+ test '#configure warns if "type" is used in <store> sections instead of "@type"' do
136
+ assert_equal [], @i.log.out.logs
137
+
138
+ conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
139
+ [
140
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
141
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
142
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
143
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
144
+ ]
145
+ )
146
+ @i.configure(conf)
147
+ assert_equal 4, @i.outputs.size
148
+
149
+ logs = @i.log.out.logs
150
+ assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
151
+ end
152
+
153
+ test '#emit_events calls #process always' do
154
+ conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
155
+ [
156
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
157
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
158
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
159
+ config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
160
+ ]
161
+ )
162
+ @i.configure(conf)
163
+ @i.start
164
+
165
+ assert @i.events.empty?
166
+
167
+ @i.emit_events(
168
+ 'test.tag',
169
+ Fluent::ArrayEventStream.new(
170
+ [
171
+ [event_time(), {"message" => "multi test 1"}],
172
+ [event_time(), {"message" => "multi test 1"}],
173
+ ]
174
+ )
175
+ )
176
+
177
+ assert_equal 2, @i.events.size
178
+ end
179
+ end
180
+ end
@@ -1,11 +1,14 @@
1
1
  require_relative '../helper'
2
- require 'fluent/test'
2
+ require 'fluent/test/driver/multi_output'
3
+ require 'fluent/plugin/out_copy'
4
+ require 'fluent/event'
3
5
 
4
6
  class CopyOutputTest < Test::Unit::TestCase
5
7
  class << self
6
8
  def startup
7
9
  $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
8
10
  require 'fluent/plugin/out_test'
11
+ require 'fluent/plugin/out_test2'
9
12
  end
10
13
 
11
14
  def shutdown
@@ -19,21 +22,21 @@ class CopyOutputTest < Test::Unit::TestCase
19
22
 
20
23
  CONFIG = %[
21
24
  <store>
22
- type test
25
+ @type test
23
26
  name c0
24
27
  </store>
25
28
  <store>
26
- type test
29
+ @type test2
27
30
  name c1
28
31
  </store>
29
32
  <store>
30
- type test
33
+ @type test
31
34
  name c2
32
35
  </store>
33
36
  ]
34
37
 
35
38
  def create_driver(conf = CONFIG)
36
- Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput).configure(conf)
39
+ Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(conf)
37
40
  end
38
41
 
39
42
  def test_configure
@@ -41,146 +44,148 @@ class CopyOutputTest < Test::Unit::TestCase
41
44
 
42
45
  outputs = d.instance.outputs
43
46
  assert_equal 3, outputs.size
44
- assert_equal Fluent::TestOutput, outputs[0].class
45
- assert_equal Fluent::TestOutput, outputs[1].class
46
- assert_equal Fluent::TestOutput, outputs[2].class
47
+ assert_equal Fluent::Plugin::TestOutput, outputs[0].class
48
+ assert_equal Fluent::Plugin::Test2Output, outputs[1].class
49
+ assert_equal Fluent::Plugin::TestOutput, outputs[2].class
47
50
  assert_equal "c0", outputs[0].name
48
51
  assert_equal "c1", outputs[1].name
49
52
  assert_equal "c2", outputs[2].name
50
53
  end
51
54
 
52
- def test_emit
55
+ def test_feed_events
53
56
  d = create_driver
54
57
 
55
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
56
- d.emit({"a"=>1}, time)
57
- d.emit({"a"=>2}, time)
58
+ assert !d.instance.outputs[0].has_router?
59
+ assert_not_nil d.instance.outputs[1].router
60
+ assert !d.instance.outputs[2].has_router?
58
61
 
59
- d.instance.outputs.each {|o|
60
- assert_equal [
61
- [time, {"a"=>1}],
62
- [time, {"a"=>2}],
63
- ], o.events
64
- }
62
+ time = event_time("2011-01-02 13:14:15 UTC")
63
+ d.run(default_tag: 'test') do
64
+ d.feed(time, {"a" => 1})
65
+ d.feed(time, {"a" => 2})
66
+ end
65
67
 
66
68
  d.instance.outputs.each {|o|
67
- assert_not_nil o.router
69
+ assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
68
70
  }
69
71
  end
70
72
 
71
- def test_msgpack_es_emit_bug
72
- d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
73
-
74
- outputs = %w(p1 p2).map do |pname|
75
- p = Fluent::Plugin.new_output('test')
76
- p.configure('name' => pname)
77
- p.define_singleton_method(:emit) do |tag, es, chain|
78
- es.each do |time, record|
79
- super(tag, [[time, record]], chain)
80
- end
81
- end
82
- p
83
- end
84
-
85
- d.instance.instance_eval { @outputs = outputs }
73
+ def test_msgpack_unpacker_cache_bug_for_msgpack_event_stream
74
+ d = create_driver
86
75
 
87
- es = if defined?(MessagePack::Packer)
88
- time = Time.parse("2013-05-26 06:37:22 UTC").to_i
89
- packer = Fluent::Engine.msgpack_factory.packer
90
- packer.pack([time, {"a" => 1}])
91
- packer.pack([time, {"a" => 2}])
92
- Fluent::MessagePackEventStream.new(packer.to_s)
93
- else
94
- events = "#{[time, {"a" => 1}].to_msgpack}#{[time, {"a" => 2}].to_msgpack}"
95
- Fluent::MessagePackEventStream.new(events)
96
- end
76
+ time = event_time("2011-01-02 13:14:15 UTC")
77
+ source = Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"a" => 2}] ])
78
+ es = Fluent::MessagePackEventStream.new(source.to_msgpack_stream)
97
79
 
98
- d.instance.emit('test', es, Fluent::NullOutputChain.instance)
80
+ d.run(default_tag: 'test') do
81
+ d.feed(es)
82
+ end
99
83
 
100
84
  d.instance.outputs.each { |o|
101
- assert_equal [
102
- [time, {"a"=>1}],
103
- [time, {"a"=>2}],
104
- ], o.events
85
+ assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
105
86
  }
106
87
  end
107
88
 
108
- def create_event_test_driver(is_deep_copy = false)
109
- deep_copy_config = %[
110
- deep_copy true
111
- ]
112
-
113
- output1 = Fluent::Plugin.new_output('test')
114
- output1.configure('name' => 'output1')
115
- output1.define_singleton_method(:emit) do |tag, es, chain|
89
+ def create_event_test_driver(does_deep_copy = false)
90
+ config = %[
91
+ deep_copy #{does_deep_copy}
92
+ <store>
93
+ @type test
94
+ name output1
95
+ </store>
96
+ <store>
97
+ @type test
98
+ name output2
99
+ </store>
100
+ ]
101
+
102
+ d = Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(config)
103
+ d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
116
104
  es.each do |time, record|
117
105
  record['foo'] = 'bar'
118
- super(tag, [[time, record]], chain)
119
106
  end
107
+ super(tag, es)
120
108
  end
109
+ d
110
+ end
121
111
 
122
- output2 = Fluent::Plugin.new_output('test')
123
- output2.configure('name' => 'output2')
124
- output2.define_singleton_method(:emit) do |tag, es, chain|
125
- es.each do |time, record|
126
- super(tag, [[time, record]], chain)
127
- end
112
+ time = event_time("2013-05-26 06:37:22 UTC")
113
+ mes0 = Fluent::MultiEventStream.new
114
+ mes0.add(time, {"a" => 1})
115
+ mes0.add(time, {"b" => 1})
116
+ mes1 = Fluent::MultiEventStream.new
117
+ mes1.add(time, {"a" => 1})
118
+ mes1.add(time, {"b" => 1})
119
+
120
+ data(
121
+ "OneEventStream without deep_copy" => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
122
+ "OneEventStream with deep_copy" => [true, Fluent::OneEventStream.new(time, {"a" => 1})],
123
+ "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
124
+ "ArrayEventStream with deep_copy" => [true, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
125
+ "MultiEventStream without deep_copy" => [false, mes0],
126
+ "MultiEventStream with deep_copy" => [true, mes1],
127
+ )
128
+ def test_deep_copy_controls_shallow_or_deep_copied(data)
129
+ does_deep_copy, es = data
130
+
131
+ d = create_event_test_driver(does_deep_copy)
132
+
133
+ d.run(default_tag: 'test') do
134
+ d.feed(es)
128
135
  end
129
136
 
130
- outputs = [output1, output2]
131
-
132
- d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
133
- d = d.configure(deep_copy_config) if is_deep_copy
134
- d.instance.instance_eval { @outputs = outputs }
135
- d
136
- end
137
+ events = d.instance.outputs.map(&:events)
137
138
 
138
- def test_one_event
139
- time = Time.parse("2013-05-26 06:37:22 UTC").to_i
139
+ if does_deep_copy
140
+ events[0].each_with_index do |entry0, i|
141
+ record0 = entry0.last
142
+ record1 = events[1][i].last
140
143
 
141
- d = create_event_test_driver(false)
142
- es = Fluent::OneEventStream.new(time, {"a" => 1})
143
- d.instance.emit('test', es, Fluent::NullOutputChain.instance)
144
+ assert{ record0.object_id != record1.object_id }
145
+ assert_equal "bar", record0["foo"]
146
+ assert !record1.has_key?("foo")
147
+ end
148
+ else
149
+ events[0].each_with_index do |entry0, i|
150
+ record0 = entry0.last
151
+ record1 = events[1][i].last
152
+
153
+ assert{ record0.object_id == record1.object_id }
154
+ assert_equal "bar", record0["foo"]
155
+ assert_equal "bar", record1["foo"]
156
+ end
157
+ end
158
+ end
144
159
 
145
- assert_equal [
146
- [[time, {"a"=>1, "foo"=>"bar"}]],
147
- [[time, {"a"=>1, "foo"=>"bar"}]]
148
- ], d.instance.outputs.map{ |o| o.events }
160
+ IGNORE_ERROR_CONFIG = %[
161
+ <store ignore_error>
162
+ @type test
163
+ name c0
164
+ </store>
165
+ <store ignore_error>
166
+ @type test
167
+ name c1
168
+ </store>
169
+ <store>
170
+ @type test
171
+ name c2
172
+ </store>
173
+ ]
149
174
 
150
- d = create_event_test_driver(true)
151
- es = Fluent::OneEventStream.new(time, {"a" => 1})
152
- d.instance.emit('test', es, Fluent::NullOutputChain.instance)
175
+ def test_ignore_error
176
+ d = create_driver(IGNORE_ERROR_CONFIG)
153
177
 
154
- assert_equal [
155
- [[time, {"a"=>1, "foo"=>"bar"}]],
156
- [[time, {"a"=>1}]]
157
- ], d.instance.outputs.map{ |o| o.events }
158
- end
178
+ # override to raise an error
179
+ d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
180
+ raise ArgumentError, 'Failed'
181
+ end
159
182
 
160
- def test_multi_event
161
- time = Time.parse("2013-05-26 06:37:22 UTC").to_i
162
-
163
- d = create_event_test_driver(false)
164
- es = Fluent::MultiEventStream.new
165
- es.add(time, {"a" => 1})
166
- es.add(time, {"b" => 2})
167
- d.instance.emit('test', es, Fluent::NullOutputChain.instance)
168
-
169
- assert_equal [
170
- [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
171
- [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]]
172
- ], d.instance.outputs.map{ |o| o.events }
173
-
174
- d = create_event_test_driver(true)
175
- es = Fluent::MultiEventStream.new
176
- es.add(time, {"a" => 1})
177
- es.add(time, {"b" => 2})
178
- d.instance.emit('test', es, Fluent::NullOutputChain.instance)
179
-
180
- assert_equal [
181
- [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
182
- [[time, {"a"=>1}], [time, {"b"=>2}]]
183
- ], d.instance.outputs.map{ |o| o.events }
183
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
184
+ assert_nothing_raised do
185
+ d.run(default_tag: 'test') do
186
+ d.feed(time, {"a"=>1})
187
+ end
188
+ end
184
189
  end
185
190
  end
186
191