fluentd-hubspot 0.14.14.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (396) hide show
  1. data/.github/ISSUE_TEMPLATE.md +6 -0
  2. data/.gitignore +28 -0
  3. data/.travis.yml +51 -0
  4. data/AUTHORS +2 -0
  5. data/CONTRIBUTING.md +42 -0
  6. data/COPYING +14 -0
  7. data/ChangeLog +593 -0
  8. data/Gemfile +9 -0
  9. data/README.md +76 -0
  10. data/Rakefile +74 -0
  11. data/Vagrantfile +17 -0
  12. data/appveyor.yml +43 -0
  13. data/bin/fluent-binlog-reader +7 -0
  14. data/bin/fluent-debug +5 -0
  15. data/bin/fluent-plugin-config-format +5 -0
  16. data/bin/fluent-plugin-generate +5 -0
  17. data/code-of-conduct.md +3 -0
  18. data/example/copy_roundrobin.conf +39 -0
  19. data/example/filter_stdout.conf +22 -0
  20. data/example/in_dummy_blocks.conf +17 -0
  21. data/example/in_dummy_with_compression.conf +23 -0
  22. data/example/in_forward.conf +14 -0
  23. data/example/in_forward_client.conf +37 -0
  24. data/example/in_forward_shared_key.conf +15 -0
  25. data/example/in_forward_tls.conf +14 -0
  26. data/example/in_forward_users.conf +24 -0
  27. data/example/in_forward_workers.conf +21 -0
  28. data/example/in_http.conf +14 -0
  29. data/example/in_out_forward.conf +17 -0
  30. data/example/in_syslog.conf +15 -0
  31. data/example/in_tail.conf +14 -0
  32. data/example/in_tcp.conf +13 -0
  33. data/example/in_udp.conf +13 -0
  34. data/example/logevents.conf +25 -0
  35. data/example/multi_filters.conf +61 -0
  36. data/example/out_copy.conf +20 -0
  37. data/example/out_exec_filter.conf +42 -0
  38. data/example/out_file.conf +13 -0
  39. data/example/out_forward.conf +35 -0
  40. data/example/out_forward_buf_file.conf +23 -0
  41. data/example/out_forward_client.conf +109 -0
  42. data/example/out_forward_heartbeat_none.conf +16 -0
  43. data/example/out_forward_shared_key.conf +36 -0
  44. data/example/out_forward_tls.conf +18 -0
  45. data/example/out_forward_users.conf +65 -0
  46. data/example/out_null.conf +36 -0
  47. data/example/secondary_file.conf +41 -0
  48. data/example/suppress_config_dump.conf +7 -0
  49. data/example/v0_12_filter.conf +78 -0
  50. data/example/v1_literal_example.conf +36 -0
  51. data/fluent.conf +139 -0
  52. data/fluentd.gemspec +51 -0
  53. data/lib/fluent/agent.rb +163 -0
  54. data/lib/fluent/clock.rb +62 -0
  55. data/lib/fluent/command/binlog_reader.rb +234 -0
  56. data/lib/fluent/command/bundler_injection.rb +45 -0
  57. data/lib/fluent/command/cat.rb +330 -0
  58. data/lib/fluent/command/debug.rb +102 -0
  59. data/lib/fluent/command/fluentd.rb +301 -0
  60. data/lib/fluent/command/plugin_config_formatter.rb +258 -0
  61. data/lib/fluent/command/plugin_generator.rb +301 -0
  62. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  63. data/lib/fluent/compat/detach_process_mixin.rb +25 -0
  64. data/lib/fluent/compat/exec_util.rb +129 -0
  65. data/lib/fluent/compat/file_util.rb +54 -0
  66. data/lib/fluent/compat/filter.rb +68 -0
  67. data/lib/fluent/compat/formatter.rb +111 -0
  68. data/lib/fluent/compat/formatter_utils.rb +85 -0
  69. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  70. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  71. data/lib/fluent/compat/input.rb +59 -0
  72. data/lib/fluent/compat/output.rb +728 -0
  73. data/lib/fluent/compat/output_chain.rb +60 -0
  74. data/lib/fluent/compat/parser.rb +310 -0
  75. data/lib/fluent/compat/parser_utils.rb +40 -0
  76. data/lib/fluent/compat/propagate_default.rb +62 -0
  77. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  78. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  79. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  80. data/lib/fluent/compat/socket_util.rb +165 -0
  81. data/lib/fluent/compat/string_util.rb +34 -0
  82. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  83. data/lib/fluent/compat/type_converter.rb +90 -0
  84. data/lib/fluent/config.rb +56 -0
  85. data/lib/fluent/config/basic_parser.rb +123 -0
  86. data/lib/fluent/config/configure_proxy.rb +418 -0
  87. data/lib/fluent/config/dsl.rb +149 -0
  88. data/lib/fluent/config/element.rb +218 -0
  89. data/lib/fluent/config/error.rb +26 -0
  90. data/lib/fluent/config/literal_parser.rb +251 -0
  91. data/lib/fluent/config/parser.rb +107 -0
  92. data/lib/fluent/config/section.rb +223 -0
  93. data/lib/fluent/config/types.rb +136 -0
  94. data/lib/fluent/config/v1_parser.rb +190 -0
  95. data/lib/fluent/configurable.rb +200 -0
  96. data/lib/fluent/daemon.rb +15 -0
  97. data/lib/fluent/engine.rb +266 -0
  98. data/lib/fluent/env.rb +28 -0
  99. data/lib/fluent/error.rb +30 -0
  100. data/lib/fluent/event.rb +334 -0
  101. data/lib/fluent/event_router.rb +269 -0
  102. data/lib/fluent/filter.rb +21 -0
  103. data/lib/fluent/formatter.rb +23 -0
  104. data/lib/fluent/input.rb +21 -0
  105. data/lib/fluent/label.rb +46 -0
  106. data/lib/fluent/load.rb +35 -0
  107. data/lib/fluent/log.rb +546 -0
  108. data/lib/fluent/match.rb +178 -0
  109. data/lib/fluent/mixin.rb +31 -0
  110. data/lib/fluent/msgpack_factory.rb +62 -0
  111. data/lib/fluent/output.rb +29 -0
  112. data/lib/fluent/output_chain.rb +23 -0
  113. data/lib/fluent/parser.rb +23 -0
  114. data/lib/fluent/plugin.rb +183 -0
  115. data/lib/fluent/plugin/bare_output.rb +63 -0
  116. data/lib/fluent/plugin/base.rb +165 -0
  117. data/lib/fluent/plugin/buf_file.rb +184 -0
  118. data/lib/fluent/plugin/buf_memory.rb +34 -0
  119. data/lib/fluent/plugin/buffer.rb +617 -0
  120. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  121. data/lib/fluent/plugin/buffer/file_chunk.rb +364 -0
  122. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  123. data/lib/fluent/plugin/compressable.rb +92 -0
  124. data/lib/fluent/plugin/exec_util.rb +22 -0
  125. data/lib/fluent/plugin/file_util.rb +22 -0
  126. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  127. data/lib/fluent/plugin/filter.rb +93 -0
  128. data/lib/fluent/plugin/filter_grep.rb +75 -0
  129. data/lib/fluent/plugin/filter_parser.rb +119 -0
  130. data/lib/fluent/plugin/filter_record_transformer.rb +322 -0
  131. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  132. data/lib/fluent/plugin/formatter.rb +50 -0
  133. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  134. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  135. data/lib/fluent/plugin/formatter_json.rb +55 -0
  136. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  137. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  138. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  139. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  140. data/lib/fluent/plugin/formatter_stdout.rb +75 -0
  141. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  142. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  143. data/lib/fluent/plugin/in_dummy.rb +139 -0
  144. data/lib/fluent/plugin/in_exec.rb +108 -0
  145. data/lib/fluent/plugin/in_forward.rb +455 -0
  146. data/lib/fluent/plugin/in_gc_stat.rb +56 -0
  147. data/lib/fluent/plugin/in_http.rb +433 -0
  148. data/lib/fluent/plugin/in_monitor_agent.rb +448 -0
  149. data/lib/fluent/plugin/in_object_space.rb +93 -0
  150. data/lib/fluent/plugin/in_syslog.rb +209 -0
  151. data/lib/fluent/plugin/in_tail.rb +905 -0
  152. data/lib/fluent/plugin/in_tcp.rb +85 -0
  153. data/lib/fluent/plugin/in_udp.rb +81 -0
  154. data/lib/fluent/plugin/in_unix.rb +201 -0
  155. data/lib/fluent/plugin/input.rb +37 -0
  156. data/lib/fluent/plugin/multi_output.rb +157 -0
  157. data/lib/fluent/plugin/out_copy.rb +46 -0
  158. data/lib/fluent/plugin/out_exec.rb +105 -0
  159. data/lib/fluent/plugin/out_exec_filter.rb +317 -0
  160. data/lib/fluent/plugin/out_file.rb +302 -0
  161. data/lib/fluent/plugin/out_forward.rb +912 -0
  162. data/lib/fluent/plugin/out_null.rb +74 -0
  163. data/lib/fluent/plugin/out_relabel.rb +32 -0
  164. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  165. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  166. data/lib/fluent/plugin/out_stdout.rb +75 -0
  167. data/lib/fluent/plugin/out_stream.rb +130 -0
  168. data/lib/fluent/plugin/output.rb +1291 -0
  169. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  170. data/lib/fluent/plugin/parser.rb +191 -0
  171. data/lib/fluent/plugin/parser_apache.rb +28 -0
  172. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  173. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  174. data/lib/fluent/plugin/parser_csv.rb +39 -0
  175. data/lib/fluent/plugin/parser_json.rb +81 -0
  176. data/lib/fluent/plugin/parser_ltsv.rb +42 -0
  177. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  178. data/lib/fluent/plugin/parser_multiline.rb +105 -0
  179. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  180. data/lib/fluent/plugin/parser_none.rb +36 -0
  181. data/lib/fluent/plugin/parser_regexp.rb +63 -0
  182. data/lib/fluent/plugin/parser_syslog.rb +121 -0
  183. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  184. data/lib/fluent/plugin/socket_util.rb +22 -0
  185. data/lib/fluent/plugin/storage.rb +84 -0
  186. data/lib/fluent/plugin/storage_local.rb +159 -0
  187. data/lib/fluent/plugin/string_util.rb +22 -0
  188. data/lib/fluent/plugin_helper.rb +70 -0
  189. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  190. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  191. data/lib/fluent/plugin_helper/compat_parameters.rb +331 -0
  192. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  193. data/lib/fluent/plugin_helper/event_loop.rb +161 -0
  194. data/lib/fluent/plugin_helper/extract.rb +104 -0
  195. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  196. data/lib/fluent/plugin_helper/inject.rb +151 -0
  197. data/lib/fluent/plugin_helper/parser.rb +147 -0
  198. data/lib/fluent/plugin_helper/retry_state.rb +201 -0
  199. data/lib/fluent/plugin_helper/server.rb +738 -0
  200. data/lib/fluent/plugin_helper/socket.rb +241 -0
  201. data/lib/fluent/plugin_helper/socket_option.rb +69 -0
  202. data/lib/fluent/plugin_helper/storage.rb +349 -0
  203. data/lib/fluent/plugin_helper/thread.rb +179 -0
  204. data/lib/fluent/plugin_helper/timer.rb +91 -0
  205. data/lib/fluent/plugin_id.rb +80 -0
  206. data/lib/fluent/process.rb +22 -0
  207. data/lib/fluent/registry.rb +116 -0
  208. data/lib/fluent/root_agent.rb +323 -0
  209. data/lib/fluent/rpc.rb +94 -0
  210. data/lib/fluent/supervisor.rb +741 -0
  211. data/lib/fluent/system_config.rb +159 -0
  212. data/lib/fluent/test.rb +58 -0
  213. data/lib/fluent/test/base.rb +78 -0
  214. data/lib/fluent/test/driver/base.rb +224 -0
  215. data/lib/fluent/test/driver/base_owned.rb +70 -0
  216. data/lib/fluent/test/driver/base_owner.rb +135 -0
  217. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  218. data/lib/fluent/test/driver/filter.rb +57 -0
  219. data/lib/fluent/test/driver/formatter.rb +30 -0
  220. data/lib/fluent/test/driver/input.rb +31 -0
  221. data/lib/fluent/test/driver/multi_output.rb +53 -0
  222. data/lib/fluent/test/driver/output.rb +102 -0
  223. data/lib/fluent/test/driver/parser.rb +30 -0
  224. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  225. data/lib/fluent/test/filter_test.rb +77 -0
  226. data/lib/fluent/test/formatter_test.rb +65 -0
  227. data/lib/fluent/test/helpers.rb +134 -0
  228. data/lib/fluent/test/input_test.rb +174 -0
  229. data/lib/fluent/test/log.rb +79 -0
  230. data/lib/fluent/test/output_test.rb +156 -0
  231. data/lib/fluent/test/parser_test.rb +70 -0
  232. data/lib/fluent/test/startup_shutdown.rb +46 -0
  233. data/lib/fluent/time.rb +412 -0
  234. data/lib/fluent/timezone.rb +133 -0
  235. data/lib/fluent/unique_id.rb +39 -0
  236. data/lib/fluent/version.rb +21 -0
  237. data/lib/fluent/winsvc.rb +71 -0
  238. data/templates/new_gem/Gemfile +3 -0
  239. data/templates/new_gem/README.md.erb +43 -0
  240. data/templates/new_gem/Rakefile +13 -0
  241. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  242. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  243. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  244. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  245. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  246. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  247. data/templates/new_gem/test/helper.rb.erb +8 -0
  248. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  249. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  250. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  251. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  252. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  253. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  254. data/templates/plugin_config_formatter/param.md.erb +34 -0
  255. data/templates/plugin_config_formatter/section.md.erb +12 -0
  256. data/test/command/test_binlog_reader.rb +346 -0
  257. data/test/command/test_fluentd.rb +618 -0
  258. data/test/command/test_plugin_config_formatter.rb +275 -0
  259. data/test/command/test_plugin_generator.rb +66 -0
  260. data/test/compat/test_calls_super.rb +166 -0
  261. data/test/compat/test_parser.rb +92 -0
  262. data/test/config/assertions.rb +42 -0
  263. data/test/config/test_config_parser.rb +513 -0
  264. data/test/config/test_configurable.rb +1587 -0
  265. data/test/config/test_configure_proxy.rb +566 -0
  266. data/test/config/test_dsl.rb +415 -0
  267. data/test/config/test_element.rb +403 -0
  268. data/test/config/test_literal_parser.rb +297 -0
  269. data/test/config/test_section.rb +184 -0
  270. data/test/config/test_system_config.rb +168 -0
  271. data/test/config/test_types.rb +191 -0
  272. data/test/helper.rb +153 -0
  273. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  274. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  275. data/test/plugin/data/2010/01/20100102.log +0 -0
  276. data/test/plugin/data/log/bar +0 -0
  277. data/test/plugin/data/log/foo/bar.log +0 -0
  278. data/test/plugin/data/log/foo/bar2 +0 -0
  279. data/test/plugin/data/log/test.log +0 -0
  280. data/test/plugin/test_bare_output.rb +118 -0
  281. data/test/plugin/test_base.rb +115 -0
  282. data/test/plugin/test_buf_file.rb +843 -0
  283. data/test/plugin/test_buf_memory.rb +42 -0
  284. data/test/plugin/test_buffer.rb +1220 -0
  285. data/test/plugin/test_buffer_chunk.rb +198 -0
  286. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  287. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  288. data/test/plugin/test_compressable.rb +84 -0
  289. data/test/plugin/test_file_util.rb +96 -0
  290. data/test/plugin/test_filter.rb +357 -0
  291. data/test/plugin/test_filter_grep.rb +119 -0
  292. data/test/plugin/test_filter_parser.rb +700 -0
  293. data/test/plugin/test_filter_record_transformer.rb +556 -0
  294. data/test/plugin/test_filter_stdout.rb +202 -0
  295. data/test/plugin/test_formatter_csv.rb +111 -0
  296. data/test/plugin/test_formatter_hash.rb +35 -0
  297. data/test/plugin/test_formatter_json.rb +51 -0
  298. data/test/plugin/test_formatter_ltsv.rb +59 -0
  299. data/test/plugin/test_formatter_msgpack.rb +28 -0
  300. data/test/plugin/test_formatter_out_file.rb +95 -0
  301. data/test/plugin/test_formatter_single_value.rb +38 -0
  302. data/test/plugin/test_in_debug_agent.rb +28 -0
  303. data/test/plugin/test_in_dummy.rb +192 -0
  304. data/test/plugin/test_in_exec.rb +245 -0
  305. data/test/plugin/test_in_forward.rb +1120 -0
  306. data/test/plugin/test_in_gc_stat.rb +39 -0
  307. data/test/plugin/test_in_http.rb +588 -0
  308. data/test/plugin/test_in_monitor_agent.rb +516 -0
  309. data/test/plugin/test_in_object_space.rb +64 -0
  310. data/test/plugin/test_in_syslog.rb +271 -0
  311. data/test/plugin/test_in_tail.rb +1216 -0
  312. data/test/plugin/test_in_tcp.rb +118 -0
  313. data/test/plugin/test_in_udp.rb +152 -0
  314. data/test/plugin/test_in_unix.rb +126 -0
  315. data/test/plugin/test_input.rb +126 -0
  316. data/test/plugin/test_multi_output.rb +180 -0
  317. data/test/plugin/test_out_copy.rb +160 -0
  318. data/test/plugin/test_out_exec.rb +310 -0
  319. data/test/plugin/test_out_exec_filter.rb +613 -0
  320. data/test/plugin/test_out_file.rb +873 -0
  321. data/test/plugin/test_out_forward.rb +685 -0
  322. data/test/plugin/test_out_null.rb +105 -0
  323. data/test/plugin/test_out_relabel.rb +28 -0
  324. data/test/plugin/test_out_roundrobin.rb +146 -0
  325. data/test/plugin/test_out_secondary_file.rb +442 -0
  326. data/test/plugin/test_out_stdout.rb +170 -0
  327. data/test/plugin/test_out_stream.rb +93 -0
  328. data/test/plugin/test_output.rb +870 -0
  329. data/test/plugin/test_output_as_buffered.rb +1932 -0
  330. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  331. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  332. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  333. data/test/plugin/test_output_as_buffered_secondary.rb +877 -0
  334. data/test/plugin/test_output_as_standard.rb +374 -0
  335. data/test/plugin/test_owned_by.rb +35 -0
  336. data/test/plugin/test_parser.rb +359 -0
  337. data/test/plugin/test_parser_apache.rb +42 -0
  338. data/test/plugin/test_parser_apache2.rb +46 -0
  339. data/test/plugin/test_parser_apache_error.rb +45 -0
  340. data/test/plugin/test_parser_csv.rb +103 -0
  341. data/test/plugin/test_parser_json.rb +114 -0
  342. data/test/plugin/test_parser_labeled_tsv.rb +128 -0
  343. data/test/plugin/test_parser_multiline.rb +100 -0
  344. data/test/plugin/test_parser_nginx.rb +48 -0
  345. data/test/plugin/test_parser_none.rb +52 -0
  346. data/test/plugin/test_parser_regexp.rb +281 -0
  347. data/test/plugin/test_parser_syslog.rb +242 -0
  348. data/test/plugin/test_parser_tsv.rb +122 -0
  349. data/test/plugin/test_storage.rb +167 -0
  350. data/test/plugin/test_storage_local.rb +335 -0
  351. data/test/plugin/test_string_util.rb +26 -0
  352. data/test/plugin_helper/test_child_process.rb +794 -0
  353. data/test/plugin_helper/test_compat_parameters.rb +331 -0
  354. data/test/plugin_helper/test_event_emitter.rb +51 -0
  355. data/test/plugin_helper/test_event_loop.rb +52 -0
  356. data/test/plugin_helper/test_extract.rb +194 -0
  357. data/test/plugin_helper/test_formatter.rb +255 -0
  358. data/test/plugin_helper/test_inject.rb +519 -0
  359. data/test/plugin_helper/test_parser.rb +264 -0
  360. data/test/plugin_helper/test_retry_state.rb +422 -0
  361. data/test/plugin_helper/test_server.rb +1677 -0
  362. data/test/plugin_helper/test_storage.rb +542 -0
  363. data/test/plugin_helper/test_thread.rb +164 -0
  364. data/test/plugin_helper/test_timer.rb +132 -0
  365. data/test/scripts/exec_script.rb +32 -0
  366. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  367. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  368. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  369. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  370. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  371. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  372. data/test/test_clock.rb +164 -0
  373. data/test/test_config.rb +179 -0
  374. data/test/test_configdsl.rb +148 -0
  375. data/test/test_event.rb +515 -0
  376. data/test/test_event_router.rb +331 -0
  377. data/test/test_event_time.rb +186 -0
  378. data/test/test_filter.rb +121 -0
  379. data/test/test_formatter.rb +312 -0
  380. data/test/test_input.rb +31 -0
  381. data/test/test_log.rb +828 -0
  382. data/test/test_match.rb +137 -0
  383. data/test/test_mixin.rb +351 -0
  384. data/test/test_output.rb +273 -0
  385. data/test/test_plugin.rb +251 -0
  386. data/test/test_plugin_classes.rb +253 -0
  387. data/test/test_plugin_helper.rb +81 -0
  388. data/test/test_plugin_id.rb +101 -0
  389. data/test/test_process.rb +14 -0
  390. data/test/test_root_agent.rb +611 -0
  391. data/test/test_supervisor.rb +373 -0
  392. data/test/test_test_drivers.rb +135 -0
  393. data/test/test_time_formatter.rb +282 -0
  394. data/test/test_time_parser.rb +211 -0
  395. data/test/test_unique_id.rb +47 -0
  396. metadata +898 -0
@@ -0,0 +1,170 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_stdout'
4
+
5
+ class StdoutOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ CONFIG = %[
11
+ ]
12
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
13
+
14
+ def create_driver(conf = CONFIG)
15
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
16
+ end
17
+
18
+ sub_test_case 'non-buffered' do
19
+ test 'configure' do
20
+ d = create_driver
21
+ assert_equal 1, d.instance.formatter_configs.size # init: true
22
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
23
+ assert_equal 'json', d.instance.formatter.output_type
24
+ end
25
+
26
+ test 'configure output_type' do
27
+ d = create_driver(CONFIG + "\noutput_type json")
28
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
29
+ assert_equal 'json', d.instance.formatter.output_type
30
+
31
+ d = create_driver(CONFIG + "\noutput_type hash")
32
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
33
+ assert_equal 'hash', d.instance.formatter.output_type
34
+
35
+ assert_raise(Fluent::ConfigError) do
36
+ d = create_driver(CONFIG + "\noutput_type foo")
37
+ end
38
+ end
39
+
40
+ test 'emit with default configuration' do
41
+ d = create_driver
42
+ time = event_time()
43
+ out = capture_log do
44
+ d.run(default_tag: 'test') do
45
+ d.feed(time, {'test' => 'test1'})
46
+ end
47
+ end
48
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
49
+ end
50
+
51
+ data('oj' => 'oj', 'yajl' => 'yajl')
52
+ test 'emit in json format' do |data|
53
+ d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
54
+ time = event_time()
55
+ out = capture_log do
56
+ d.run(default_tag: 'test') do
57
+ d.feed(time, {'test' => 'test1'})
58
+ end
59
+ end
60
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
61
+
62
+ if data == 'yajl'
63
+ # NOTE: Float::NAN is not jsonable
64
+ assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
65
+ else
66
+ out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
67
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":NaN}\n", out
68
+ end
69
+ end
70
+
71
+ test 'emit in hash format' do
72
+ d = create_driver(CONFIG + "\noutput_type hash")
73
+ time = event_time()
74
+ out = capture_log do
75
+ d.run(default_tag: 'test') do
76
+ d.feed(time, {'test' => 'test2'})
77
+ end
78
+ end
79
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out
80
+
81
+ # NOTE: Float::NAN is not jsonable, but hash string can output it.
82
+ out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
83
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out
84
+ end
85
+ end
86
+
87
+ sub_test_case 'buffered' do
88
+ test 'configure' do
89
+ d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
90
+ assert_equal 1, d.instance.formatter_configs.size
91
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
92
+ assert_equal 'json', d.instance.formatter.output_type
93
+ assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
94
+ assert d.instance.buffer_config.flush_at_shutdown
95
+ assert_equal ['tag'], d.instance.buffer_config.chunk_keys
96
+ assert d.instance.chunk_key_tag
97
+ assert !d.instance.chunk_key_time
98
+ assert_equal [], d.instance.chunk_keys
99
+ end
100
+
101
+ test 'configure with output_type' do
102
+ d = create_driver(config_element("ROOT", "", {"output_type" => "json"}, [config_element("buffer")]))
103
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
104
+ assert_equal 'json', d.instance.formatter.output_type
105
+
106
+ d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
107
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
108
+ assert_equal 'hash', d.instance.formatter.output_type
109
+
110
+ assert_raise(Fluent::ConfigError) do
111
+ create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
112
+ end
113
+ end
114
+
115
+ sub_test_case "emit with default config" do
116
+ test '#write(synchronous)' do
117
+ d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
118
+ time = event_time()
119
+
120
+ out = capture_log do
121
+ d.run(default_tag: 'test', flush: true) do
122
+ d.feed(time, {'test' => 'test'})
123
+ end
124
+ end
125
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
126
+ end
127
+ end
128
+
129
+ sub_test_case "emit json" do
130
+ data('oj' => 'oj', 'yajl' => 'yajl')
131
+ test '#write(synchronous)' do |data|
132
+ d = create_driver(config_element("ROOT", "", {"output_type" => "json", "json_parser" => data}, [config_element("buffer")]))
133
+ time = event_time()
134
+
135
+ out = capture_log do
136
+ d.run(default_tag: 'test', flush: true) do
137
+ d.feed(time, {'test' => 'test'})
138
+ end
139
+ end
140
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
141
+ end
142
+ end
143
+
144
+ sub_test_case 'emit hash' do
145
+ test '#write(synchronous)' do
146
+ d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
147
+ time = event_time()
148
+
149
+ out = capture_log do
150
+ d.run(default_tag: 'test', flush: true) do
151
+ d.feed(time, {'test' => 'test'})
152
+ end
153
+ end
154
+
155
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
156
+ end
157
+ end
158
+ end
159
+
160
+ # Capture the log output of the block given
161
+ def capture_log(&block)
162
+ tmp = $log
163
+ $log = StringIO.new
164
+ yield
165
+ return $log.string
166
+ ensure
167
+ $log = tmp
168
+ end
169
+ end
170
+
@@ -0,0 +1,93 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test'
3
+ require 'fluent/plugin/out_stream'
4
+
5
+ module StreamOutputTest
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def test_write
11
+ d = create_driver
12
+
13
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
14
+ d.emit({"a"=>1}, time)
15
+ d.emit({"a"=>2}, time)
16
+
17
+ expect = ["test",
18
+ [time,{"a"=>1}].to_msgpack +
19
+ [time,{"a"=>2}].to_msgpack
20
+ ].to_msgpack
21
+
22
+ result = d.run
23
+ assert_equal(expect, result)
24
+ end
25
+
26
+ def test_write_event_time
27
+ d = create_driver
28
+
29
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
30
+ d.emit({"a"=>1}, time)
31
+ d.emit({"a"=>2}, time)
32
+
33
+ expect = ["test",
34
+ Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>1}]).to_s +
35
+ Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>2}]).to_s
36
+ ]
37
+ expect = Fluent::Engine.msgpack_factory.packer.write(expect).to_s
38
+
39
+ result = d.run
40
+ assert_equal(expect, result)
41
+ end
42
+
43
+ def create_driver(klass, conf)
44
+ Fluent::Test::BufferedOutputTestDriver.new(klass) do
45
+ def write(chunk)
46
+ chunk.read
47
+ end
48
+ end.configure(conf)
49
+ end
50
+ end
51
+
52
+ class TcpOutputTest < Test::Unit::TestCase
53
+ include StreamOutputTest
54
+
55
+ PORT = unused_port
56
+ CONFIG = %[
57
+ port #{PORT}
58
+ host 127.0.0.1
59
+ send_timeout 51
60
+ ]
61
+
62
+ def create_driver(conf=CONFIG)
63
+ super(Fluent::TcpOutput, conf)
64
+ end
65
+
66
+ def test_configure
67
+ d = create_driver
68
+ assert_equal PORT, d.instance.port
69
+ assert_equal '127.0.0.1', d.instance.host
70
+ assert_equal 51, d.instance.send_timeout
71
+ end
72
+ end
73
+
74
+ class UnixOutputTest < Test::Unit::TestCase
75
+ include StreamOutputTest
76
+
77
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_unix#{ENV['TEST_ENV_NUMBER']}"
78
+ CONFIG = %[
79
+ path #{TMP_DIR}/unix
80
+ send_timeout 52
81
+ ]
82
+
83
+ def create_driver(conf=CONFIG)
84
+ super(Fluent::UnixOutput, conf)
85
+ end
86
+
87
+ def test_configure
88
+ d = create_driver
89
+ assert_equal "#{TMP_DIR}/unix", d.instance.path
90
+ assert_equal 52, d.instance.send_timeout
91
+ end
92
+ end
93
+
@@ -0,0 +1,870 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/event'
5
+
6
+ require 'json'
7
+ require 'time'
8
+ require 'timeout'
9
+
10
+ module FluentPluginOutputTest
11
+ class DummyBareOutput < Fluent::Plugin::Output
12
+ def register(name, &block)
13
+ instance_variable_set("@#{name}", block)
14
+ end
15
+ end
16
+ class DummySyncOutput < DummyBareOutput
17
+ def initialize
18
+ super
19
+ @process = nil
20
+ end
21
+ def process(tag, es)
22
+ @process ? @process.call(tag, es) : nil
23
+ end
24
+ end
25
+ class DummyAsyncOutput < DummyBareOutput
26
+ def initialize
27
+ super
28
+ @format = nil
29
+ @write = nil
30
+ end
31
+ def format(tag, time, record)
32
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
33
+ end
34
+ def write(chunk)
35
+ @write ? @write.call(chunk) : nil
36
+ end
37
+ end
38
+ class DummyAsyncStandardOutput < DummyBareOutput
39
+ def initialize
40
+ super
41
+ @write = nil
42
+ end
43
+ def write(chunk)
44
+ @write ? @write.call(chunk) : nil
45
+ end
46
+ end
47
+ class DummyDelayedOutput < DummyBareOutput
48
+ def initialize
49
+ super
50
+ @format = nil
51
+ @try_write = nil
52
+ end
53
+ def format(tag, time, record)
54
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
55
+ end
56
+ def try_write(chunk)
57
+ @try_write ? @try_write.call(chunk) : nil
58
+ end
59
+ end
60
+ class DummyDelayedStandardOutput < DummyBareOutput
61
+ def initialize
62
+ super
63
+ @try_write = nil
64
+ end
65
+ def try_write(chunk)
66
+ @try_write ? @try_write.call(chunk) : nil
67
+ end
68
+ end
69
+ class DummyFullFeatureOutput < DummyBareOutput
70
+ def initialize
71
+ super
72
+ @prefer_buffered_processing = nil
73
+ @prefer_delayed_commit = nil
74
+ @process = nil
75
+ @format = nil
76
+ @write = nil
77
+ @try_write = nil
78
+ end
79
+ def prefer_buffered_processing
80
+ @prefer_buffered_processing ? @prefer_buffered_processing.call : false
81
+ end
82
+ def prefer_delayed_commit
83
+ @prefer_delayed_commit ? @prefer_delayed_commit.call : false
84
+ end
85
+ def process(tag, es)
86
+ @process ? @process.call(tag, es) : nil
87
+ end
88
+ def format(tag, time, record)
89
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
90
+ end
91
+ def write(chunk)
92
+ @write ? @write.call(chunk) : nil
93
+ end
94
+ def try_write(chunk)
95
+ @try_write ? @try_write.call(chunk) : nil
96
+ end
97
+ end
98
+ end
99
+
100
+ class OutputTest < Test::Unit::TestCase
101
+ class << self
102
+ def startup
103
+ $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))
104
+ require 'fluent/plugin/out_test'
105
+ end
106
+
107
+ def shutdown
108
+ $LOAD_PATH.shift
109
+ end
110
+ end
111
+
112
+ def create_output(type=:full)
113
+ case type
114
+ when :bare then FluentPluginOutputTest::DummyBareOutput.new
115
+ when :sync then FluentPluginOutputTest::DummySyncOutput.new
116
+ when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new
117
+ when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new
118
+ when :delayed then FluentPluginOutputTest::DummyDelayedOutput.new
119
+ when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new
120
+ when :full then FluentPluginOutputTest::DummyFullFeatureOutput.new
121
+ else
122
+ raise ArgumentError, "unknown type: #{type}"
123
+ end
124
+ end
125
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
126
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
127
+ end
128
+ def waiting(seconds)
129
+ begin
130
+ Timeout.timeout(seconds) do
131
+ yield
132
+ end
133
+ rescue Timeout::Error
134
+ STDERR.print(*@i.log.out.logs)
135
+ raise
136
+ end
137
+ end
138
+
139
+ sub_test_case 'basic output feature' do
140
+ setup do
141
+ @i = create_output(:full)
142
+ end
143
+
144
+ test 'are not available with multi workers configuration in default' do
145
+ assert_false @i.multi_workers_ready?
146
+ end
147
+
148
+ test '#implement? can return features for plugin instances' do
149
+ i1 = FluentPluginOutputTest::DummyBareOutput.new
150
+ assert !i1.implement?(:synchronous)
151
+ assert !i1.implement?(:buffered)
152
+ assert !i1.implement?(:delayed_commit)
153
+ assert !i1.implement?(:custom_format)
154
+
155
+ i2 = FluentPluginOutputTest::DummySyncOutput.new
156
+ assert i2.implement?(:synchronous)
157
+ assert !i2.implement?(:buffered)
158
+ assert !i2.implement?(:delayed_commit)
159
+ assert !i2.implement?(:custom_format)
160
+
161
+ i3 = FluentPluginOutputTest::DummyAsyncOutput.new
162
+ assert !i3.implement?(:synchronous)
163
+ assert i3.implement?(:buffered)
164
+ assert !i3.implement?(:delayed_commit)
165
+ assert i3.implement?(:custom_format)
166
+
167
+ i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new
168
+ assert !i4.implement?(:synchronous)
169
+ assert i4.implement?(:buffered)
170
+ assert !i4.implement?(:delayed_commit)
171
+ assert !i4.implement?(:custom_format)
172
+
173
+ i5 = FluentPluginOutputTest::DummyDelayedOutput.new
174
+ assert !i5.implement?(:synchronous)
175
+ assert !i5.implement?(:buffered)
176
+ assert i5.implement?(:delayed_commit)
177
+ assert i5.implement?(:custom_format)
178
+
179
+ i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new
180
+ assert !i6.implement?(:synchronous)
181
+ assert !i6.implement?(:buffered)
182
+ assert i6.implement?(:delayed_commit)
183
+ assert !i6.implement?(:custom_format)
184
+
185
+ i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new
186
+ assert i6.implement?(:synchronous)
187
+ assert i6.implement?(:buffered)
188
+ assert i6.implement?(:delayed_commit)
189
+ assert i6.implement?(:custom_format)
190
+ end
191
+
192
+ test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do
193
+ assert !@i.configured?
194
+ @i.configure(config_element())
195
+ assert @i.configured?
196
+ assert !@i.started?
197
+ @i.start
198
+ assert @i.started?
199
+ assert !@i.after_started?
200
+ @i.after_start
201
+ assert @i.after_started?
202
+ assert !@i.stopped?
203
+ @i.stop
204
+ assert @i.stopped?
205
+ assert !@i.before_shutdown?
206
+ @i.before_shutdown
207
+ assert @i.before_shutdown?
208
+ assert !@i.shutdown?
209
+ @i.shutdown
210
+ assert @i.shutdown?
211
+ assert !@i.after_shutdown?
212
+ @i.after_shutdown
213
+ assert @i.after_shutdown?
214
+ assert !@i.closed?
215
+ @i.close
216
+ assert @i.closed?
217
+ assert !@i.terminated?
218
+ @i.terminate
219
+ assert @i.terminated?
220
+ end
221
+
222
+ test '#extract_placeholders does nothing if chunk key is not specified' do
223
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
224
+ assert !@i.chunk_key_time
225
+ assert !@i.chunk_key_tag
226
+ assert_equal [], @i.chunk_keys
227
+ tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
228
+ t = event_time('2016-04-11 20:30:00 +0900')
229
+ v = {key1: "value1", key2: "value2"}
230
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
231
+ assert_equal tmpl, @i.extract_placeholders(tmpl, m)
232
+ end
233
+
234
+ test '#extract_placeholders can extract time if time key and range are configured' do
235
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
236
+ assert @i.chunk_key_time
237
+ assert !@i.chunk_key_tag
238
+ assert_equal [], @i.chunk_keys
239
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
240
+ t = event_time('2016-04-11 20:30:00 +0900')
241
+ v = {key1: "value1", key2: "value2"}
242
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
243
+ assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
244
+ end
245
+
246
+ test '#extract_placeholders can extract tag and parts of tag if tag is configured' do
247
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
248
+ assert !@i.chunk_key_time
249
+ assert @i.chunk_key_tag
250
+ assert_equal [], @i.chunk_keys
251
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
252
+ t = event_time('2016-04-11 20:30:00 +0900')
253
+ v = {key1: "value1", key2: "value2"}
254
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
255
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
256
+ end
257
+
258
+ test '#extract_placeholders can extract variables if variables are configured' do
259
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
260
+ assert !@i.chunk_key_time
261
+ assert !@i.chunk_key_tag
262
+ assert_equal ['key1','key2'], @i.chunk_keys
263
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
264
+ t = event_time('2016-04-11 20:30:00 +0900')
265
+ v = {key1: "value1", key2: "value2"}
266
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
267
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
268
+ end
269
+
270
+ test '#extract_placeholders can extract all chunk keys if configured' do
271
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
272
+ assert @i.chunk_key_time
273
+ assert @i.chunk_key_tag
274
+ assert_equal ['key1','key2'], @i.chunk_keys
275
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
276
+ t = event_time('2016-04-11 20:30:00 +0900')
277
+ v = {key1: "value1", key2: "value2"}
278
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
279
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m)
280
+ end
281
+
282
+ test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do
283
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
284
+ assert @i.chunk_key_time
285
+ assert @i.chunk_key_tag
286
+ assert_equal ['key1','key2'], @i.chunk_keys
287
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
288
+ t = event_time('2016-04-11 20:30:00 +0900')
289
+ v = {key1: "value1", key2: "value2"}
290
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
291
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
292
+ end
293
+
294
+ sub_test_case '#placeholder_validators' do
295
+ test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
296
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
297
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
298
+ assert_equal 3, validators.size
299
+ assert_equal 1, validators.select(&:time?).size
300
+ assert_equal 1, validators.select(&:tag?).size
301
+ assert_equal 1, validators.select(&:keys?).size
302
+ end
303
+
304
+ test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
305
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
306
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
307
+ assert_equal 3, validators.size
308
+ assert_equal 1, validators.select(&:time?).size
309
+ assert_equal 1, validators.select(&:tag?).size
310
+ assert_equal 1, validators.select(&:keys?).size
311
+ end
312
+
313
+ test 'returns a validator for time if a template has timestamp placeholders' do
314
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
315
+ validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
316
+ assert_equal 1, validators.size
317
+ assert_equal 1, validators.select(&:time?).size
318
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
319
+ validators.first.validate!
320
+ end
321
+ end
322
+
323
+ test 'returns a validator for time if a plugin is configured with time key' do
324
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
325
+ validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
326
+ assert_equal 1, validators.size
327
+ assert_equal 1, validators.select(&:time?).size
328
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
329
+ validators.first.validate!
330
+ end
331
+ end
332
+
333
+ test 'returns a validator for tag if a template has tag placeholders' do
334
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
335
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
336
+ assert_equal 1, validators.size
337
+ assert_equal 1, validators.select(&:tag?).size
338
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
339
+ validators.first.validate!
340
+ end
341
+ end
342
+
343
+ test 'returns a validator for tag if a plugin is configured with tag key' do
344
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
345
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
346
+ assert_equal 1, validators.size
347
+ assert_equal 1, validators.select(&:tag?).size
348
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
349
+ validators.first.validate!
350
+ end
351
+ end
352
+
353
+ test 'returns a validator for variable keys if a template has variable placeholders' do
354
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
355
+ validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
356
+ assert_equal 1, validators.size
357
+ assert_equal 1, validators.select(&:keys?).size
358
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
359
+ validators.first.validate!
360
+ end
361
+ end
362
+
363
+ test 'returns a validator for variable keys if a plugin is configured with variable keys' do
364
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
365
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
366
+ assert_equal 1, validators.size
367
+ assert_equal 1, validators.select(&:keys?).size
368
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
369
+ validators.first.validate!
370
+ end
371
+ end
372
+ end
373
+
374
+ sub_test_case '#placeholder_validate!' do
375
+ test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
376
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
377
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
378
+ @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
379
+ end
380
+ end
381
+
382
+ test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
383
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
384
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have timestamp placeholders for timekey 180") do
385
+ @i.placeholder_validate!(:path, "/my/path/file.log")
386
+ end
387
+ assert_nothing_raised do
388
+ @i.placeholder_validate!(:path, "/my/path/%Y%m%d/file.%H%M.log")
389
+ end
390
+ end
391
+
392
+ test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
393
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
394
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y%m%d_%H.log' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
395
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
396
+ end
397
+ assert_nothing_raised do
398
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H%M.log")
399
+ end
400
+ end
401
+
402
+ test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
403
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
404
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.${tag[2]}.log' has tag placeholders, but chunk key 'tag' is not configured") do
405
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
406
+ end
407
+ end
408
+
409
+ test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
410
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
411
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
412
+ @i.placeholder_validate!(:path, "/my/path/file.log")
413
+ end
414
+ assert_nothing_raised do
415
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
416
+ end
417
+ end
418
+
419
+ test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
420
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
421
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.${username}.log' has placeholders, but chunk keys doesn't have keys service,username") do
422
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
423
+ end
424
+ end
425
+
426
+ test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
427
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
428
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys service,username") do
429
+ @i.placeholder_validate!(:path, "/my/path/file.log")
430
+ end
431
+ assert_nothing_raised do
432
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
433
+ end
434
+ end
435
+
436
+ test 'raise configuration error for a template and configuration with keys mismatch' do
437
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
438
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.${username}.log' doesn't have enough placeholders for keys service") do
439
+ @i.placeholder_validate!(:path, "/my/path/file.${username}.log")
440
+ end
441
+ assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.log' doesn't have enough placeholders for keys username") do
442
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.log")
443
+ end
444
+ assert_nothing_raised do
445
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
446
+ end
447
+ end
448
+ end
449
+
450
+ test '#get_placeholders_time returns seconds,title and example placeholder for a template' do
451
+ s, t, e = @i.get_placeholders_time("/path/to/dir/yay")
452
+ assert_nil s
453
+ assert_nil t
454
+ assert_nil e
455
+
456
+ s, t, e = @i.get_placeholders_time("/path/to/%Y%m%d/yay")
457
+ assert_equal 86400, s
458
+ assert_equal :day, t
459
+ assert_equal '%d', e
460
+ s, t, e = @i.get_placeholders_time("my birthiday! at %F")
461
+ assert_equal 86400, s
462
+ assert_equal :day, t
463
+ assert_equal '%d', e
464
+
465
+ s, t, e = @i.get_placeholders_time("myfile.%Y-%m-%d_%H.log")
466
+ assert_equal 3600, s
467
+ assert_equal :hour, t
468
+ assert_equal '%H', e
469
+
470
+ s, t, e = @i.get_placeholders_time("part-%Y%m%d-%H%M.ts")
471
+ assert_equal 60, s
472
+ assert_equal :minute, t
473
+ assert_equal '%M', e
474
+
475
+ s, t, e = @i.get_placeholders_time("my first data at %F %T %z")
476
+ assert_equal 1, s
477
+ assert_equal :second, t
478
+ assert_equal '%S', e
479
+ end
480
+
481
+ test '#get_placeholders_tag returns a list of tag part position for a template' do
482
+ assert_equal [], @i.get_placeholders_tag("db.table")
483
+ assert_equal [], @i.get_placeholders_tag("db.table_${non_tag}")
484
+ assert_equal [-1], @i.get_placeholders_tag("table_${tag}")
485
+ assert_equal [0, 1], @i.get_placeholders_tag("db_${tag[0]}.table_${tag[1]}")
486
+ assert_equal [-1, 0], @i.get_placeholders_tag("/treedir/${tag[0]}/${tag}")
487
+ end
488
+
489
+ test '#get_placeholders_keys returns a list of keys for a template' do
490
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/data/file.log")
491
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/${tag}/file.log")
492
+ assert_equal ['key1', 'key2'], @i.get_placeholders_keys("/path/to/${key2}/${tag}/file.${key1}.log")
493
+ assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
494
+ end
495
+
496
+ test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
497
+ tag = 'test.output'
498
+ time = event_time('2016-04-12 15:31:23 -0700')
499
+ timekey = event_time('2016-04-12 15:00:00 -0700')
500
+ record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
501
+
502
+ i1 = create_output(:buffered)
503
+ i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
504
+ assert_equal create_metadata(), i1.metadata(tag, time, record)
505
+
506
+ i2 = create_output(:buffered)
507
+ i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))
508
+ assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)
509
+
510
+ i3 = create_output(:buffered)
511
+ i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
512
+ assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)
513
+
514
+ i4 = create_output(:buffered)
515
+ i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))
516
+ assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record)
517
+
518
+ i5 = create_output(:buffered)
519
+ i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))
520
+ assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record)
521
+
522
+ i6 = create_output(:buffered)
523
+ i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
524
+ assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)
525
+
526
+ i7 = create_output(:buffered)
527
+ i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
528
+ assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)
529
+
530
+ i8 = create_output(:buffered)
531
+ i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
532
+ assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
533
+ end
534
+
535
+ test '#emit calls #process via #emit_sync for non-buffered output' do
536
+ i = create_output(:sync)
537
+ process_called = false
538
+ i.register(:process){|tag, es| process_called = true }
539
+ i.configure(config_element())
540
+ i.start
541
+ i.after_start
542
+
543
+ t = event_time()
544
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
545
+
546
+ assert process_called
547
+
548
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
549
+ end
550
+
551
+ test '#emit calls #format for buffered output' do
552
+ i = create_output(:buffered)
553
+ format_called_times = 0
554
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
555
+ i.configure(config_element())
556
+ i.start
557
+ i.after_start
558
+
559
+ t = event_time()
560
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
561
+
562
+ assert_equal 2, format_called_times
563
+
564
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
565
+ end
566
+
567
+ test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do
568
+ i = create_output(:full)
569
+
570
+ process_called = false
571
+ format_called_times = 0
572
+ i.register(:process){|tag, es| process_called = true }
573
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
574
+
575
+ i.configure(config_element())
576
+ i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
577
+ i.start
578
+ i.after_start
579
+
580
+ assert !i.prefer_buffered_processing
581
+
582
+ t = event_time()
583
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
584
+
585
+ waiting(4){ Thread.pass until process_called }
586
+
587
+ assert process_called
588
+ assert_equal 0, format_called_times
589
+
590
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
591
+ end
592
+
593
+ test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do
594
+ i = create_output(:full)
595
+
596
+ process_called = false
597
+ format_called_times = 0
598
+ i.register(:process){|tag, es| process_called = true }
599
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
600
+
601
+ i.configure(config_element())
602
+ i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
603
+ i.start
604
+ i.after_start
605
+
606
+ assert i.prefer_buffered_processing
607
+
608
+ t = event_time()
609
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
610
+
611
+ assert !process_called
612
+ assert_equal 2, format_called_times
613
+
614
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
615
+ end
616
+
617
+ test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do
618
+ i = create_output(:buffered)
619
+ write_called = false
620
+ i.register(:write){ |chunk| write_called = true }
621
+
622
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
623
+ i.start
624
+ i.after_start
625
+
626
+ t = event_time()
627
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
628
+ i.force_flush
629
+
630
+ waiting(4){ Thread.pass until write_called }
631
+
632
+ assert write_called
633
+
634
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
635
+ end
636
+
637
+ test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
638
+ i = create_output(:delayed)
639
+ try_write_called = false
640
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
641
+
642
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
643
+ i.start
644
+ i.after_start
645
+
646
+ t = event_time()
647
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
648
+ i.force_flush
649
+
650
+ waiting(4){ Thread.pass until try_write_called }
651
+
652
+ assert try_write_called
653
+
654
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
655
+ end
656
+
657
+ test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
658
+ i = create_output(:full)
659
+ write_called = false
660
+ try_write_called = false
661
+ i.register(:write){ |chunk| write_called = true }
662
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
663
+
664
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
665
+ i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
666
+ i.start
667
+ i.after_start
668
+
669
+ assert !i.prefer_delayed_commit
670
+
671
+ t = event_time()
672
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
673
+ i.force_flush
674
+
675
+ waiting(4){ Thread.pass until write_called || try_write_called }
676
+
677
+ assert write_called
678
+ assert !try_write_called
679
+
680
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
681
+ end
682
+
683
+ test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
684
+ i = create_output(:full)
685
+ write_called = false
686
+ try_write_called = false
687
+ i.register(:write){ |chunk| write_called = true }
688
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
689
+
690
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
691
+ i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
692
+ i.start
693
+ i.after_start
694
+
695
+ assert i.prefer_delayed_commit
696
+
697
+ t = event_time()
698
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
699
+ i.force_flush
700
+
701
+ waiting(4){ Thread.pass until write_called || try_write_called }
702
+
703
+ assert !write_called
704
+ assert try_write_called
705
+
706
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
707
+ end
708
+
709
+ test 'flush_interval is ignored when flush_mode is not interval' do
710
+ mock(@i.log).warn("'flush_interval' is ignored because default 'flush_mode' is not 'interval': 'lazy'")
711
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'flush_interval' => 10})]))
712
+ end
713
+
714
+ data(:lazy => 'lazy', :immediate => 'immediate')
715
+ test 'flush_interval and non-interval flush_mode is exclusive ' do |mode|
716
+ assert_raise Fluent::ConfigError.new("'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{mode}'") do
717
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'flush_mode' => mode, 'flush_interval' => 10})]))
718
+ end
719
+ end
720
+
721
+ test 'flush_mode is set to interval when flush_interval with v0.12 configuration is given' do
722
+ mock(@i.log).info("'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour")
723
+ @i.configure(config_element('ROOT', '', {'flush_interval' => 60}, []))
724
+ assert_equal :interval, @i.instance_variable_get(:@flush_mode)
725
+ end
726
+
727
+ test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
728
+ o = create_output(:buffered)
729
+ mock(o.log).warn("secondary type should be same with primary one",
730
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
731
+
732
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
733
+ assert_not_nil o.instance_variable_get(:@secondary)
734
+ end
735
+
736
+ test "don't warn if primary type is the same as secondary type" do
737
+ o = Fluent::Plugin::TestOutput.new
738
+ mock(o.log).warn("secondary type should be same with primary one",
739
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
740
+
741
+ o.configure(config_element('ROOT','',{'name' => "cool2"},
742
+ [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
743
+ config_element('buffer','',{'@type'=>'memory'})]
744
+ ))
745
+ assert_not_nil o.instance_variable_get(:@secondary)
746
+ end
747
+
748
+ test "don't warn if primary type is different from secondary type and both don't have custom_format" do
749
+ o = create_output(:standard)
750
+ mock(o.log).warn("secondary type should be same with primary one",
751
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
752
+
753
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
754
+ assert_not_nil o.instance_variable_get(:@secondary)
755
+ end
756
+ end
757
+
758
+ sub_test_case 'sync output feature' do
759
+ setup do
760
+ @i = create_output(:sync)
761
+ end
762
+
763
+ test 'raises configuration error if <buffer> section is specified' do
764
+ assert_raise Fluent::ConfigError do
765
+ @i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
766
+ end
767
+ end
768
+
769
+ test 'raises configuration error if <secondary> section is specified' do
770
+ assert_raise Fluent::ConfigError do
771
+ @i.configure(config_element('ROOT','',{},[config_element('secondary','')]))
772
+ end
773
+ end
774
+
775
+ test '#process is called for each event streams' do
776
+ ary = []
777
+ @i.register(:process){|tag, es| ary << [tag, es] }
778
+ @i.configure(config_element())
779
+ @i.start
780
+ @i.after_start
781
+
782
+ t = event_time()
783
+ es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
784
+ 5.times do
785
+ @i.emit_events('tag', es)
786
+ end
787
+ assert_equal 5, ary.size
788
+
789
+ @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
790
+ end
791
+ end
792
+
793
+ sub_test_case '#generate_format_proc' do
794
+ test "when output doesn't have <buffer>" do
795
+ i = create_output(:sync)
796
+ i.configure(config_element('ROOT', '', {}, []))
797
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
798
+ end
799
+
800
+ test "when output doesn't have <buffer> and time_as_integer is true" do
801
+ i = create_output(:sync)
802
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}))
803
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
804
+ end
805
+
806
+ test 'when output has <buffer> and compress is gzip' do
807
+ i = create_output(:buffered)
808
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))
809
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM, i.generate_format_proc
810
+ end
811
+
812
+ test 'when output has <buffer> and compress is gzip and time_as_integer is true' do
813
+ i = create_output(:buffered)
814
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))
815
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
816
+ end
817
+
818
+ test 'when output has <buffer> and compress is text' do
819
+ i = create_output(:buffered)
820
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))
821
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
822
+ end
823
+ end
824
+
825
+ sub_test_case 'slow_flush_log_threshold' do
826
+ def invoke_slow_flush_log_threshold_test(i)
827
+ i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
828
+ i.start
829
+ i.after_start
830
+
831
+ t = event_time()
832
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
833
+ i.force_flush
834
+
835
+ waiting(4) { Thread.pass until i.test_finished? }
836
+
837
+ yield
838
+ ensure
839
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
840
+ end
841
+
842
+ test '#write flush took longer time than slow_flush_log_threshold' do
843
+ i = create_output(:buffered)
844
+ write_called = false
845
+ i.register(:write) { |chunk| sleep 3 }
846
+ i.define_singleton_method(:test_finished?) { write_called }
847
+ i.define_singleton_method(:try_flush) { super(); write_called = true }
848
+
849
+ invoke_slow_flush_log_threshold_test(i) {
850
+ assert write_called
851
+ logs = i.log.out.logs
852
+ assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
853
+ }
854
+ end
855
+
856
+ test '#try_write flush took longer time than slow_flush_log_threshold' do
857
+ i = create_output(:delayed)
858
+ try_write_called = false
859
+ i.register(:try_write){ |chunk| sleep 3 }
860
+ i.define_singleton_method(:test_finished?) { try_write_called }
861
+ i.define_singleton_method(:try_flush) { super(); try_write_called = true }
862
+
863
+ invoke_slow_flush_log_threshold_test(i) {
864
+ assert try_write_called
865
+ logs = i.log.out.logs
866
+ assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
867
+ }
868
+ end
869
+ end
870
+ end