fluentd 0.14.4-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,29 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_null'
4
+
5
+ class NullOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def create_driver(conf = "")
11
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
12
+ end
13
+
14
+ def test_configure
15
+ assert_nothing_raised do
16
+ create_driver
17
+ end
18
+ end
19
+
20
+ def test_process
21
+ d = create_driver
22
+ assert_nothing_raised do
23
+ d.run do
24
+ d.feed("test", Fluent::EventTime.now, {"test" => "null"})
25
+ end
26
+ end
27
+ assert_equal([], d.events(tag: "test"))
28
+ end
29
+ end
@@ -0,0 +1,28 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_relabel'
4
+
5
+ class RelabelOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def default_config
11
+ config_element('ROOT', '', {"@type"=>"relabel", "@label"=>"@RELABELED"})
12
+ end
13
+
14
+ def create_driver(conf = default_config)
15
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::RelabelOutput).configure(conf)
16
+ end
17
+
18
+ def test_process
19
+ d = create_driver
20
+
21
+ time = event_time("2011-01-02 13:14:15 UTC")
22
+ d.run(default_tag: 'test') do
23
+ d.feed(time, {"a"=>1})
24
+ d.feed(time, {"a"=>2})
25
+ end
26
+ assert_equal [["test", time, {"a"=>1}], ["test", time, {"a"=>2}]], d.events
27
+ end
28
+ end
@@ -0,0 +1,146 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/multi_output'
3
+ require 'fluent/plugin/out_roundrobin'
4
+
5
+ class RoundRobinOutputTest < Test::Unit::TestCase
6
+ class << self
7
+ def startup
8
+ $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
9
+ require 'fluent/plugin/out_test'
10
+ require 'fluent/plugin/out_test2'
11
+ end
12
+
13
+ def shutdown
14
+ $LOAD_PATH.shift
15
+ end
16
+ end
17
+
18
+ def setup
19
+ Fluent::Test.setup
20
+ end
21
+
22
+ CONFIG = %[
23
+ <store>
24
+ @type test
25
+ name c0
26
+ </store>
27
+ <store>
28
+ @type test2
29
+ name c1
30
+ </store>
31
+ <store>
32
+ @type test
33
+ name c2
34
+ </store>
35
+ ]
36
+ CONFIG_WITH_WEIGHT = %[
37
+ <store>
38
+ @type test
39
+ name c0
40
+ weight 3
41
+ </store>
42
+ <store>
43
+ @type test2
44
+ name c1
45
+ weight 3
46
+ </store>
47
+ <store>
48
+ @type test
49
+ name c2
50
+ </store>
51
+ ]
52
+
53
+ def create_driver(conf = CONFIG)
54
+ Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::RoundRobinOutput).configure(conf)
55
+ end
56
+
57
+ def test_configure
58
+ d = create_driver
59
+
60
+ outputs = d.instance.outputs
61
+ assert_equal 3, outputs.size
62
+
63
+ assert_equal Fluent::Plugin::TestOutput, outputs[0].class
64
+ assert_equal Fluent::Plugin::Test2Output, outputs[1].class
65
+ assert_equal Fluent::Plugin::TestOutput, outputs[2].class
66
+
67
+ assert !outputs[0].has_router?
68
+ assert outputs[1].has_router?
69
+ assert outputs[1].router
70
+ assert !outputs[2].has_router?
71
+
72
+ assert_equal "c0", outputs[0].name
73
+ assert_equal "c1", outputs[1].name
74
+ assert_equal "c2", outputs[2].name
75
+
76
+ weights = d.instance.weights
77
+ assert_equal 3, weights.size
78
+ assert_equal 1, weights[0]
79
+ assert_equal 1, weights[1]
80
+ assert_equal 1, weights[2]
81
+
82
+ d = create_driver(CONFIG_WITH_WEIGHT)
83
+
84
+ outputs = d.instance.outputs
85
+ assert_equal 3, outputs.size
86
+
87
+ assert_equal Fluent::Plugin::TestOutput, outputs[0].class
88
+ assert_equal Fluent::Plugin::Test2Output, outputs[1].class
89
+ assert_equal Fluent::Plugin::TestOutput, outputs[2].class
90
+
91
+ assert_equal "c0", outputs[0].name
92
+ assert_equal "c1", outputs[1].name
93
+ assert_equal "c2", outputs[2].name
94
+
95
+ weights = d.instance.weights
96
+ assert_equal 3, weights.size
97
+ assert_equal 3, weights[0]
98
+ assert_equal 3, weights[1]
99
+ assert_equal 1, weights[2]
100
+ end
101
+
102
+ def test_events_feeded_to_plugins_by_roundrobin
103
+ d = create_driver
104
+
105
+ time = event_time("2011-01-02 13:14:15 UTC")
106
+ d.run(default_tag: 'test') do
107
+ d.feed(time, {"a" => 1})
108
+ d.feed(time, {"a" => 2})
109
+ d.feed(time, {"a" => 3})
110
+ d.feed(time, {"a" => 4})
111
+ end
112
+
113
+ os = d.instance.outputs
114
+
115
+ assert_equal [
116
+ [time, {"a"=>1}],
117
+ [time, {"a"=>4}],
118
+ ], os[0].events
119
+
120
+ assert_equal [
121
+ [time, {"a"=>2}],
122
+ ], os[1].events
123
+
124
+ assert_equal [
125
+ [time, {"a"=>3}],
126
+ ], os[2].events
127
+ end
128
+
129
+ def test_events_feeded_with_specified_weights
130
+ d = create_driver(CONFIG_WITH_WEIGHT)
131
+
132
+ time = event_time("2011-01-02 13:14:15 UTC")
133
+ d.run(default_tag: 'test') do
134
+ 14.times do |i|
135
+ d.feed(time, {"a" => i})
136
+ end
137
+ end
138
+
139
+ os = d.instance.outputs
140
+
141
+ assert_equal 6, os[0].events.size # weight=3
142
+ assert_equal 6, os[1].events.size # weight=3
143
+ assert_equal 2, os[2].events.size # weight=1
144
+ end
145
+ end
146
+
@@ -0,0 +1,92 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_stdout'
4
+
5
+ class StdoutOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ CONFIG = %[
11
+ ]
12
+
13
+ def create_driver(conf = CONFIG)
14
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
15
+ end
16
+
17
+ def test_configure
18
+ d = create_driver
19
+ assert_equal [], d.instance.formatter_configs
20
+ end
21
+
22
+ def test_configure_output_type
23
+ d = create_driver(CONFIG + "\noutput_type json")
24
+ assert_equal 'json', d.instance.formatter_configs.first[:@type]
25
+
26
+ d = create_driver(CONFIG + "\noutput_type hash")
27
+ assert_equal 'hash', d.instance.formatter_configs.first[:@type]
28
+
29
+ assert_raise(Fluent::ConfigError) do
30
+ d = create_driver(CONFIG + "\noutput_type foo")
31
+ end
32
+ end
33
+
34
+ def test_emit_in_default
35
+ d = create_driver
36
+ time = event_time()
37
+ out = capture_log do
38
+ d.run(default_tag: 'test') do
39
+ d.feed(time, {'test' => 'test1'})
40
+ end
41
+ end
42
+ assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
43
+ end
44
+
45
+ data('oj' => 'oj', 'yajl' => 'yajl')
46
+ def test_emit_json(data)
47
+ d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
48
+ time = event_time()
49
+ out = capture_log do
50
+ d.run(default_tag: 'test') do
51
+ d.feed(time, {'test' => 'test1'})
52
+ end
53
+ end
54
+ assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
55
+
56
+ if data == 'yajl'
57
+ # NOTE: Float::NAN is not jsonable
58
+ assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
59
+ else
60
+ out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
61
+ assert_equal "#{Time.at(time).localtime} test: {\"test\":NaN}\n", out
62
+ end
63
+ end
64
+
65
+ def test_emit_hash
66
+ d = create_driver(CONFIG + "\noutput_type hash")
67
+ time = event_time()
68
+ out = capture_log do
69
+ d.run(default_tag: 'test') do
70
+ d.feed(time, {'test' => 'test2'})
71
+ end
72
+ end
73
+ assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test2\"}\n", out
74
+
75
+ # NOTE: Float::NAN is not jsonable, but hash string can output it.
76
+ out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
77
+ assert_equal "#{Time.at(time).localtime} test: {\"test\"=>NaN}\n", out
78
+ end
79
+
80
+ private
81
+
82
+ # Capture the log output of the block given
83
+ def capture_log(&block)
84
+ tmp = $log
85
+ $log = StringIO.new
86
+ yield
87
+ return $log.string
88
+ ensure
89
+ $log = tmp
90
+ end
91
+ end
92
+
@@ -0,0 +1,93 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test'
3
+ require 'fluent/plugin/out_stream'
4
+
5
+ module StreamOutputTest
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def test_write
11
+ d = create_driver
12
+
13
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
14
+ d.emit({"a"=>1}, time)
15
+ d.emit({"a"=>2}, time)
16
+
17
+ expect = ["test",
18
+ [time,{"a"=>1}].to_msgpack +
19
+ [time,{"a"=>2}].to_msgpack
20
+ ].to_msgpack
21
+
22
+ result = d.run
23
+ assert_equal(expect, result)
24
+ end
25
+
26
+ def test_write_event_time
27
+ d = create_driver
28
+
29
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
30
+ d.emit({"a"=>1}, time)
31
+ d.emit({"a"=>2}, time)
32
+
33
+ expect = ["test",
34
+ Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>1}]).to_s +
35
+ Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>2}]).to_s
36
+ ]
37
+ expect = Fluent::Engine.msgpack_factory.packer.write(expect).to_s
38
+
39
+ result = d.run
40
+ assert_equal(expect, result)
41
+ end
42
+
43
+ def create_driver(klass, conf)
44
+ Fluent::Test::BufferedOutputTestDriver.new(klass) do
45
+ def write(chunk)
46
+ chunk.read
47
+ end
48
+ end.configure(conf)
49
+ end
50
+ end
51
+
52
+ class TcpOutputTest < Test::Unit::TestCase
53
+ include StreamOutputTest
54
+
55
+ PORT = unused_port
56
+ CONFIG = %[
57
+ port #{PORT}
58
+ host 127.0.0.1
59
+ send_timeout 51
60
+ ]
61
+
62
+ def create_driver(conf=CONFIG)
63
+ super(Fluent::TcpOutput, conf)
64
+ end
65
+
66
+ def test_configure
67
+ d = create_driver
68
+ assert_equal PORT, d.instance.port
69
+ assert_equal '127.0.0.1', d.instance.host
70
+ assert_equal 51, d.instance.send_timeout
71
+ end
72
+ end
73
+
74
+ class UnixOutputTest < Test::Unit::TestCase
75
+ include StreamOutputTest
76
+
77
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_unix#{ENV['TEST_ENV_NUMBER']}"
78
+ CONFIG = %[
79
+ path #{TMP_DIR}/unix
80
+ send_timeout 52
81
+ ]
82
+
83
+ def create_driver(conf=CONFIG)
84
+ super(Fluent::UnixOutput, conf)
85
+ end
86
+
87
+ def test_configure
88
+ d = create_driver
89
+ assert_equal "#{TMP_DIR}/unix", d.instance.path
90
+ assert_equal 52, d.instance.send_timeout
91
+ end
92
+ end
93
+
@@ -0,0 +1,568 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/event'
5
+
6
+ require 'json'
7
+ require 'time'
8
+ require 'timeout'
9
+
10
+ module FluentPluginOutputTest
11
+ class DummyBareOutput < Fluent::Plugin::Output
12
+ def register(name, &block)
13
+ instance_variable_set("@#{name}", block)
14
+ end
15
+ end
16
+ class DummySyncOutput < DummyBareOutput
17
+ def initialize
18
+ super
19
+ @process = nil
20
+ end
21
+ def process(tag, es)
22
+ @process ? @process.call(tag, es) : nil
23
+ end
24
+ end
25
+ class DummyAsyncOutput < DummyBareOutput
26
+ def initialize
27
+ super
28
+ @format = nil
29
+ @write = nil
30
+ end
31
+ def format(tag, time, record)
32
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
33
+ end
34
+ def write(chunk)
35
+ @write ? @write.call(chunk) : nil
36
+ end
37
+ end
38
+ class DummyAsyncStandardOutput < DummyBareOutput
39
+ def initialize
40
+ super
41
+ @write = nil
42
+ end
43
+ def write(chunk)
44
+ @write ? @write.call(chunk) : nil
45
+ end
46
+ end
47
+ class DummyDelayedOutput < DummyBareOutput
48
+ def initialize
49
+ super
50
+ @format = nil
51
+ @try_write = nil
52
+ end
53
+ def format(tag, time, record)
54
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
55
+ end
56
+ def try_write(chunk)
57
+ @try_write ? @try_write.call(chunk) : nil
58
+ end
59
+ end
60
+ class DummyDelayedStandardOutput < DummyBareOutput
61
+ def initialize
62
+ super
63
+ @try_write = nil
64
+ end
65
+ def try_write(chunk)
66
+ @try_write ? @try_write.call(chunk) : nil
67
+ end
68
+ end
69
+ class DummyFullFeatureOutput < DummyBareOutput
70
+ def initialize
71
+ super
72
+ @prefer_buffered_processing = nil
73
+ @prefer_delayed_commit = nil
74
+ @process = nil
75
+ @format = nil
76
+ @write = nil
77
+ @try_write = nil
78
+ end
79
+ def prefer_buffered_processing
80
+ @prefer_buffered_processing ? @prefer_buffered_processing.call : false
81
+ end
82
+ def prefer_delayed_commit
83
+ @prefer_delayed_commit ? @prefer_delayed_commit.call : false
84
+ end
85
+ def process(tag, es)
86
+ @process ? @process.call(tag, es) : nil
87
+ end
88
+ def format(tag, time, record)
89
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
90
+ end
91
+ def write(chunk)
92
+ @write ? @write.call(chunk) : nil
93
+ end
94
+ def try_write(chunk)
95
+ @try_write ? @try_write.call(chunk) : nil
96
+ end
97
+ end
98
+ end
99
+
100
+ class OutputTest < Test::Unit::TestCase
101
+ class << self
102
+ def startup
103
+ $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))
104
+ require 'fluent/plugin/out_test'
105
+ end
106
+
107
+ def shutdown
108
+ $LOAD_PATH.shift
109
+ end
110
+ end
111
+
112
+ def create_output(type=:full)
113
+ case type
114
+ when :bare then FluentPluginOutputTest::DummyBareOutput.new
115
+ when :sync then FluentPluginOutputTest::DummySyncOutput.new
116
+ when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new
117
+ when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new
118
+ when :delayed then FluentPluginOutputTest::DummyDelayedOutput.new
119
+ when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new
120
+ when :full then FluentPluginOutputTest::DummyFullFeatureOutput.new
121
+ else
122
+ raise ArgumentError, "unknown type: #{type}"
123
+ end
124
+ end
125
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
126
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
127
+ end
128
+ def waiting(seconds)
129
+ begin
130
+ Timeout.timeout(seconds) do
131
+ yield
132
+ end
133
+ rescue Timeout::Error
134
+ STDERR.print(*@i.log.out.logs)
135
+ raise
136
+ end
137
+ end
138
+
139
+ sub_test_case 'basic output feature' do
140
+ setup do
141
+ @i = create_output(:full)
142
+ end
143
+
144
+ test '#implement? can return features for plugin instances' do
145
+ i1 = FluentPluginOutputTest::DummyBareOutput.new
146
+ assert !i1.implement?(:synchronous)
147
+ assert !i1.implement?(:buffered)
148
+ assert !i1.implement?(:delayed_commit)
149
+ assert !i1.implement?(:custom_format)
150
+
151
+ i2 = FluentPluginOutputTest::DummySyncOutput.new
152
+ assert i2.implement?(:synchronous)
153
+ assert !i2.implement?(:buffered)
154
+ assert !i2.implement?(:delayed_commit)
155
+ assert !i2.implement?(:custom_format)
156
+
157
+ i3 = FluentPluginOutputTest::DummyAsyncOutput.new
158
+ assert !i3.implement?(:synchronous)
159
+ assert i3.implement?(:buffered)
160
+ assert !i3.implement?(:delayed_commit)
161
+ assert i3.implement?(:custom_format)
162
+
163
+ i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new
164
+ assert !i4.implement?(:synchronous)
165
+ assert i4.implement?(:buffered)
166
+ assert !i4.implement?(:delayed_commit)
167
+ assert !i4.implement?(:custom_format)
168
+
169
+ i5 = FluentPluginOutputTest::DummyDelayedOutput.new
170
+ assert !i5.implement?(:synchronous)
171
+ assert !i5.implement?(:buffered)
172
+ assert i5.implement?(:delayed_commit)
173
+ assert i5.implement?(:custom_format)
174
+
175
+ i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new
176
+ assert !i6.implement?(:synchronous)
177
+ assert !i6.implement?(:buffered)
178
+ assert i6.implement?(:delayed_commit)
179
+ assert !i6.implement?(:custom_format)
180
+
181
+ i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new
182
+ assert i6.implement?(:synchronous)
183
+ assert i6.implement?(:buffered)
184
+ assert i6.implement?(:delayed_commit)
185
+ assert i6.implement?(:custom_format)
186
+ end
187
+
188
+ test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do
189
+ assert !@i.configured?
190
+ @i.configure(config_element())
191
+ assert @i.configured?
192
+ assert !@i.started?
193
+ @i.start
194
+ assert @i.started?
195
+ assert !@i.after_started?
196
+ @i.after_start
197
+ assert @i.after_started?
198
+ assert !@i.stopped?
199
+ @i.stop
200
+ assert @i.stopped?
201
+ assert !@i.before_shutdown?
202
+ @i.before_shutdown
203
+ assert @i.before_shutdown?
204
+ assert !@i.shutdown?
205
+ @i.shutdown
206
+ assert @i.shutdown?
207
+ assert !@i.after_shutdown?
208
+ @i.after_shutdown
209
+ assert @i.after_shutdown?
210
+ assert !@i.closed?
211
+ @i.close
212
+ assert @i.closed?
213
+ assert !@i.terminated?
214
+ @i.terminate
215
+ assert @i.terminated?
216
+ end
217
+
218
+ test '#extract_placeholders does nothing if chunk key is not specified' do
219
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
220
+ assert !@i.chunk_key_time
221
+ assert !@i.chunk_key_tag
222
+ assert_equal [], @i.chunk_keys
223
+ tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
224
+ t = event_time('2016-04-11 20:30:00 +0900')
225
+ v = {key1: "value1", key2: "value2"}
226
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
227
+ assert_equal tmpl, @i.extract_placeholders(tmpl, m)
228
+ end
229
+
230
+ test '#extract_placeholders can extract time if time key and range are configured' do
231
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
232
+ assert @i.chunk_key_time
233
+ assert !@i.chunk_key_tag
234
+ assert_equal [], @i.chunk_keys
235
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
236
+ t = event_time('2016-04-11 20:30:00 +0900')
237
+ v = {key1: "value1", key2: "value2"}
238
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
239
+ assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
240
+ end
241
+
242
+ test '#extract_placeholders can extract tag and parts of tag if tag is configured' do
243
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
244
+ assert !@i.chunk_key_time
245
+ assert @i.chunk_key_tag
246
+ assert_equal [], @i.chunk_keys
247
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
248
+ t = event_time('2016-04-11 20:30:00 +0900')
249
+ v = {key1: "value1", key2: "value2"}
250
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
251
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
252
+ end
253
+
254
+ test '#extract_placeholders can extract variables if variables are configured' do
255
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
256
+ assert !@i.chunk_key_time
257
+ assert !@i.chunk_key_tag
258
+ assert_equal ['key1','key2'], @i.chunk_keys
259
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
260
+ t = event_time('2016-04-11 20:30:00 +0900')
261
+ v = {key1: "value1", key2: "value2"}
262
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
263
+ assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
264
+ end
265
+
266
+ test '#extract_placeholders can extract all chunk keys if configured' do
267
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
268
+ assert @i.chunk_key_time
269
+ assert @i.chunk_key_tag
270
+ assert_equal ['key1','key2'], @i.chunk_keys
271
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
272
+ t = event_time('2016-04-11 20:30:00 +0900')
273
+ v = {key1: "value1", key2: "value2"}
274
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
275
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m)
276
+ end
277
+
278
+ test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do
279
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
280
+ assert @i.chunk_key_time
281
+ assert @i.chunk_key_tag
282
+ assert_equal ['key1','key2'], @i.chunk_keys
283
+ tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
284
+ t = event_time('2016-04-11 20:30:00 +0900')
285
+ v = {key1: "value1", key2: "value2"}
286
+ m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
287
+ assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
288
+ end
289
+
290
+ test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
291
+ tag = 'test.output'
292
+ time = event_time('2016-04-12 15:31:23 -0700')
293
+ timekey = event_time('2016-04-12 15:00:00 -0700')
294
+ record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
295
+
296
+ i1 = create_output(:buffered)
297
+ i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
298
+ assert_equal create_metadata(), i1.metadata(tag, time, record)
299
+
300
+ i2 = create_output(:buffered)
301
+ i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))
302
+ assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)
303
+
304
+ i3 = create_output(:buffered)
305
+ i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
306
+ assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)
307
+
308
+ i4 = create_output(:buffered)
309
+ i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))
310
+ assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record)
311
+
312
+ i5 = create_output(:buffered)
313
+ i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))
314
+ assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record)
315
+
316
+ i6 = create_output(:buffered)
317
+ i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
318
+ assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)
319
+
320
+ i7 = create_output(:buffered)
321
+ i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
322
+ assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)
323
+
324
+ i8 = create_output(:buffered)
325
+ i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
326
+ assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
327
+ end
328
+
329
+ test '#emit calls #process via #emit_sync for non-buffered output' do
330
+ i = create_output(:sync)
331
+ process_called = false
332
+ i.register(:process){|tag, es| process_called = true }
333
+ i.configure(config_element())
334
+ i.start
335
+ i.after_start
336
+
337
+ t = event_time()
338
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
339
+
340
+ assert process_called
341
+
342
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
343
+ end
344
+
345
+ test '#emit calls #format for buffered output' do
346
+ i = create_output(:buffered)
347
+ format_called_times = 0
348
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
349
+ i.configure(config_element())
350
+ i.start
351
+ i.after_start
352
+
353
+ t = event_time()
354
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
355
+
356
+ assert_equal 2, format_called_times
357
+
358
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
359
+ end
360
+
361
+ test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do
362
+ i = create_output(:full)
363
+
364
+ process_called = false
365
+ format_called_times = 0
366
+ i.register(:process){|tag, es| process_called = true }
367
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
368
+
369
+ i.configure(config_element())
370
+ i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
371
+ i.start
372
+ i.after_start
373
+
374
+ assert !i.prefer_buffered_processing
375
+
376
+ t = event_time()
377
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
378
+
379
+ waiting(4){ Thread.pass until process_called }
380
+
381
+ assert process_called
382
+ assert_equal 0, format_called_times
383
+
384
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
385
+ end
386
+
387
+ test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do
388
+ i = create_output(:full)
389
+
390
+ process_called = false
391
+ format_called_times = 0
392
+ i.register(:process){|tag, es| process_called = true }
393
+ i.register(:format){|tag, time, record| format_called_times += 1; '' }
394
+
395
+ i.configure(config_element())
396
+ i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
397
+ i.start
398
+ i.after_start
399
+
400
+ assert i.prefer_buffered_processing
401
+
402
+ t = event_time()
403
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
404
+
405
+ assert !process_called
406
+ assert_equal 2, format_called_times
407
+
408
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
409
+ end
410
+
411
+ test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do
412
+ i = create_output(:buffered)
413
+ write_called = false
414
+ i.register(:write){ |chunk| write_called = true }
415
+
416
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
417
+ i.start
418
+ i.after_start
419
+
420
+ t = event_time()
421
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
422
+ i.force_flush
423
+
424
+ waiting(4){ Thread.pass until write_called }
425
+
426
+ assert write_called
427
+
428
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
429
+ end
430
+
431
+ test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
432
+ i = create_output(:delayed)
433
+ try_write_called = false
434
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
435
+
436
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
437
+ i.start
438
+ i.after_start
439
+
440
+ t = event_time()
441
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
442
+ i.force_flush
443
+
444
+ waiting(4){ Thread.pass until try_write_called }
445
+
446
+ assert try_write_called
447
+
448
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
449
+ end
450
+
451
+ test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
452
+ i = create_output(:full)
453
+ write_called = false
454
+ try_write_called = false
455
+ i.register(:write){ |chunk| write_called = true }
456
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
457
+
458
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
459
+ i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
460
+ i.start
461
+ i.after_start
462
+
463
+ assert !i.prefer_delayed_commit
464
+
465
+ t = event_time()
466
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
467
+ i.force_flush
468
+
469
+ waiting(4){ Thread.pass until write_called || try_write_called }
470
+
471
+ assert write_called
472
+ assert !try_write_called
473
+
474
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
475
+ end
476
+
477
+ test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
478
+ i = create_output(:full)
479
+ write_called = false
480
+ try_write_called = false
481
+ i.register(:write){ |chunk| write_called = true }
482
+ i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
483
+
484
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
485
+ i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
486
+ i.start
487
+ i.after_start
488
+
489
+ assert i.prefer_delayed_commit
490
+
491
+ t = event_time()
492
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
493
+ i.force_flush
494
+
495
+ waiting(4){ Thread.pass until write_called || try_write_called }
496
+
497
+ assert !write_called
498
+ assert try_write_called
499
+
500
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
501
+ end
502
+
503
+ test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
504
+ o = create_output(:buffered)
505
+ mock(o.log).warn("secondary type should be same with primary one",
506
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
507
+
508
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
509
+ assert_not_nil o.instance_variable_get(:@secondary)
510
+ end
511
+
512
+ test "don't warn if primary type is the same as secondary type" do
513
+ o = Fluent::Plugin::TestOutput.new
514
+ mock(o.log).warn("secondary type should be same with primary one",
515
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
516
+
517
+ o.configure(config_element('ROOT','',{'name' => "cool2"},
518
+ [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
519
+ config_element('buffer','',{'@type'=>'memory'})]
520
+ ))
521
+ assert_not_nil o.instance_variable_get(:@secondary)
522
+ end
523
+
524
+ test "don't warn if primary type is different from secondary type and both don't have custom_format" do
525
+ o = create_output(:standard)
526
+ mock(o.log).warn("secondary type should be same with primary one",
527
+ { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
528
+
529
+ o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
530
+ assert_not_nil o.instance_variable_get(:@secondary)
531
+ end
532
+ end
533
+
534
+ sub_test_case 'sync output feature' do
535
+ setup do
536
+ @i = create_output(:sync)
537
+ end
538
+
539
+ test 'raises configuration error if <buffer> section is specified' do
540
+ assert_raise Fluent::ConfigError do
541
+ @i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
542
+ end
543
+ end
544
+
545
+ test 'raises configuration error if <secondary> section is specified' do
546
+ assert_raise Fluent::ConfigError do
547
+ @i.configure(config_element('ROOT','',{},[config_element('secondary','')]))
548
+ end
549
+ end
550
+
551
+ test '#process is called for each event streams' do
552
+ ary = []
553
+ @i.register(:process){|tag, es| ary << [tag, es] }
554
+ @i.configure(config_element())
555
+ @i.start
556
+ @i.after_start
557
+
558
+ t = event_time()
559
+ es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
560
+ 5.times do
561
+ @i.emit_events('tag', es)
562
+ end
563
+ assert_equal 5, ary.size
564
+
565
+ @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
566
+ end
567
+ end
568
+ end