fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,817 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/event'
5
+
6
+ require 'json'
7
+ require 'time'
8
+ require 'timeout'
9
+ require 'timecop'
10
+
11
+ module FluentPluginOutputAsBufferedSecondaryTest
12
+ class DummyBareOutput < Fluent::Plugin::Output
13
+ def register(name, &block)
14
+ instance_variable_set("@#{name}", block)
15
+ end
16
+ end
17
+ class DummySyncOutput < DummyBareOutput
18
+ def initialize
19
+ super
20
+ @process = nil
21
+ end
22
+ def process(tag, es)
23
+ @process ? @process.call(tag, es) : nil
24
+ end
25
+ end
26
+ class DummyFullFeatureOutput < DummyBareOutput
27
+ def initialize
28
+ super
29
+ @prefer_buffered_processing = nil
30
+ @prefer_delayed_commit = nil
31
+ @process = nil
32
+ @format = nil
33
+ @write = nil
34
+ @try_write = nil
35
+ end
36
+ def prefer_buffered_processing
37
+ @prefer_buffered_processing ? @prefer_buffered_processing.call : false
38
+ end
39
+ def prefer_delayed_commit
40
+ @prefer_delayed_commit ? @prefer_delayed_commit.call : false
41
+ end
42
+ def process(tag, es)
43
+ @process ? @process.call(tag, es) : nil
44
+ end
45
+ def format(tag, time, record)
46
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
47
+ end
48
+ def write(chunk)
49
+ @write ? @write.call(chunk) : nil
50
+ end
51
+ def try_write(chunk)
52
+ @try_write ? @try_write.call(chunk) : nil
53
+ end
54
+ end
55
+ class DummyFullFeatureOutput2 < DummyFullFeatureOutput
56
+ def prefer_buffered_processing; true; end
57
+ def prefer_delayed_commit; super; end
58
+ def format(tag, time, record); super; end
59
+ def write(chunk); super; end
60
+ def try_write(chunk); super; end
61
+ end
62
+ end
63
+
64
+ class BufferedOutputSecondaryTest < Test::Unit::TestCase
65
+ def create_output(type=:full)
66
+ case type
67
+ when :bare then FluentPluginOutputAsBufferedSecondaryTest::DummyBareOutput.new
68
+ when :sync then FluentPluginOutputAsBufferedSecondaryTest::DummySyncOutput.new
69
+ when :full then FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput.new
70
+ else
71
+ raise ArgumentError, "unknown type: #{type}"
72
+ end
73
+ end
74
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
75
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
76
+ end
77
+ def waiting(seconds)
78
+ begin
79
+ Timeout.timeout(seconds) do
80
+ yield
81
+ end
82
+ rescue Timeout::Error
83
+ STDERR.print(*@i.log.out.logs)
84
+ raise
85
+ end
86
+ end
87
+ def dummy_event_stream
88
+ Fluent::ArrayEventStream.new([
89
+ [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
90
+ [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
91
+ [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
92
+ ])
93
+ end
94
+
95
+ setup do
96
+ @i = create_output
97
+ end
98
+
99
+ teardown do
100
+ if @i
101
+ @i.stop unless @i.stopped?
102
+ @i.before_shutdown unless @i.before_shutdown?
103
+ @i.shutdown unless @i.shutdown?
104
+ @i.after_shutdown unless @i.after_shutdown?
105
+ @i.close unless @i.closed?
106
+ @i.terminate unless @i.terminated?
107
+ end
108
+ Timecop.return
109
+ end
110
+
111
+ sub_test_case 'secondary plugin feature for buffered output with periodical retry' do
112
+ setup do
113
+ Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)
114
+ Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)
115
+ end
116
+
117
+ test 'raises configuration error if primary does not support buffering' do
118
+ i = create_output(:sync)
119
+ assert_raise Fluent::ConfigError do
120
+ i.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'output_secondary_test'})]))
121
+ end
122
+ end
123
+
124
+ test 'raises configuration error if <buffer>/<secondary> section is specified in <secondary> section' do
125
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
126
+ secconf1 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('buffer', 'time')])
127
+ secconf2 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('secondary', '')])
128
+ i = create_output()
129
+ assert_raise Fluent::ConfigError do
130
+ i.configure(config_element('ROOT','',{},[priconf,secconf1]))
131
+ end
132
+ assert_raise Fluent::ConfigError do
133
+ i.configure(config_element('ROOT','',{},[priconf,secconf2]))
134
+ end
135
+ end
136
+
137
+ test 'uses same plugin type with primary if @type is missing in secondary' do
138
+ bufconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
139
+ secconf = config_element('secondary','',{})
140
+ priconf = config_element('ROOT', '', {'@type' => 'output_secondary_test'}, [bufconf, secconf])
141
+ i = create_output()
142
+ assert_nothing_raised do
143
+ i.configure(priconf)
144
+ end
145
+ logs = i.log.out.logs
146
+ assert{ logs.empty? }
147
+ assert{ i.secondary.is_a? FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput }
148
+ end
149
+
150
+ test 'warns if secondary plugin is different type from primary one' do
151
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
152
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
153
+ i = create_output()
154
+ i.configure(config_element('ROOT','',{},[priconf,secconf]))
155
+ logs = i.log.out.logs
156
+ assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } }
157
+ end
158
+
159
+ test 'secondary plugin lifecycle is kicked by primary' do
160
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
161
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
162
+ i = create_output()
163
+ i.configure(config_element('ROOT','',{},[priconf,secconf]))
164
+ logs = i.log.out.logs
165
+ assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } }
166
+
167
+ assert i.secondary.configured?
168
+
169
+ assert !i.secondary.started?
170
+ i.start
171
+ assert i.secondary.started?
172
+
173
+ assert !i.secondary.after_started?
174
+ i.after_start
175
+ assert i.secondary.after_started?
176
+
177
+ assert !i.secondary.stopped?
178
+ i.stop
179
+ assert i.secondary.stopped?
180
+
181
+ assert !i.secondary.before_shutdown?
182
+ i.before_shutdown
183
+ assert i.secondary.before_shutdown?
184
+
185
+ assert !i.secondary.shutdown?
186
+ i.shutdown
187
+ assert i.secondary.shutdown?
188
+
189
+ assert !i.secondary.after_shutdown?
190
+ i.after_shutdown
191
+ assert i.secondary.after_shutdown?
192
+
193
+ assert !i.secondary.closed?
194
+ i.close
195
+ assert i.secondary.closed?
196
+
197
+ assert !i.secondary.terminated?
198
+ i.terminate
199
+ assert i.secondary.terminated?
200
+ end
201
+
202
+ test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do
203
+ written = []
204
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
205
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
206
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
207
+ @i.register(:prefer_buffered_processing){ true }
208
+ @i.register(:prefer_delayed_commit){ false }
209
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
210
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
211
+ @i.secondary.register(:prefer_delayed_commit){ false }
212
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
213
+ @i.start
214
+ @i.after_start
215
+
216
+ @i.interrupt_flushes
217
+
218
+ now = Time.parse('2016-04-13 18:33:30 -0700')
219
+ Timecop.freeze( now )
220
+
221
+ @i.emit_events("test.tag.1", dummy_event_stream())
222
+
223
+ now = Time.parse('2016-04-13 18:33:31 -0700')
224
+ Timecop.freeze( now )
225
+
226
+ @i.emit_events("test.tag.2", dummy_event_stream())
227
+
228
+ assert_equal 0, @i.write_count
229
+ assert_equal 0, @i.num_errors
230
+
231
+ @i.enqueue_thread_wait
232
+ @i.flush_thread_wakeup
233
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
234
+
235
+ assert{ @i.buffer.queue.size > 0 }
236
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
237
+
238
+ assert{ @i.write_count > 0 }
239
+ assert{ @i.num_errors > 0 }
240
+
241
+ prev_write_count = @i.write_count
242
+ prev_num_errors = @i.num_errors
243
+
244
+ first_failure = @i.retry.start
245
+
246
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
247
+
248
+ now = first_failure + 60 * 0.8 + 1
249
+
250
+ Timecop.freeze( now )
251
+ @i.enqueue_thread_wait
252
+ @i.flush_thread_wakeup
253
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
254
+
255
+ assert{ @i.write_count > prev_write_count }
256
+ assert{ @i.num_errors == prev_num_errors }
257
+
258
+ assert_nil @i.retry
259
+
260
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
261
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
262
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
263
+
264
+ assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
265
+ end
266
+
267
+ test 'secondary can do non-delayed commit even if primary do delayed commit' do
268
+ written = []
269
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
270
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
271
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
272
+ @i.register(:prefer_buffered_processing){ true }
273
+ @i.register(:prefer_delayed_commit){ true }
274
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
275
+ @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
276
+ @i.secondary.register(:prefer_delayed_commit){ false }
277
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
278
+ @i.start
279
+ @i.after_start
280
+
281
+ @i.interrupt_flushes
282
+
283
+ now = Time.parse('2016-04-13 18:33:30 -0700')
284
+ Timecop.freeze( now )
285
+
286
+ @i.emit_events("test.tag.1", dummy_event_stream())
287
+
288
+ now = Time.parse('2016-04-13 18:33:31 -0700')
289
+ Timecop.freeze( now )
290
+
291
+ @i.emit_events("test.tag.2", dummy_event_stream())
292
+
293
+ assert_equal 0, @i.write_count
294
+ assert_equal 0, @i.num_errors
295
+
296
+ @i.enqueue_thread_wait
297
+ @i.flush_thread_wakeup
298
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
299
+
300
+ assert{ @i.buffer.queue.size > 0 }
301
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
302
+
303
+ assert{ @i.write_count > 0 }
304
+ assert{ @i.num_errors > 0 }
305
+
306
+ prev_write_count = @i.write_count
307
+ prev_num_errors = @i.num_errors
308
+
309
+ first_failure = @i.retry.start
310
+
311
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
312
+
313
+ now = first_failure + 60 * 0.8 + 1
314
+
315
+ Timecop.freeze( now )
316
+ @i.enqueue_thread_wait
317
+ @i.flush_thread_wakeup
318
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
319
+
320
+ assert{ @i.write_count > prev_write_count }
321
+ assert{ @i.num_errors == prev_num_errors }
322
+
323
+ assert_nil @i.retry
324
+
325
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
326
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
327
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
328
+
329
+ assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
330
+ end
331
+
332
+ test 'secondary plugin can do delayed commit if primary do it' do
333
+ written = []
334
+ chunks = []
335
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
336
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
337
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
338
+ @i.register(:prefer_buffered_processing){ true }
339
+ @i.register(:prefer_delayed_commit){ true }
340
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
341
+ @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
342
+ @i.secondary.register(:prefer_delayed_commit){ true }
343
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
344
+ @i.start
345
+ @i.after_start
346
+
347
+ @i.interrupt_flushes
348
+
349
+ now = Time.parse('2016-04-13 18:33:30 -0700')
350
+ Timecop.freeze( now )
351
+
352
+ @i.emit_events("test.tag.1", dummy_event_stream())
353
+
354
+ now = Time.parse('2016-04-13 18:33:31 -0700')
355
+ Timecop.freeze( now )
356
+
357
+ @i.emit_events("test.tag.2", dummy_event_stream())
358
+
359
+ assert_equal 0, @i.write_count
360
+ assert_equal 0, @i.num_errors
361
+
362
+ @i.enqueue_thread_wait
363
+ @i.flush_thread_wakeup
364
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
365
+
366
+ assert{ @i.buffer.queue.size > 0 }
367
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
368
+
369
+ assert{ @i.write_count > 0 }
370
+ assert{ @i.num_errors > 0 }
371
+
372
+ prev_write_count = @i.write_count
373
+ prev_num_errors = @i.num_errors
374
+
375
+ first_failure = @i.retry.start
376
+
377
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
378
+
379
+ now = first_failure + 60 * 0.8 + 1
380
+
381
+ Timecop.freeze( now )
382
+ @i.enqueue_thread_wait
383
+ @i.flush_thread_wakeup
384
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
385
+
386
+ assert{ @i.write_count > prev_write_count }
387
+ assert{ @i.num_errors == prev_num_errors }
388
+
389
+ assert @i.retry
390
+
391
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
392
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
393
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
394
+
395
+ assert{ @i.buffer.dequeued.size > 0 }
396
+ assert{ chunks.size > 0 }
397
+ assert{ !chunks.first.empty? }
398
+
399
+ @i.secondary.commit_write(chunks[0].unique_id)
400
+
401
+ assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }
402
+ assert{ chunks.first.empty? }
403
+
404
+ assert_nil @i.retry
405
+
406
+ assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
407
+ end
408
+
409
+ test 'secondary plugin can do delayed commit even if primary does not do it' do
410
+ written = []
411
+ chunks = []
412
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
413
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
414
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
415
+ @i.register(:prefer_buffered_processing){ true }
416
+ @i.register(:prefer_delayed_commit){ false }
417
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
418
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
419
+ @i.secondary.register(:prefer_delayed_commit){ true }
420
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
421
+ @i.start
422
+ @i.after_start
423
+
424
+ @i.interrupt_flushes
425
+
426
+ now = Time.parse('2016-04-13 18:33:30 -0700')
427
+ Timecop.freeze( now )
428
+
429
+ @i.emit_events("test.tag.1", dummy_event_stream())
430
+
431
+ now = Time.parse('2016-04-13 18:33:31 -0700')
432
+ Timecop.freeze( now )
433
+
434
+ @i.emit_events("test.tag.2", dummy_event_stream())
435
+
436
+ assert_equal 0, @i.write_count
437
+ assert_equal 0, @i.num_errors
438
+
439
+ @i.enqueue_thread_wait
440
+ @i.flush_thread_wakeup
441
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
442
+
443
+ assert{ @i.buffer.queue.size > 0 }
444
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
445
+
446
+ assert{ @i.write_count > 0 }
447
+ assert{ @i.num_errors > 0 }
448
+
449
+ prev_write_count = @i.write_count
450
+ prev_num_errors = @i.num_errors
451
+
452
+ first_failure = @i.retry.start
453
+
454
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
455
+
456
+ now = first_failure + 60 * 0.8 + 1
457
+
458
+ Timecop.freeze( now )
459
+ @i.enqueue_thread_wait
460
+ @i.flush_thread_wakeup
461
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
462
+
463
+ assert{ @i.write_count > prev_write_count }
464
+ assert{ @i.num_errors == prev_num_errors }
465
+
466
+ assert @i.retry
467
+
468
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
469
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
470
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
471
+
472
+ assert{ @i.buffer.dequeued.size > 0 }
473
+ assert{ chunks.size > 0 }
474
+ assert{ !chunks.first.empty? }
475
+
476
+ @i.secondary.commit_write(chunks[0].unique_id)
477
+
478
+ assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? }
479
+ assert{ chunks.first.empty? }
480
+
481
+ assert_nil @i.retry
482
+
483
+ assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
484
+ end
485
+
486
+ test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
487
+ written = []
488
+ chunks = []
489
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false})
490
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
491
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
492
+ @i.register(:prefer_buffered_processing){ true }
493
+ @i.register(:prefer_delayed_commit){ false }
494
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
495
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
496
+ @i.secondary.register(:prefer_delayed_commit){ true }
497
+ @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
498
+ @i.secondary.register(:write){|chunk| raise "don't use this" }
499
+ @i.start
500
+ @i.after_start
501
+
502
+ @i.interrupt_flushes
503
+
504
+ now = Time.parse('2016-04-13 18:33:30 -0700')
505
+ Timecop.freeze( now )
506
+
507
+ @i.emit_events("test.tag.1", dummy_event_stream())
508
+ @i.emit_events("test.tag.2", dummy_event_stream())
509
+
510
+ now = Time.parse('2016-04-13 18:33:31 -0700')
511
+ Timecop.freeze( now )
512
+
513
+ assert_equal 0, @i.write_count
514
+ assert_equal 0, @i.num_errors
515
+
516
+ @i.enqueue_thread_wait
517
+ @i.flush_thread_wakeup
518
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
519
+
520
+ assert{ @i.buffer.queue.size == 2 }
521
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
522
+
523
+ assert{ @i.write_count > 0 }
524
+ assert{ @i.num_errors > 0 }
525
+
526
+ prev_write_count = @i.write_count
527
+ prev_num_errors = @i.num_errors
528
+
529
+ first_failure = @i.retry.start
530
+
531
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
532
+
533
+ now = first_failure + 60 * 0.8 + 1
534
+ Timecop.freeze( now )
535
+ @i.enqueue_thread_wait
536
+ @i.flush_thread_wakeup
537
+ now = first_failure + 60 * 0.8 + 2
538
+ Timecop.freeze( now )
539
+ @i.enqueue_thread_wait
540
+ @i.flush_thread_wakeup
541
+
542
+ waiting(4){ Thread.pass until chunks.size == 2 }
543
+
544
+ assert{ @i.write_count > prev_write_count }
545
+ assert{ @i.num_errors == prev_num_errors }
546
+
547
+ assert @i.retry
548
+
549
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
550
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
551
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
552
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[3]
553
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[4]
554
+ assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[5]
555
+
556
+ assert{ @i.buffer.dequeued.size == 2 }
557
+ assert{ chunks.size == 2 }
558
+ assert{ !chunks[0].empty? }
559
+ assert{ !chunks[1].empty? }
560
+
561
+ 30.times do |i| # large enough
562
+ now = first_failure + 60 * 0.8 + 2 + i
563
+ Timecop.freeze( now )
564
+ @i.flush_thread_wakeup
565
+
566
+ break if @i.buffer.dequeued.size == 0
567
+ end
568
+
569
+ assert @i.retry
570
+ logs = @i.log.out.logs
571
+ assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
572
+ end
573
+
574
+ test 'retry_wait for secondary is same with one for primary' do
575
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false})
576
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
577
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
578
+ @i.register(:prefer_buffered_processing){ true }
579
+ @i.register(:prefer_delayed_commit){ false }
580
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
581
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
582
+ @i.secondary.register(:prefer_delayed_commit){ false }
583
+ @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
584
+ @i.start
585
+ @i.after_start
586
+
587
+ @i.interrupt_flushes
588
+
589
+ now = Time.parse('2016-04-13 18:33:30 -0700')
590
+ Timecop.freeze( now )
591
+
592
+ @i.emit_events("test.tag.1", dummy_event_stream())
593
+
594
+ now = Time.parse('2016-04-13 18:33:31 -0700')
595
+ Timecop.freeze( now )
596
+
597
+ @i.emit_events("test.tag.2", dummy_event_stream())
598
+
599
+ assert_equal 0, @i.write_count
600
+ assert_equal 0, @i.num_errors
601
+
602
+ @i.enqueue_thread_wait
603
+ @i.flush_thread_wakeup
604
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
605
+
606
+ assert{ @i.buffer.queue.size > 0 }
607
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
608
+
609
+ assert{ @i.write_count > 0 }
610
+ assert{ @i.num_errors > 0 }
611
+
612
+ prev_write_count = @i.write_count
613
+ prev_num_errors = @i.num_errors
614
+
615
+ first_failure = @i.retry.start
616
+
617
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
618
+
619
+ now = first_failure + 60 * 0.8 + 1
620
+
621
+ Timecop.freeze( now )
622
+ @i.enqueue_thread_wait
623
+ @i.flush_thread_wakeup
624
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
625
+
626
+ assert{ @i.write_count > prev_write_count }
627
+ assert{ @i.num_errors > prev_num_errors }
628
+
629
+ assert @i.retry
630
+
631
+ assert_equal 3, (@i.next_flush_time - Time.now)
632
+
633
+ logs = @i.log.out.logs
634
+ assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } }
635
+ end
636
+ end
637
+
638
+ sub_test_case 'secondary plugin feature for buffered output with exponential backoff' do
639
+ setup do
640
+ Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput)
641
+ Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2)
642
+ end
643
+
644
+ test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do
645
+ written = []
646
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})
647
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
648
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
649
+ @i.register(:prefer_buffered_processing){ true }
650
+ @i.register(:prefer_delayed_commit){ false }
651
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
652
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
653
+ @i.secondary.register(:prefer_delayed_commit){ false }
654
+ @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
655
+ @i.start
656
+ @i.after_start
657
+
658
+ @i.interrupt_flushes
659
+
660
+ now = Time.parse('2016-04-13 18:33:30 -0700')
661
+ Timecop.freeze( now )
662
+
663
+ @i.emit_events("test.tag.1", dummy_event_stream())
664
+
665
+ now = Time.parse('2016-04-13 18:33:31 -0700')
666
+ Timecop.freeze( now )
667
+
668
+ @i.emit_events("test.tag.2", dummy_event_stream())
669
+
670
+ assert_equal 0, @i.write_count
671
+ assert_equal 0, @i.num_errors
672
+
673
+ @i.enqueue_thread_wait
674
+ @i.flush_thread_wakeup
675
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
676
+
677
+ assert{ @i.buffer.queue.size > 0 }
678
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
679
+
680
+ assert{ @i.write_count > 0 }
681
+ assert{ @i.num_errors > 0 }
682
+
683
+ prev_write_count = @i.write_count
684
+ prev_num_errors = @i.num_errors
685
+
686
+ first_failure = @i.retry.start
687
+
688
+ 20.times do |i| # large enough
689
+ now = @i.next_flush_time
690
+ Timecop.freeze( now )
691
+ @i.enqueue_thread_wait
692
+ @i.flush_thread_wakeup
693
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
694
+
695
+ assert{ @i.write_count > prev_write_count }
696
+
697
+ break if @i.buffer.queue.size == 0
698
+
699
+ prev_write_count = @i.write_count
700
+ prev_num_errors = @i.num_errors
701
+ end
702
+
703
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
704
+
705
+ assert{ now >= first_failure + 60 * 0.8 }
706
+
707
+ assert_nil @i.retry
708
+
709
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0]
710
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
711
+ assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
712
+
713
+ assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
714
+ end
715
+
716
+ test 'exponential backoff interval will be initialized when switched to secondary' do
717
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :exponential_backoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false})
718
+ secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
719
+ @i.configure(config_element('ROOT','',{},[priconf,secconf]))
720
+ @i.register(:prefer_buffered_processing){ true }
721
+ @i.register(:prefer_delayed_commit){ false }
722
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
723
+ @i.register(:write){|chunk| raise "yay, your #write must fail" }
724
+ @i.secondary.register(:prefer_delayed_commit){ false }
725
+ @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
726
+ @i.start
727
+ @i.after_start
728
+
729
+ @i.interrupt_flushes
730
+
731
+ now = Time.parse('2016-04-13 18:33:30 -0700')
732
+ Timecop.freeze( now )
733
+
734
+ @i.emit_events("test.tag.1", dummy_event_stream())
735
+
736
+ now = Time.parse('2016-04-13 18:33:31 -0700')
737
+ Timecop.freeze( now )
738
+
739
+ @i.emit_events("test.tag.2", dummy_event_stream())
740
+
741
+ assert_equal 0, @i.write_count
742
+ assert_equal 0, @i.num_errors
743
+
744
+ @i.enqueue_thread_wait
745
+ @i.flush_thread_wakeup
746
+ waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
747
+
748
+ assert{ @i.buffer.queue.size > 0 }
749
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
750
+
751
+ assert{ @i.write_count > 0 }
752
+ assert{ @i.num_errors > 0 }
753
+
754
+ prev_write_count = @i.write_count
755
+ prev_num_errors = @i.num_errors
756
+
757
+ first_failure = @i.retry.start
758
+
759
+ 20.times do |i| # large enough
760
+ now = @i.next_flush_time
761
+ # p({i: i, now: now, diff: (now - Time.now)})
762
+ # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0}
763
+ # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0}
764
+ # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0}
765
+ # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0}
766
+ # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0}
767
+ # {:i=>5, :now=>2016-04-13 18:34:19 -0700, :diff=>17.0}
768
+ Timecop.freeze( now )
769
+ @i.enqueue_thread_wait
770
+ @i.flush_thread_wakeup
771
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
772
+
773
+ assert{ @i.write_count > prev_write_count }
774
+ assert{ @i.num_errors > prev_num_errors }
775
+
776
+ prev_write_count = @i.write_count
777
+ prev_num_errors = @i.num_errors
778
+
779
+ break if @i.retry.secondary?
780
+
781
+ assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
782
+ end
783
+
784
+ # retry_timeout == 60(sec), retry_secondary_threshold == 0.8
785
+
786
+ assert{ now >= first_failure + 60 * 0.8 }
787
+ assert @i.retry
788
+ logs = @i.log.out.logs
789
+ assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } }
790
+
791
+ assert{ (@i.next_flush_time - Time.now) <= 2 } # <= retry_wait (1s) * base (2) ** 1
792
+
793
+ 20.times do |i| # large enough again
794
+ now = @i.next_flush_time
795
+ # p({i: i, now: now, diff: (now - Time.now)})
796
+ # {:i=>0, :now=>2016-04-13 18:34:20 -0700, :diff=>1.0}
797
+ # {:i=>1, :now=>2016-04-13 18:34:24 -0700, :diff=>4.0}
798
+ # {:i=>2, :now=>2016-04-13 18:34:31 -0700, :diff=>7.0}
799
+
800
+ Timecop.freeze( now )
801
+ @i.enqueue_thread_wait
802
+ @i.flush_thread_wakeup
803
+ waiting(4){ Thread.pass until @i.write_count > prev_write_count }
804
+
805
+ assert{ @i.write_count > prev_write_count }
806
+ assert{ @i.num_errors > prev_num_errors }
807
+
808
+ break if @i.buffer.queue.size == 0
809
+ end
810
+
811
+ logs = @i.log.out.logs
812
+ assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
813
+
814
+ assert{ now >= first_failure + 60 }
815
+ end
816
+ end
817
+ end