fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,263 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/parser'
3
+ require 'fluent/plugin/base'
4
+ require 'fluent/time'
5
+
6
+ class ParserHelperTest < Test::Unit::TestCase
7
+ class ExampleParser < Fluent::Plugin::Parser
8
+ Fluent::Plugin.register_parser('example', self)
9
+ def parse(text)
10
+ ary = text.split(/\s*,\s*/)
11
+ r = {}
12
+ ary.each_with_index do |v, i|
13
+ r[i.to_s] = v
14
+ end
15
+ yield Fluent::EventTime.now, r
16
+ end
17
+ end
18
+ class Example2Parser < Fluent::Plugin::Parser
19
+ Fluent::Plugin.register_parser('example2', self)
20
+ def parse(text)
21
+ ary = text.split(/\s*,\s*/)
22
+ r = {}
23
+ ary.each_with_index do |v, i|
24
+ r[i.to_s] = v
25
+ end
26
+ yield Fluent::EventTime.now, r
27
+ end
28
+ end
29
+ class Dummy < Fluent::Plugin::TestBase
30
+ helpers :parser
31
+ end
32
+
33
+ class Dummy2 < Fluent::Plugin::TestBase
34
+ helpers :parser
35
+ config_section :parse do
36
+ config_set_default :@type, 'example2'
37
+ end
38
+ end
39
+
40
+ setup do
41
+ @d = nil
42
+ end
43
+
44
+ teardown do
45
+ if @d
46
+ @d.stop unless @d.stopped?
47
+ @d.shutdown unless @d.shutdown?
48
+ @d.close unless @d.closed?
49
+ @d.terminate unless @d.terminated?
50
+ end
51
+ end
52
+
53
+ test 'can be initialized without any parsers at first' do
54
+ d = Dummy.new
55
+ assert_equal 0, d._parsers.size
56
+ end
57
+
58
+ test 'can override default configuration parameters, but not overwrite whole definition' do
59
+ d = Dummy.new
60
+ assert_equal [], d.parser_configs
61
+
62
+ d = Dummy2.new
63
+ d.configure(config_element('ROOT', '', {}, [config_element('parse', '', {}, [])]))
64
+ assert_raise NoMethodError do
65
+ d.parse
66
+ end
67
+ assert_equal 1, d.parser_configs.size
68
+ assert_equal 'example2', d.parser_configs.first[:@type]
69
+ end
70
+
71
+ test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
72
+ d = Dummy2.new
73
+ d.configure(config_element())
74
+ i = d.parser_create(conf: config_element('parse', '', {'@type' => 'example'}), default_type: 'example2')
75
+ assert{ i.is_a?(ExampleParser) }
76
+
77
+ d = Dummy2.new
78
+ d.configure(config_element())
79
+ i = d.parser_create(conf: nil, default_type: 'example2')
80
+ assert{ i.is_a?(Example2Parser) }
81
+ end
82
+
83
+ test 'raises config error if config section is specified, but @type is not specified' do
84
+ d = Dummy2.new
85
+ d.configure(config_element())
86
+ assert_raise Fluent::ConfigError.new("@type is required in <parse>") do
87
+ d.parser_create(conf: config_element('parse', '', {}), default_type: 'example2')
88
+ end
89
+ end
90
+
91
+ test 'can be configured without parse sections' do
92
+ d = Dummy.new
93
+ assert_nothing_raised do
94
+ d.configure(config_element())
95
+ end
96
+ assert_equal 0, d._parsers.size
97
+ end
98
+
99
+ test 'can be configured with a parse section' do
100
+ d = Dummy.new
101
+ conf = config_element('ROOT', '', {}, [
102
+ config_element('parse', '', {'@type' => 'example'})
103
+ ])
104
+ assert_nothing_raised do
105
+ d.configure(conf)
106
+ end
107
+ assert_equal 1, d._parsers.size
108
+ assert{ d._parsers.values.all?{ |parser| !parser.started? } }
109
+ end
110
+
111
+ test 'can be configured with 2 or more parse sections with different usages with each other' do
112
+ d = Dummy.new
113
+ conf = config_element('ROOT', '', {}, [
114
+ config_element('parse', 'default', {'@type' => 'example'}),
115
+ config_element('parse', 'extra', {'@type' => 'example2'}),
116
+ ])
117
+ assert_nothing_raised do
118
+ d.configure(conf)
119
+ end
120
+ assert_equal 2, d._parsers.size
121
+ assert{ d._parsers.values.all?{ |parser| !parser.started? } }
122
+ end
123
+
124
+ test 'cannot be configured with 2 parse sections with same usage' do
125
+ d = Dummy.new
126
+ conf = config_element('ROOT', '', {}, [
127
+ config_element('parse', 'default', {'@type' => 'example'}),
128
+ config_element('parse', 'extra', {'@type' => 'example2'}),
129
+ config_element('parse', 'extra', {'@type' => 'example2'}),
130
+ ])
131
+ assert_raises Fluent::ConfigError do
132
+ d.configure(conf)
133
+ end
134
+ end
135
+
136
+ test 'creates a parse plugin instance which is already configured without usage' do
137
+ @d = d = Dummy.new
138
+ conf = config_element('ROOT', '', {}, [
139
+ config_element('parse', '', {'@type' => 'example'})
140
+ ])
141
+ d.configure(conf)
142
+ d.start
143
+
144
+ parser = d.parser_create
145
+ assert{ parser.is_a? ExampleParser }
146
+ assert parser.started?
147
+ end
148
+
149
+ test 'creates a parser plugin instance which is already configured with usage' do
150
+ @d = d = Dummy.new
151
+ conf = config_element('ROOT', '', {}, [
152
+ config_element('parse', 'mydata', {'@type' => 'example'})
153
+ ])
154
+ d.configure(conf)
155
+ d.start
156
+
157
+ parser = d.parser_create(usage: 'mydata')
158
+ assert{ parser.is_a? ExampleParser }
159
+ assert parser.started?
160
+ end
161
+
162
+ test 'creates a parser plugin without configurations' do
163
+ @d = d = Dummy.new
164
+ d.configure(config_element())
165
+ d.start
166
+
167
+ parser = d.parser_create(usage: 'mydata', type: 'example', conf: config_element('parse', 'mydata'))
168
+ assert{ parser.is_a? ExampleParser }
169
+ assert parser.started?
170
+ end
171
+
172
+ test 'creates 2 or more parser plugin instances' do
173
+ @d = d = Dummy.new
174
+ conf = config_element('ROOT', '', {}, [
175
+ config_element('parse', 'mydata', {'@type' => 'example'}),
176
+ config_element('parse', 'secret', {'@type' => 'example2'})
177
+ ])
178
+ d.configure(conf)
179
+ d.start
180
+
181
+ p1 = d.parser_create(usage: 'mydata')
182
+ p2 = d.parser_create(usage: 'secret')
183
+ assert{ p1.is_a? ExampleParser }
184
+ assert p1.started?
185
+ assert{ p2.is_a? Example2Parser }
186
+ assert p2.started?
187
+ end
188
+
189
+ test 'calls lifecycle methods for all plugin instances via owner plugin' do
190
+ @d = d = Dummy.new
191
+ conf = config_element('ROOT', '', {}, [ config_element('parse', '', {'@type' => 'example'}), config_element('parse', 'e2', {'@type' => 'example'}) ])
192
+ d.configure(conf)
193
+ d.start
194
+
195
+ i1 = d.parser_create(usage: '')
196
+ i2 = d.parser_create(usage: 'e2')
197
+ i3 = d.parser_create(usage: 'e3', type: 'example2')
198
+
199
+ assert i1.started?
200
+ assert i2.started?
201
+ assert i3.started?
202
+
203
+ assert !i1.stopped?
204
+ assert !i2.stopped?
205
+ assert !i3.stopped?
206
+
207
+ d.stop
208
+
209
+ assert i1.stopped?
210
+ assert i2.stopped?
211
+ assert i3.stopped?
212
+
213
+ assert !i1.before_shutdown?
214
+ assert !i2.before_shutdown?
215
+ assert !i3.before_shutdown?
216
+
217
+ d.before_shutdown
218
+
219
+ assert i1.before_shutdown?
220
+ assert i2.before_shutdown?
221
+ assert i3.before_shutdown?
222
+
223
+ assert !i1.shutdown?
224
+ assert !i2.shutdown?
225
+ assert !i3.shutdown?
226
+
227
+ d.shutdown
228
+
229
+ assert i1.shutdown?
230
+ assert i2.shutdown?
231
+ assert i3.shutdown?
232
+
233
+ assert !i1.after_shutdown?
234
+ assert !i2.after_shutdown?
235
+ assert !i3.after_shutdown?
236
+
237
+ d.after_shutdown
238
+
239
+ assert i1.after_shutdown?
240
+ assert i2.after_shutdown?
241
+ assert i3.after_shutdown?
242
+
243
+ assert !i1.closed?
244
+ assert !i2.closed?
245
+ assert !i3.closed?
246
+
247
+ d.close
248
+
249
+ assert i1.closed?
250
+ assert i2.closed?
251
+ assert i3.closed?
252
+
253
+ assert !i1.terminated?
254
+ assert !i2.terminated?
255
+ assert !i3.terminated?
256
+
257
+ d.terminate
258
+
259
+ assert i1.terminated?
260
+ assert i2.terminated?
261
+ assert i3.terminated?
262
+ end
263
+ end
@@ -0,0 +1,399 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/retry_state'
3
+ require 'fluent/plugin/base'
4
+
5
+ require 'time'
6
+
7
+ class RetryStateHelperTest < Test::Unit::TestCase
8
+ def override_current_time(state, time)
9
+ mojule = Module.new do
10
+ define_method(:current_time){ time }
11
+ end
12
+ state.singleton_class.module_eval do
13
+ prepend mojule
14
+ end
15
+ end
16
+
17
+ class Dummy < Fluent::Plugin::TestBase
18
+ helpers :retry_state
19
+ end
20
+
21
+ setup do
22
+ @d = Dummy.new
23
+ end
24
+
25
+ test 'randomize can generate value within specified +/- range' do
26
+ s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30) # default enabled w/ 0.125
27
+ 500.times do
28
+ r = s.randomize(1000)
29
+ assert{ r >= 875 && r < 1125 }
30
+ end
31
+
32
+ s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30, randomize_width: 0.25)
33
+ 500.times do
34
+ r = s.randomize(1000)
35
+ assert{ r >= 750 && r < 1250 }
36
+ end
37
+ end
38
+
39
+ test 'plugin can create retry_state machine' do
40
+ s = @d.retry_state_create(:t1, :exponential_backoff, 0.1, 30)
41
+ # attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_times
42
+
43
+ assert_equal :t1, s.title
44
+ start_time = s.start
45
+
46
+ assert_equal 0, s.steps
47
+ assert_equal (start_time + 0.1).to_i, s.next_time.to_i
48
+ assert_equal (start_time + 0.1).nsec, s.next_time.nsec
49
+ assert_equal (start_time + 30), s.timeout_at
50
+
51
+ assert_equal :primary, s.current
52
+ assert{ s.is_a? Fluent::PluginHelper::RetryState::ExponentialBackOffRetry }
53
+ end
54
+
55
+ test 'periodic retries' do
56
+ s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false)
57
+ dummy_current_time = s.start
58
+ override_current_time(s, dummy_current_time)
59
+
60
+ assert_equal dummy_current_time, s.current_time
61
+ assert_equal (dummy_current_time + 29), s.timeout_at
62
+ assert_equal (dummy_current_time + 3), s.next_time
63
+
64
+ i = 1
65
+ while i < 9
66
+ override_current_time(s, s.next_time)
67
+ s.step
68
+ assert_equal i, s.steps
69
+ assert_equal (s.current_time + 3), s.next_time
70
+ assert !s.limit?
71
+ i += 1
72
+ end
73
+
74
+ assert_equal 9, i
75
+ override_current_time(s, s.next_time)
76
+ s.step
77
+ assert_equal s.timeout_at, s.next_time
78
+ assert s.limit?
79
+ end
80
+
81
+ test 'periodic retries with max_steps' do
82
+ s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false, max_steps: 5)
83
+ dummy_current_time = s.start
84
+ override_current_time(s, dummy_current_time)
85
+
86
+ assert_equal dummy_current_time, s.current_time
87
+ assert_equal (dummy_current_time + 29), s.timeout_at
88
+ assert_equal (dummy_current_time + 3), s.next_time
89
+
90
+ i = 1
91
+ while i < 5
92
+ override_current_time(s, s.next_time)
93
+ s.step
94
+ assert_equal i, s.steps
95
+ assert_equal (s.current_time + 3), s.next_time
96
+ assert !s.limit?
97
+ i += 1
98
+ end
99
+
100
+ assert_equal 5, i
101
+ override_current_time(s, s.next_time)
102
+ s.step
103
+ assert_equal (s.current_time + 3), s.next_time
104
+ assert s.limit?
105
+ end
106
+
107
+ test 'periodic retries with secondary' do
108
+ s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true) # threshold 0.8
109
+ dummy_current_time = s.start
110
+ override_current_time(s, dummy_current_time)
111
+
112
+ assert_equal dummy_current_time, s.current_time
113
+ assert_equal (dummy_current_time + 100), s.timeout_at
114
+ assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at
115
+
116
+ assert_equal (dummy_current_time + 3), s.next_time
117
+ assert !s.secondary?
118
+
119
+ i = 1
120
+ while i < 26
121
+ override_current_time(s, s.next_time)
122
+ assert !s.secondary?
123
+
124
+ s.step
125
+ assert_equal i, s.steps
126
+ assert_equal (s.current_time + 3), s.next_time
127
+ assert !s.limit?
128
+ i += 1
129
+ end
130
+
131
+ assert_equal 26, i
132
+ override_current_time(s, s.next_time) # 78
133
+ assert !s.secondary?
134
+
135
+ s.step
136
+ assert_equal 26, s.steps
137
+ assert_equal s.secondary_transition_at, s.next_time
138
+ assert !s.limit?
139
+
140
+ i += 1
141
+ assert_equal 27, i
142
+ override_current_time(s, s.next_time) # 80
143
+ assert s.secondary?
144
+
145
+ s.step
146
+ assert_equal (s.current_time + 3), s.next_time
147
+ assert_equal s.steps, s.secondary_transition_steps
148
+ assert !s.limit?
149
+
150
+ i += 1
151
+
152
+ while i < 33
153
+ override_current_time(s, s.next_time)
154
+ assert s.secondary?
155
+
156
+ s.step
157
+ assert_equal (s.current_time + 3), s.next_time
158
+ assert !s.limit?
159
+ i += 1
160
+ end
161
+
162
+ assert_equal 33, i
163
+ override_current_time(s, s.next_time) # 98
164
+ assert s.secondary?
165
+
166
+ s.step
167
+ assert_equal s.timeout_at, s.next_time
168
+ assert s.limit?
169
+ end
170
+
171
+ test 'periodic retries with secondary and specified threshold' do
172
+ s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true, secondary_threshold: 0.75)
173
+ dummy_current_time = s.start
174
+ override_current_time(s, dummy_current_time)
175
+
176
+ assert_equal dummy_current_time, s.current_time
177
+ assert_equal (dummy_current_time + 100), s.timeout_at
178
+ assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
179
+ end
180
+
181
+ test 'exponential backoff forever without randomization' do
182
+ s = @d.retry_state_create(:t11, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2)
183
+ dummy_current_time = s.start
184
+ override_current_time(s, dummy_current_time)
185
+
186
+ assert_equal dummy_current_time, s.current_time
187
+
188
+ assert_equal 0, s.steps
189
+ assert_equal (dummy_current_time + 0.1), s.next_time
190
+
191
+ i = 1
192
+ while i < 300
193
+ s.step
194
+ assert_equal i, s.steps
195
+ assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time
196
+ assert !s.limit?
197
+ i += 1
198
+ end
199
+ end
200
+
201
+ test 'exponential backoff with max_interval' do
202
+ s = @d.retry_state_create(:t12, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2, max_interval: 100)
203
+ dummy_current_time = s.start
204
+ override_current_time(s, dummy_current_time)
205
+
206
+ assert_equal dummy_current_time, s.current_time
207
+
208
+ assert_equal 0, s.steps
209
+ assert_equal (dummy_current_time + 0.1), s.next_time
210
+
211
+ # 0.1 * (2 ** (10 - 1)) == 0.1 * 2 ** 9 == 51.2
212
+ # 0.1 * (2 ** (11 - 1)) == 0.1 * 2 ** 10 == 102.4
213
+ i = 1
214
+ while i < 11
215
+ s.step
216
+ assert_equal i, s.steps
217
+ assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time, "start:#{dummy_current_time}, i:#{i}"
218
+ i += 1
219
+ end
220
+
221
+ s.step
222
+ assert_equal 11, s.steps
223
+ assert_equal (dummy_current_time + 100), s.next_time
224
+
225
+ s.step
226
+ assert_equal 12, s.steps
227
+ assert_equal (dummy_current_time + 100), s.next_time
228
+ end
229
+
230
+ test 'exponential backoff with shorter timeout' do
231
+ s = @d.retry_state_create(:t13, :exponential_backoff, 1, 12, randomize: false, backoff_base: 2, max_interval: 10)
232
+ dummy_current_time = s.start
233
+ override_current_time(s, dummy_current_time)
234
+
235
+ assert_equal dummy_current_time, s.current_time
236
+
237
+ assert_equal (dummy_current_time + 12), s.timeout_at
238
+
239
+ assert_equal 0, s.steps
240
+ assert_equal (dummy_current_time + 1), s.next_time
241
+
242
+ # 1 + 1 + 2 + 4 (=8)
243
+
244
+ override_current_time(s, s.next_time)
245
+ s.step
246
+ assert_equal 1, s.steps
247
+ assert_equal (s.current_time + 1), s.next_time
248
+
249
+ override_current_time(s, s.next_time)
250
+ s.step
251
+ assert_equal 2, s.steps
252
+ assert_equal (s.current_time + 2), s.next_time
253
+
254
+ override_current_time(s, s.next_time)
255
+ s.step
256
+ assert_equal 3, s.steps
257
+ assert_equal (s.current_time + 4), s.next_time
258
+
259
+ assert !s.limit?
260
+
261
+ # + 8 (=16) > 12
262
+
263
+ override_current_time(s, s.next_time)
264
+ s.step
265
+ assert_equal 4, s.steps
266
+ assert_equal s.timeout_at, s.next_time
267
+
268
+ assert s.limit?
269
+ end
270
+
271
+ test 'exponential backoff with max_steps' do
272
+ s = @d.retry_state_create(:t14, :exponential_backoff, 1, 120, randomize: false, backoff_base: 2, max_interval: 10, max_steps: 6)
273
+ dummy_current_time = s.start
274
+ override_current_time(s, dummy_current_time)
275
+
276
+ assert_equal dummy_current_time, s.current_time
277
+
278
+ assert_equal (dummy_current_time + 120), s.timeout_at
279
+
280
+ assert_equal 0, s.steps
281
+ assert_equal (dummy_current_time + 1), s.next_time
282
+
283
+ override_current_time(s, s.next_time)
284
+ s.step
285
+ assert_equal 1, s.steps
286
+ assert_equal (s.current_time + 1), s.next_time
287
+
288
+ override_current_time(s, s.next_time)
289
+ s.step
290
+ assert_equal 2, s.steps
291
+ assert_equal (s.current_time + 2), s.next_time
292
+
293
+ override_current_time(s, s.next_time)
294
+ s.step
295
+ assert_equal 3, s.steps
296
+ assert_equal (s.current_time + 4), s.next_time
297
+
298
+ assert !s.limit?
299
+
300
+ override_current_time(s, s.next_time)
301
+ s.step
302
+ assert_equal 4, s.steps
303
+ assert_equal (s.current_time + 8), s.next_time
304
+
305
+ assert !s.limit?
306
+
307
+ override_current_time(s, s.next_time)
308
+ s.step
309
+ assert_equal 5, s.steps
310
+ assert_equal (s.current_time + 10), s.next_time
311
+
312
+ assert !s.limit?
313
+
314
+ override_current_time(s, s.next_time)
315
+ s.step
316
+ assert_equal 6, s.steps
317
+ assert_equal (s.current_time + 10), s.next_time
318
+
319
+ assert s.limit?
320
+ end
321
+
322
+ test 'exponential backoff retries with secondary' do
323
+ s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, backoff_base: 2, secondary: true) # threshold 0.8
324
+ dummy_current_time = s.start
325
+ override_current_time(s, dummy_current_time)
326
+
327
+ assert_equal dummy_current_time, s.current_time
328
+ assert_equal (dummy_current_time + 100), s.timeout_at
329
+ assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at
330
+
331
+ assert_equal (dummy_current_time + 1), s.next_time
332
+ assert !s.secondary?
333
+
334
+ # 1, 1(2), 2(4), 4(8), 8(16), 16(32), 32(64), (80), (81), (83), (87), (95), (100)
335
+ i = 1
336
+ while i < 7
337
+ override_current_time(s, s.next_time)
338
+ assert !s.secondary?
339
+
340
+ s.step
341
+ assert_equal i, s.steps
342
+ assert_equal (s.current_time + 1 * (2 ** (i - 1))), s.next_time
343
+ assert !s.limit?
344
+ i += 1
345
+ end
346
+
347
+ assert_equal 7, i
348
+ override_current_time(s, s.next_time) # 64
349
+ assert !s.secondary?
350
+
351
+ s.step
352
+ assert_equal 7, s.steps
353
+ assert_equal s.secondary_transition_at, s.next_time
354
+ assert !s.limit?
355
+
356
+ i += 1
357
+ assert_equal 8, i
358
+ override_current_time(s, s.next_time) # 80
359
+ assert s.secondary?
360
+
361
+ s.step
362
+ assert_equal 8, s.steps
363
+ assert_equal s.steps, s.secondary_transition_steps
364
+ assert_equal (s.secondary_transition_at + 1.0), s.next_time
365
+ assert !s.limit?
366
+
367
+ # 81, 82, 84, 88, 96, 100
368
+ j = 1
369
+ while j < 4
370
+ override_current_time(s, s.next_time)
371
+ assert s.secondary?
372
+ assert_equal :secondary, s.current
373
+
374
+ s.step
375
+ assert_equal (8 + j), s.steps
376
+ assert_equal (s.current_time + (1 * (2 ** j))), s.next_time
377
+ assert !s.limit?, "j:#{j}"
378
+ j += 1
379
+ end
380
+
381
+ assert_equal 4, j
382
+ override_current_time(s, s.next_time) # 96
383
+ assert s.secondary?
384
+
385
+ s.step
386
+ assert_equal s.timeout_at, s.next_time
387
+ assert s.limit?
388
+ end
389
+
390
+ test 'exponential backoff retries with secondary and specified threshold' do
391
+ s = @d.retry_state_create(:t16, :exponential_backoff, 1, 100, randomize: false, secondary: true, backoff_base: 2, secondary_threshold: 0.75)
392
+ dummy_current_time = s.start
393
+ override_current_time(s, dummy_current_time)
394
+
395
+ assert_equal dummy_current_time, s.current_time
396
+ assert_equal (dummy_current_time + 100), s.timeout_at
397
+ assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
398
+ end
399
+ end