fluentd 0.12.40 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (252) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +2 -0
  4. data/.travis.yml +33 -21
  5. data/CONTRIBUTING.md +1 -0
  6. data/ChangeLog +810 -237
  7. data/README.md +0 -25
  8. data/Rakefile +2 -1
  9. data/Vagrantfile +17 -0
  10. data/appveyor.yml +35 -0
  11. data/example/filter_stdout.conf +5 -5
  12. data/example/in_forward.conf +2 -2
  13. data/example/in_http.conf +2 -2
  14. data/example/in_out_forward.conf +17 -0
  15. data/example/in_syslog.conf +2 -2
  16. data/example/in_tail.conf +2 -2
  17. data/example/in_tcp.conf +2 -2
  18. data/example/in_udp.conf +2 -2
  19. data/example/out_copy.conf +4 -4
  20. data/example/out_file.conf +2 -2
  21. data/example/out_forward.conf +2 -2
  22. data/example/out_forward_buf_file.conf +23 -0
  23. data/example/v0_12_filter.conf +8 -8
  24. data/fluent.conf +29 -0
  25. data/fluentd.gemspec +18 -11
  26. data/lib/fluent/agent.rb +60 -58
  27. data/lib/fluent/command/cat.rb +1 -1
  28. data/lib/fluent/command/debug.rb +7 -5
  29. data/lib/fluent/command/fluentd.rb +97 -2
  30. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  31. data/lib/fluent/compat/filter.rb +50 -0
  32. data/lib/fluent/compat/formatter.rb +109 -0
  33. data/lib/fluent/compat/input.rb +50 -0
  34. data/lib/fluent/compat/output.rb +617 -0
  35. data/lib/fluent/compat/output_chain.rb +60 -0
  36. data/lib/fluent/compat/parser.rb +163 -0
  37. data/lib/fluent/compat/propagate_default.rb +62 -0
  38. data/lib/fluent/config.rb +23 -20
  39. data/lib/fluent/config/configure_proxy.rb +119 -70
  40. data/lib/fluent/config/dsl.rb +5 -18
  41. data/lib/fluent/config/element.rb +72 -8
  42. data/lib/fluent/config/error.rb +0 -3
  43. data/lib/fluent/config/literal_parser.rb +0 -2
  44. data/lib/fluent/config/parser.rb +4 -4
  45. data/lib/fluent/config/section.rb +39 -28
  46. data/lib/fluent/config/types.rb +2 -13
  47. data/lib/fluent/config/v1_parser.rb +1 -3
  48. data/lib/fluent/configurable.rb +48 -16
  49. data/lib/fluent/daemon.rb +15 -0
  50. data/lib/fluent/engine.rb +26 -52
  51. data/lib/fluent/env.rb +6 -4
  52. data/lib/fluent/event.rb +58 -11
  53. data/lib/fluent/event_router.rb +5 -5
  54. data/lib/fluent/filter.rb +2 -50
  55. data/lib/fluent/formatter.rb +4 -293
  56. data/lib/fluent/input.rb +2 -32
  57. data/lib/fluent/label.rb +2 -2
  58. data/lib/fluent/load.rb +3 -2
  59. data/lib/fluent/log.rb +107 -38
  60. data/lib/fluent/match.rb +0 -36
  61. data/lib/fluent/mixin.rb +117 -7
  62. data/lib/fluent/msgpack_factory.rb +62 -0
  63. data/lib/fluent/output.rb +7 -612
  64. data/lib/fluent/output_chain.rb +23 -0
  65. data/lib/fluent/parser.rb +4 -800
  66. data/lib/fluent/plugin.rb +100 -121
  67. data/lib/fluent/plugin/bare_output.rb +63 -0
  68. data/lib/fluent/plugin/base.rb +121 -0
  69. data/lib/fluent/plugin/buf_file.rb +101 -182
  70. data/lib/fluent/plugin/buf_memory.rb +9 -92
  71. data/lib/fluent/plugin/buffer.rb +473 -0
  72. data/lib/fluent/plugin/buffer/chunk.rb +135 -0
  73. data/lib/fluent/plugin/buffer/file_chunk.rb +339 -0
  74. data/lib/fluent/plugin/buffer/memory_chunk.rb +100 -0
  75. data/lib/fluent/plugin/exec_util.rb +80 -75
  76. data/lib/fluent/plugin/file_util.rb +33 -28
  77. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  78. data/lib/fluent/plugin/filter.rb +51 -0
  79. data/lib/fluent/plugin/filter_grep.rb +13 -40
  80. data/lib/fluent/plugin/filter_record_transformer.rb +22 -18
  81. data/lib/fluent/plugin/formatter.rb +93 -0
  82. data/lib/fluent/plugin/formatter_csv.rb +48 -0
  83. data/lib/fluent/plugin/formatter_hash.rb +32 -0
  84. data/lib/fluent/plugin/formatter_json.rb +47 -0
  85. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  86. data/lib/fluent/plugin/formatter_msgpack.rb +32 -0
  87. data/lib/fluent/plugin/formatter_out_file.rb +45 -0
  88. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  89. data/lib/fluent/plugin/formatter_stdout.rb +39 -0
  90. data/lib/fluent/plugin/in_debug_agent.rb +4 -0
  91. data/lib/fluent/plugin/in_dummy.rb +22 -18
  92. data/lib/fluent/plugin/in_exec.rb +18 -8
  93. data/lib/fluent/plugin/in_forward.rb +36 -79
  94. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  95. data/lib/fluent/plugin/in_http.rb +21 -18
  96. data/lib/fluent/plugin/in_monitor_agent.rb +15 -48
  97. data/lib/fluent/plugin/in_object_space.rb +6 -1
  98. data/lib/fluent/plugin/in_stream.rb +7 -3
  99. data/lib/fluent/plugin/in_syslog.rb +46 -95
  100. data/lib/fluent/plugin/in_tail.rb +51 -595
  101. data/lib/fluent/plugin/in_tcp.rb +8 -1
  102. data/lib/fluent/plugin/in_udp.rb +8 -14
  103. data/lib/fluent/plugin/input.rb +33 -0
  104. data/lib/fluent/plugin/multi_output.rb +95 -0
  105. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  106. data/lib/fluent/plugin/out_copy.rb +11 -7
  107. data/lib/fluent/plugin/out_exec.rb +15 -11
  108. data/lib/fluent/plugin/out_exec_filter.rb +18 -10
  109. data/lib/fluent/plugin/out_file.rb +34 -5
  110. data/lib/fluent/plugin/out_forward.rb +19 -9
  111. data/lib/fluent/plugin/out_null.rb +0 -14
  112. data/lib/fluent/plugin/out_roundrobin.rb +11 -7
  113. data/lib/fluent/plugin/out_stdout.rb +5 -7
  114. data/lib/fluent/plugin/out_stream.rb +3 -1
  115. data/lib/fluent/plugin/output.rb +979 -0
  116. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  117. data/lib/fluent/plugin/parser.rb +244 -0
  118. data/lib/fluent/plugin/parser_apache.rb +24 -0
  119. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  120. data/lib/fluent/plugin/parser_apache_error.rb +21 -0
  121. data/lib/fluent/plugin/parser_csv.rb +31 -0
  122. data/lib/fluent/plugin/parser_json.rb +79 -0
  123. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  124. data/lib/fluent/plugin/parser_multiline.rb +102 -0
  125. data/lib/fluent/plugin/parser_nginx.rb +24 -0
  126. data/lib/fluent/plugin/parser_none.rb +36 -0
  127. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  128. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  129. data/lib/fluent/plugin/socket_util.rb +120 -114
  130. data/lib/fluent/plugin/storage.rb +84 -0
  131. data/lib/fluent/plugin/storage_local.rb +116 -0
  132. data/lib/fluent/plugin/string_util.rb +16 -13
  133. data/lib/fluent/plugin_helper.rb +39 -0
  134. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  135. data/lib/fluent/plugin_helper/compat_parameters.rb +99 -0
  136. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  137. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  138. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  139. data/lib/fluent/plugin_helper/storage.rb +308 -0
  140. data/lib/fluent/plugin_helper/thread.rb +147 -0
  141. data/lib/fluent/plugin_helper/timer.rb +85 -0
  142. data/lib/fluent/plugin_id.rb +63 -0
  143. data/lib/fluent/process.rb +21 -30
  144. data/lib/fluent/registry.rb +21 -9
  145. data/lib/fluent/root_agent.rb +115 -40
  146. data/lib/fluent/supervisor.rb +330 -320
  147. data/lib/fluent/system_config.rb +42 -18
  148. data/lib/fluent/test.rb +6 -1
  149. data/lib/fluent/test/base.rb +23 -3
  150. data/lib/fluent/test/driver/base.rb +247 -0
  151. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  152. data/lib/fluent/test/driver/filter.rb +35 -0
  153. data/lib/fluent/test/driver/input.rb +31 -0
  154. data/lib/fluent/test/driver/output.rb +78 -0
  155. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  156. data/lib/fluent/test/filter_test.rb +0 -1
  157. data/lib/fluent/test/formatter_test.rb +2 -1
  158. data/lib/fluent/test/input_test.rb +23 -17
  159. data/lib/fluent/test/output_test.rb +28 -39
  160. data/lib/fluent/test/parser_test.rb +1 -1
  161. data/lib/fluent/time.rb +104 -1
  162. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  163. data/lib/fluent/version.rb +1 -1
  164. data/lib/fluent/winsvc.rb +72 -0
  165. data/test/compat/test_calls_super.rb +164 -0
  166. data/test/config/test_config_parser.rb +83 -0
  167. data/test/config/test_configurable.rb +547 -274
  168. data/test/config/test_configure_proxy.rb +146 -29
  169. data/test/config/test_dsl.rb +3 -181
  170. data/test/config/test_element.rb +274 -0
  171. data/test/config/test_literal_parser.rb +1 -1
  172. data/test/config/test_section.rb +79 -7
  173. data/test/config/test_system_config.rb +21 -0
  174. data/test/config/test_types.rb +3 -26
  175. data/test/helper.rb +78 -8
  176. data/test/plugin/test_bare_output.rb +118 -0
  177. data/test/plugin/test_base.rb +75 -0
  178. data/test/plugin/test_buf_file.rb +420 -521
  179. data/test/plugin/test_buf_memory.rb +32 -194
  180. data/test/plugin/test_buffer.rb +981 -0
  181. data/test/plugin/test_buffer_chunk.rb +110 -0
  182. data/test/plugin/test_buffer_file_chunk.rb +770 -0
  183. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  184. data/test/plugin/test_filter.rb +255 -0
  185. data/test/plugin/test_filter_grep.rb +2 -73
  186. data/test/plugin/test_filter_record_transformer.rb +24 -68
  187. data/test/plugin/test_filter_stdout.rb +6 -6
  188. data/test/plugin/test_in_debug_agent.rb +2 -0
  189. data/test/plugin/test_in_dummy.rb +11 -17
  190. data/test/plugin/test_in_exec.rb +6 -25
  191. data/test/plugin/test_in_forward.rb +112 -151
  192. data/test/plugin/test_in_gc_stat.rb +2 -0
  193. data/test/plugin/test_in_http.rb +106 -157
  194. data/test/plugin/test_in_object_space.rb +21 -5
  195. data/test/plugin/test_in_stream.rb +14 -13
  196. data/test/plugin/test_in_syslog.rb +30 -275
  197. data/test/plugin/test_in_tail.rb +95 -234
  198. data/test/plugin/test_in_tcp.rb +14 -0
  199. data/test/plugin/test_in_udp.rb +21 -13
  200. data/test/plugin/test_input.rb +122 -0
  201. data/test/plugin/test_multi_output.rb +180 -0
  202. data/test/plugin/test_out_buffered_null.rb +79 -0
  203. data/test/plugin/test_out_copy.rb +15 -2
  204. data/test/plugin/test_out_exec.rb +75 -25
  205. data/test/plugin/test_out_exec_filter.rb +74 -8
  206. data/test/plugin/test_out_file.rb +61 -7
  207. data/test/plugin/test_out_forward.rb +92 -15
  208. data/test/plugin/test_out_roundrobin.rb +1 -0
  209. data/test/plugin/test_out_stdout.rb +22 -13
  210. data/test/plugin/test_out_stream.rb +18 -0
  211. data/test/plugin/test_output.rb +515 -0
  212. data/test/plugin/test_output_as_buffered.rb +1540 -0
  213. data/test/plugin/test_output_as_buffered_overflow.rb +247 -0
  214. data/test/plugin/test_output_as_buffered_retries.rb +808 -0
  215. data/test/plugin/test_output_as_buffered_secondary.rb +776 -0
  216. data/test/plugin/test_output_as_standard.rb +362 -0
  217. data/test/plugin/test_owned_by.rb +35 -0
  218. data/test/plugin/test_storage.rb +167 -0
  219. data/test/plugin/test_storage_local.rb +8 -0
  220. data/test/plugin_helper/test_child_process.rb +599 -0
  221. data/test/plugin_helper/test_compat_parameters.rb +175 -0
  222. data/test/plugin_helper/test_event_emitter.rb +51 -0
  223. data/test/plugin_helper/test_event_loop.rb +52 -0
  224. data/test/plugin_helper/test_retry_state.rb +399 -0
  225. data/test/plugin_helper/test_storage.rb +411 -0
  226. data/test/plugin_helper/test_thread.rb +164 -0
  227. data/test/plugin_helper/test_timer.rb +100 -0
  228. data/test/scripts/exec_script.rb +0 -6
  229. data/test/scripts/fluent/plugin/out_test.rb +3 -0
  230. data/test/test_config.rb +13 -4
  231. data/test/test_event.rb +24 -13
  232. data/test/test_event_router.rb +8 -7
  233. data/test/test_event_time.rb +187 -0
  234. data/test/test_formatter.rb +13 -51
  235. data/test/test_input.rb +1 -1
  236. data/test/test_log.rb +239 -16
  237. data/test/test_mixin.rb +1 -1
  238. data/test/test_output.rb +53 -66
  239. data/test/test_parser.rb +105 -323
  240. data/test/test_plugin_helper.rb +81 -0
  241. data/test/test_root_agent.rb +4 -52
  242. data/test/test_supervisor.rb +272 -0
  243. data/test/test_unique_id.rb +47 -0
  244. metadata +180 -54
  245. data/lib/fluent/buffer.rb +0 -365
  246. data/lib/fluent/plugin/filter_parser.rb +0 -107
  247. data/lib/fluent/plugin/in_status.rb +0 -76
  248. data/lib/fluent/test/helpers.rb +0 -86
  249. data/test/plugin/data/log/foo/bar2 +0 -0
  250. data/test/plugin/test_filter_parser.rb +0 -744
  251. data/test/plugin/test_in_status.rb +0 -38
  252. data/test/test_buffer.rb +0 -624
@@ -0,0 +1,99 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/element'
18
+
19
+ module Fluent
20
+ module PluginHelper
21
+ module CompatParameters
22
+ # This plugin helper is to bring old-fashioned buffer/other
23
+ # configuration parameters to v0.14 plugin API configurations.
24
+ # This helper is mainly to convert plugins from v0.12 API
25
+ # to v0.14 API safely, without breaking user deployment.
26
+
27
+ PARAMS_MAP = {
28
+ "buffer_type" => "@type",
29
+ "buffer_path" => "path",
30
+ "num_threads" => "flush_thread_count",
31
+ "flush_interval" => "flush_interval",
32
+ "try_flush_interval" => "flush_thread_interval",
33
+ "queued_chunk_flush_interval" => "flush_thread_burst_interval",
34
+ "disable_retry_limit" => "retry_forever",
35
+ "retry_limit" => "retry_max_times",
36
+ "max_retry_wait" => "retry_max_interval",
37
+ "buffer_chunk_limit" => "chunk_limit_size",
38
+ "buffer_queue_limit" => "queue_length_limit",
39
+ "buffer_queue_full_action" => "overflow_action",
40
+ "flush_at_shutdown" => "flush_at_shutdown",
41
+ }
42
+
43
+ TIME_SLICED_PARAMS = {
44
+ "time_slice_format" => nil,
45
+ "time_slice_wait" => "timekey_wait",
46
+ }
47
+
48
+ def compat_parameters_default_chunk_key
49
+ # '', 'time' or 'tag'
50
+ raise NotImplementedError, "return one of '', 'time' or 'tag'"
51
+ end
52
+
53
+ def configure(conf)
54
+ if conf.elements('buffer').empty? && PARAMS_MAP.keys.any?{|k| conf.has_key?(k) } || TIME_SLICED_PARAMS.keys.any?{|k| conf.has_key?(k) }
55
+ # TODO: warn obsolete parameters if these are deprecated
56
+ attr = {}
57
+ PARAMS_MAP.each do |compat, current|
58
+ next unless current
59
+ if conf.has_key?(compat)
60
+ if compat == 'buffer_queue_full_action' && conf[compat] == 'exception'
61
+ attr[current] = 'throw_exception'
62
+ else
63
+ attr[current] = conf[compat]
64
+ end
65
+ end
66
+ end
67
+ TIME_SLICED_PARAMS.each do |compat, current|
68
+ next unless current
69
+ attr[current] = conf[compat] if conf.has_key?(compat)
70
+ end
71
+
72
+ chunk_key = nil
73
+
74
+ if conf.has_key?('time_slice_format')
75
+ chunk_key = 'time'
76
+ attr['timekey'] = case conf['time_slice_format']
77
+ when /\%S/ then 1
78
+ when /\%M/ then 60
79
+ when /\%H/ then 3600
80
+ when /\%d/ then 86400
81
+ else
82
+ raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
83
+ end
84
+ else
85
+ chunk_key = compat_parameters_default_chunk_key
86
+ if chunk_key == 'time'
87
+ attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
88
+ end
89
+ end
90
+
91
+ e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
92
+ conf.elements << e
93
+ end
94
+
95
+ super
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,80 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/time'
18
+
19
+ module Fluent
20
+ module PluginHelper
21
+ module EventEmitter
22
+ # stop : [-]
23
+ # shutdown : disable @router
24
+ # close : [-]
25
+ # terminate: [-]
26
+
27
+ def router
28
+ @_event_emitter_used_actually = true
29
+ @router
30
+ end
31
+
32
+ def router=(r)
33
+ @router = r
34
+ end
35
+
36
+ def has_router?
37
+ true
38
+ end
39
+
40
+ def event_emitter_used_actually?
41
+ @_event_emitter_used_actually
42
+ end
43
+
44
+ def event_emitter_router(label_name)
45
+ if label_name
46
+ Engine.root_agent.find_label(label_name).event_router
47
+ else
48
+ Engine.root_agent.event_router
49
+ end
50
+ end
51
+
52
+ def initialize
53
+ super
54
+ @_event_emitter_used_actually = false
55
+ @router = nil
56
+ end
57
+
58
+ def configure(conf)
59
+ require 'fluent/engine'
60
+ super
61
+ @router = event_emitter_router(conf['@label'])
62
+ end
63
+
64
+ def after_shutdown
65
+ @router = nil
66
+ super
67
+ end
68
+
69
+ def close # unset router many times to reduce test cost
70
+ @router = nil
71
+ super
72
+ end
73
+
74
+ def terminate
75
+ @router = nil
76
+ super
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,118 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'fluent/plugin_helper/thread'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module EventLoop
23
+ # Currently this plugin helper is only for other helpers, not plugins.
24
+ # there's no way to create customized watchers to attach event loops.
25
+ include Fluent::PluginHelper::Thread
26
+
27
+ # stop : [-]
28
+ # shutdown : detach all event watchers on event loop
29
+ # close : stop event loop
30
+ # terminate: initialize internal state
31
+
32
+ EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
33
+
34
+ attr_reader :_event_loop # for tests
35
+
36
+ def event_loop_attach(watcher)
37
+ @_event_loop_mutex.synchronize do
38
+ @_event_loop.attach(watcher)
39
+ end
40
+ end
41
+
42
+ def event_loop_wait_until_start
43
+ sleep(0.1) until event_loop_running?
44
+ end
45
+
46
+ def event_loop_wait_until_stop
47
+ sleep(0.1) while event_loop_running?
48
+ end
49
+
50
+ def event_loop_running?
51
+ @_event_loop_running
52
+ end
53
+
54
+ def initialize
55
+ super
56
+ @_event_loop = Coolio::Loop.new
57
+ @_event_loop_running = false
58
+ @_event_loop_mutex = Mutex.new
59
+ # plugin MAY configure loop run timeout in #configure
60
+ @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
61
+ end
62
+
63
+ def start
64
+ super
65
+
66
+ # event loop does not run here, so mutex lock is not required
67
+ thread_create :event_loop do
68
+ default_watcher = DefaultWatcher.new
69
+ @_event_loop.attach(default_watcher)
70
+ @_event_loop_running = true
71
+ @_event_loop.run(@_event_loop_run_timeout) # this method blocks
72
+ @_event_loop_running = false
73
+ end
74
+ end
75
+
76
+ def shutdown
77
+ @_event_loop_mutex.synchronize do
78
+ @_event_loop.watchers.each {|w| w.detach if w.attached? }
79
+ end
80
+ while @_event_loop_running
81
+ sleep 0.1
82
+ end
83
+
84
+ super
85
+ end
86
+
87
+ def close
88
+ if @_event_loop_running
89
+ begin
90
+ @_event_loop.stop # we cannot check loop is running or not
91
+ rescue RuntimeError => e
92
+ raise unless e.message == 'loop not running'
93
+ end
94
+ end
95
+
96
+ super
97
+ end
98
+
99
+ def terminate
100
+ @_event_loop = nil
101
+ @_event_loop_running = false
102
+ @_event_loop_mutex = nil
103
+ @_event_loop_run_timeout = nil
104
+
105
+ super
106
+ end
107
+
108
+ # watcher to block to run event loop until shutdown
109
+ class DefaultWatcher < Coolio::TimerWatcher
110
+ def initialize
111
+ super(1, true) # interval: 1, repeat: true
112
+ end
113
+ # do nothing
114
+ def on_timer; end
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,177 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module PluginHelper
19
+ module RetryState
20
+ def retry_state_create(
21
+ title, retry_type, wait, timeout,
22
+ forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,
23
+ secondary: false, secondary_threshold: 0.8
24
+ )
25
+ case retry_type
26
+ when :exponential_backoff
27
+ ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
28
+ when :periodic
29
+ PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
30
+ else
31
+ raise "BUG: unknown retry_type specified: '#{retry_type}'"
32
+ end
33
+ end
34
+
35
+ class RetryStateMachine
36
+ attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps
37
+
38
+ def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
39
+ @title = title
40
+
41
+ @start = current_time
42
+ @steps = 0
43
+ @next_time = nil # should be initialized for first retry by child class
44
+
45
+ @timeout = timeout
46
+ @timeout_at = @start + timeout
47
+ @current = :primary
48
+
49
+ if randomize_width < 0 || randomize_width > 0.5
50
+ raise "BUG: randomize_width MUST be between 0 and 0.5"
51
+ end
52
+
53
+ @randomize = randomize
54
+ @randomize_width = randomize_width
55
+
56
+ if forever && secondary
57
+ raise "BUG: forever and secondary are exclusive to each other"
58
+ end
59
+
60
+ @forever = forever
61
+ @max_steps = max_steps
62
+
63
+ @secondary = secondary
64
+ @secondary_threshold = secondary_threshold
65
+ if @secondary
66
+ raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
67
+ @secondary_transition_at = @start + timeout * @secondary_threshold
68
+ @secondary_transition_steps = nil
69
+ end
70
+ end
71
+
72
+ def current_time
73
+ Time.now
74
+ end
75
+
76
+ def randomize(interval)
77
+ return interval unless @randomize
78
+
79
+ interval + (interval * @randomize_width * (2 * rand - 1.0))
80
+ end
81
+
82
+ def calc_next_time
83
+ if @forever || !@secondary # primary
84
+ naive = naive_next_time(@steps)
85
+ if @forever
86
+ naive
87
+ elsif naive >= @timeout_at
88
+ @timeout_at
89
+ else
90
+ naive
91
+ end
92
+ elsif @current == :primary && @secondary
93
+ naive = naive_next_time(@steps)
94
+ if naive >= @secondary_transition_at
95
+ @secondary_transition_at
96
+ else
97
+ naive
98
+ end
99
+ elsif @current == :secondary
100
+ naive = naive_next_time(@steps - @secondary_transition_steps + 1)
101
+ if naive >= @timeout_at
102
+ @timeout_at
103
+ else
104
+ naive
105
+ end
106
+ else
107
+ raise "BUG: it's out of design"
108
+ end
109
+ end
110
+
111
+ def naive_next_time(retry_times)
112
+ raise NotImplementedError
113
+ end
114
+
115
+ def secondary?
116
+ @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
117
+ end
118
+
119
+ def step
120
+ @steps += 1
121
+ if @secondary && @current != :secondary && current_time >= @secondary_transition_at
122
+ @current = :secondary
123
+ @secondary_transition_steps = @steps
124
+ end
125
+ @next_time = calc_next_time
126
+ nil
127
+ end
128
+
129
+ def limit?
130
+ if @forever
131
+ false
132
+ else
133
+ @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
134
+ end
135
+ end
136
+ end
137
+
138
+ class ExponentialBackOffRetry < RetryStateMachine
139
+ def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold)
140
+ super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
141
+ @constant_factor = wait
142
+ @backoff_base = backoff_base
143
+ @max_interval = max_interval
144
+
145
+ @next_time = @start + @constant_factor
146
+ end
147
+
148
+ def naive_next_time(retry_next_times)
149
+ # make it infinite if calculated "interval" is too big
150
+ interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
151
+ intr = if interval.finite?
152
+ if @max_interval && interval > @max_interval
153
+ @max_interval
154
+ else
155
+ interval
156
+ end
157
+ else
158
+ interval
159
+ end
160
+ current_time + randomize(intr)
161
+ end
162
+ end
163
+
164
+ class PeriodicRetry < RetryStateMachine
165
+ def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
166
+ super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
167
+ @retry_wait = wait
168
+ @next_time = @start + @retry_wait
169
+ end
170
+
171
+ def naive_next_time(retry_next_times)
172
+ current_time + randomize(@retry_wait)
173
+ end
174
+ end
175
+ end
176
+ end
177
+ end