fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,331 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'forwardable'
18
+
19
+ require 'fluent/plugin'
20
+ require 'fluent/plugin/storage'
21
+ require 'fluent/plugin_helper/timer'
22
+ require 'fluent/config/element'
23
+ require 'fluent/configurable'
24
+
25
+ module Fluent
26
+ module PluginHelper
27
+ module Storage
28
+ include Fluent::PluginHelper::Timer
29
+
30
+ StorageState = Struct.new(:storage, :running)
31
+
32
+ def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
33
+ s = @_storages[usage]
34
+ if s && s.running
35
+ return s.storage
36
+ elsif s
37
+ # storage is already created, but not loaded / started
38
+ else # !s
39
+ type = if type
40
+ type
41
+ elsif conf && conf.respond_to?(:[])
42
+ raise Fluent::ConfigError, "@type is required in <storage>" unless conf['@type']
43
+ conf['@type']
44
+ elsif default_type
45
+ default_type
46
+ else
47
+ raise ArgumentError, "BUG: both type and conf are not specified"
48
+ end
49
+ storage = Plugin.new_storage(type, parent: self)
50
+ config = case conf
51
+ when Fluent::Config::Element
52
+ conf
53
+ when Hash
54
+ # in code, programmer may use symbols as keys, but Element needs strings
55
+ conf = Hash[conf.map{|k,v| [k.to_s, v]}]
56
+ Fluent::Config::Element.new('storage', usage, conf, [])
57
+ when nil
58
+ Fluent::Config::Element.new('storage', usage, {}, [])
59
+ else
60
+ raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
61
+ end
62
+ storage.configure(config)
63
+ if @_storages_started
64
+ storage.start
65
+ end
66
+ s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)
67
+ end
68
+
69
+ s.storage
70
+ end
71
+
72
+ module StorageParams
73
+ include Fluent::Configurable
74
+ # minimum section definition to instantiate storage plugin instances
75
+ config_section :storage, required: false, multi: true, param_name: :storage_configs do
76
+ config_argument :usage, :string, default: ''
77
+ config_param :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
78
+ end
79
+ end
80
+
81
+ def self.included(mod)
82
+ mod.include StorageParams
83
+ end
84
+
85
+ attr_reader :_storages # for tests
86
+
87
+ def initialize
88
+ super
89
+ @_storages_started = false
90
+ @_storages = {} # usage => storage_state
91
+ end
92
+
93
+ def configure(conf)
94
+ super
95
+
96
+ @storage_configs.each do |section|
97
+ if @_storages[section.usage]
98
+ raise Fluent::ConfigError, "duplicated storages configured: #{section.usage}"
99
+ end
100
+ storage = Plugin.new_storage(section[:@type], parent: self)
101
+ storage.configure(section.corresponding_config_element)
102
+ @_storages[section.usage] = StorageState.new(wrap_instance(storage), false)
103
+ end
104
+ end
105
+
106
+ def start
107
+ super
108
+
109
+ @_storages_started = true
110
+ @_storages.each_pair do |usage, s|
111
+ s.storage.start
112
+ s.storage.load
113
+
114
+ if s.storage.autosave && !s.storage.persistent
115
+ timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
116
+ begin
117
+ s.storage.save
118
+ rescue => e
119
+ log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
120
+ end
121
+ end
122
+ end
123
+ s.running = true
124
+ end
125
+ end
126
+
127
+ def storage_operate(method_name, &block)
128
+ @_storages.each_pair do |usage, s|
129
+ begin
130
+ block.call(s) if block_given?
131
+ s.storage.send(method_name)
132
+ rescue => e
133
+ log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
134
+ end
135
+ end
136
+ end
137
+
138
+ def stop
139
+ super
140
+ # timer stops automatically in super
141
+ storage_operate(:stop)
142
+ end
143
+
144
+ def before_shutdown
145
+ storage_operate(:before_shutdown)
146
+ super
147
+ end
148
+
149
+ def shutdown
150
+ storage_operate(:shutdown) do |s|
151
+ s.storage.save if s.storage.save_at_shutdown
152
+ end
153
+ super
154
+ end
155
+
156
+ def after_shutdown
157
+ storage_operate(:after_shutdown)
158
+ super
159
+ end
160
+
161
+ def close
162
+ storage_operate(:close){|s| s.running = false }
163
+ super
164
+ end
165
+
166
+ def terminate
167
+ storage_operate(:terminate)
168
+ @_storages = {}
169
+ super
170
+ end
171
+
172
+ def wrap_instance(storage)
173
+ if storage.persistent && storage.persistent_always?
174
+ storage
175
+ elsif storage.persistent
176
+ PersistentWrapper.new(storage)
177
+ elsif !storage.synchronized?
178
+ SynchronizeWrapper.new(storage)
179
+ else
180
+ storage
181
+ end
182
+ end
183
+
184
+ class PersistentWrapper
185
+ # PersistentWrapper always provides synchronized operations
186
+ extend Forwardable
187
+
188
+ def initialize(storage)
189
+ @storage = storage
190
+ @monitor = Monitor.new
191
+ end
192
+
193
+ def_delegators :@storage, :autosave_interval, :save_at_shutdown
194
+ def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
195
+ def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
196
+
197
+ def persistent_always?
198
+ true
199
+ end
200
+
201
+ def persistent
202
+ true
203
+ end
204
+
205
+ def autosave
206
+ false
207
+ end
208
+
209
+ def synchronized?
210
+ true
211
+ end
212
+
213
+ def implementation
214
+ @storage
215
+ end
216
+
217
+ def load
218
+ @monitor.synchronize do
219
+ @storage.load
220
+ end
221
+ end
222
+
223
+ def save
224
+ @monitor.synchronize do
225
+ @storage.save
226
+ end
227
+ end
228
+
229
+ def get(key)
230
+ @monitor.synchronize do
231
+ @storage.load
232
+ @storage.get(key)
233
+ end
234
+ end
235
+
236
+ def fetch(key, defval)
237
+ @monitor.synchronize do
238
+ @storage.load
239
+ @storage.fetch(key, defval)
240
+ end
241
+ end
242
+
243
+ def put(key, value)
244
+ @monitor.synchronize do
245
+ @storage.load
246
+ @storage.put(key, value)
247
+ @storage.save
248
+ value
249
+ end
250
+ end
251
+
252
+ def delete(key)
253
+ @monitor.synchronize do
254
+ @storage.load
255
+ val = @storage.delete(key)
256
+ @storage.save
257
+ val
258
+ end
259
+ end
260
+
261
+ def update(key, &block)
262
+ @monitor.synchronize do
263
+ @storage.load
264
+ v = block.call(@storage.get(key))
265
+ @storage.put(key, v)
266
+ @storage.save
267
+ v
268
+ end
269
+ end
270
+ end
271
+
272
+ class SynchronizeWrapper
273
+ extend Forwardable
274
+
275
+ def initialize(storage)
276
+ @storage = storage
277
+ @mutex = Mutex.new
278
+ end
279
+
280
+ def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
281
+ def_delegators :@storage, :persistent_always?
282
+ def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
283
+ def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
284
+
285
+ def synchronized?
286
+ true
287
+ end
288
+
289
+ def implementation
290
+ @storage
291
+ end
292
+
293
+ def load
294
+ @mutex.synchronize do
295
+ @storage.load
296
+ end
297
+ end
298
+
299
+ def save
300
+ @mutex.synchronize do
301
+ @storage.save
302
+ end
303
+ end
304
+
305
+ def get(key)
306
+ @mutex.synchronize{ @storage.get(key) }
307
+ end
308
+
309
+ def fetch(key, defval)
310
+ @mutex.synchronize{ @storage.fetch(key, defval) }
311
+ end
312
+
313
+ def put(key, value)
314
+ @mutex.synchronize{ @storage.put(key, value) }
315
+ end
316
+
317
+ def delete(key)
318
+ @mutex.synchronize{ @storage.delete(key) }
319
+ end
320
+
321
+ def update(key, &block)
322
+ @mutex.synchronize do
323
+ v = block.call(@storage.get(key))
324
+ @storage.put(key, v)
325
+ v
326
+ end
327
+ end
328
+ end
329
+ end
330
+ end
331
+ end
@@ -0,0 +1,147 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module PluginHelper
19
+ module Thread
20
+ THREAD_DEFAULT_WAIT_SECONDS = 1
21
+
22
+ # stop : mark callback thread as stopped
23
+ # shutdown : [-]
24
+ # close : correct stopped threads
25
+ # terminate: kill all threads
26
+
27
+ attr_reader :_threads # for test driver
28
+
29
+ def thread_current_running?
30
+ # checker for code in callback of thread_create
31
+ ::Thread.current[:_fluentd_plugin_helper_thread_running] || false
32
+ end
33
+
34
+ def thread_wait_until_start
35
+ until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
36
+ sleep 0.1
37
+ end
38
+ end
39
+
40
+ def thread_wait_until_stop
41
+ until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && ![:_fluentd_plugin_helper_thread_running] } }
42
+ sleep 0.1
43
+ end
44
+ end
45
+
46
+ # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
47
+ # by passing splatted argument to `yield`
48
+ # https://bugs.ruby-lang.org/issues/11027
49
+ # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
50
+ # def thread_create(title, *args)
51
+ # Thread.new(*args) do |*t_args|
52
+ # yield *t_args
53
+ def thread_create(title)
54
+ raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
55
+ raise ArgumentError, "BUG: callback not specified" unless block_given?
56
+ m = Mutex.new
57
+ m.lock
58
+ thread = ::Thread.new do
59
+ m.lock # run thread after that thread is successfully set into @_threads
60
+ m.unlock
61
+ thread_exit = false
62
+ ::Thread.current[:_fluentd_plugin_helper_thread_title] = title
63
+ ::Thread.current[:_fluentd_plugin_helper_thread_started] = true
64
+ ::Thread.current[:_fluentd_plugin_helper_thread_running] = true
65
+ begin
66
+ yield
67
+ thread_exit = true
68
+ rescue Exception => e
69
+ log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
70
+ thread_exit = true
71
+ raise
72
+ ensure
73
+ unless thread_exit
74
+ log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
75
+ end
76
+ @_threads_mutex.synchronize do
77
+ @_threads.delete(::Thread.current.object_id)
78
+ end
79
+ ::Thread.current[:_fluentd_plugin_helper_thread_running] = false
80
+ end
81
+ end
82
+ thread.abort_on_exception = true
83
+ @_threads_mutex.synchronize do
84
+ @_threads[thread.object_id] = thread
85
+ end
86
+ m.unlock
87
+ thread
88
+ end
89
+
90
+ def thread_exist?(title)
91
+ @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
92
+ end
93
+
94
+ def thread_started?(title)
95
+ t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
96
+ t && t[:_fluentd_plugin_helper_thread_started]
97
+ end
98
+
99
+ def thread_running?(title)
100
+ t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
101
+ t && t[:_fluentd_plugin_helper_thread_running]
102
+ end
103
+
104
+ def initialize
105
+ super
106
+ @_threads_mutex = Mutex.new
107
+ @_threads = {}
108
+ @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
109
+ end
110
+
111
+ def stop
112
+ super
113
+ @_threads_mutex.synchronize do
114
+ @_threads.each_pair do |obj_id, thread|
115
+ thread[:_fluentd_plugin_helper_thread_running] = false
116
+ end
117
+ end
118
+ end
119
+
120
+ def close
121
+ @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
122
+ thread = @_threads[obj_id]
123
+ if !thread || thread.join(@_thread_wait_seconds)
124
+ @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
125
+ end
126
+ end
127
+
128
+ super
129
+ end
130
+
131
+ def terminate
132
+ super
133
+ @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
134
+ thread = @_threads[obj_id]
135
+ log.warn "killing existing thead", thread: thread
136
+ thread.kill if thread
137
+ end
138
+ @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
139
+ thread = @_threads[obj_id]
140
+ thread.join
141
+ @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
142
+ end
143
+ @_thread_wait_seconds = nil
144
+ end
145
+ end
146
+ end
147
+ end