fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,132 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/storage'
19
+
20
+ require 'fileutils'
21
+ require 'yajl'
22
+
23
+ module Fluent
24
+ module Plugin
25
+ class LocalStorage < Storage
26
+ Fluent::Plugin.register_storage('local', self)
27
+
28
+ DEFAULT_DIR_MODE = 0755
29
+ DEFAULT_FILE_MODE = 0644
30
+
31
+ config_param :path, :string, default: nil
32
+ config_param :mode, :integer, default: DEFAULT_FILE_MODE
33
+ config_param :dir_mode, :integer, default: DEFAULT_DIR_MODE
34
+ config_param :pretty_print, :bool, default: false
35
+
36
+ def initialize
37
+ super
38
+ @store = {}
39
+ end
40
+
41
+ def configure(conf)
42
+ super
43
+
44
+ @on_memory = false
45
+ if !@path && !@_plugin_id_configured
46
+ if @persistent
47
+ raise Fluent::ConfigError, "Plugin @id or path for <storage> required to save data"
48
+ else
49
+ if @autosave
50
+ log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
51
+ else
52
+ log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
53
+ end
54
+ @on_memory = true
55
+ end
56
+ elsif @path
57
+ # ok
58
+ else # @_plugin_id_configured is true
59
+ raise NotImplementedError, "implement this feature later with system_config"
60
+ ## TODO: get process-wide directory for plugin storage, and generate path for this plugin storage instance
61
+ # path =
62
+ end
63
+
64
+ if !@on_memory
65
+ dir = File.dirname(@path)
66
+ FileUtils.mkdir_p(dir, mode: @dir_mode) unless File.exist?(dir)
67
+ if File.exist?(@path)
68
+ raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
69
+ begin
70
+ data = Yajl::Parser.parse(open(@path, 'r:utf-8'){ |io| io.read })
71
+ raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
72
+ rescue => e
73
+ log.error "failed to read data from plugin storage file", path: @path, error: e
74
+ raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
75
+ end
76
+ else
77
+ raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{dir}'" unless File.writable?(dir)
78
+ end
79
+ end
80
+ end
81
+
82
+ def load
83
+ return if @on_memory
84
+ return unless File.exist?(@path)
85
+ begin
86
+ json_string = open(@path, 'r:utf-8'){ |io| io.read }
87
+ json = Yajl::Parser.parse(json_string)
88
+ unless json.is_a?(Hash)
89
+ log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
90
+ log.debug "broken content", content: json_string
91
+ return
92
+ end
93
+ @store = json
94
+ rescue => e
95
+ log.error "failed to load data for plugin storage from file", path: @path, error: e
96
+ end
97
+ end
98
+
99
+ def save
100
+ return if @on_memory
101
+ tmp_path = @path + '.tmp'
102
+ begin
103
+ json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
104
+ open(tmp_path, 'w:utf-8', @mode){ |io| io.write json_string }
105
+ File.rename(tmp_path, @path)
106
+ rescue => e
107
+ log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
108
+ end
109
+ end
110
+
111
+ def get(key)
112
+ @store[key.to_s]
113
+ end
114
+
115
+ def fetch(key, defval)
116
+ @store.fetch(key.to_s, defval)
117
+ end
118
+
119
+ def put(key, value)
120
+ @store[key.to_s] = value
121
+ end
122
+
123
+ def delete(key)
124
+ @store.delete(key.to_s)
125
+ end
126
+
127
+ def update(key, &block)
128
+ @store[key.to_s] = block.call(@store[key.to_s])
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,22 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/compat/string_util'
18
+
19
+ module Fluent
20
+ # obsolete
21
+ StringUtil = Fluent::Compat::StringUtil
22
+ end
@@ -0,0 +1,42 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/event_emitter'
18
+ require 'fluent/plugin_helper/thread'
19
+ require 'fluent/plugin_helper/event_loop'
20
+ require 'fluent/plugin_helper/timer'
21
+ require 'fluent/plugin_helper/child_process'
22
+ require 'fluent/plugin_helper/storage'
23
+ require 'fluent/plugin_helper/parser'
24
+ require 'fluent/plugin_helper/formatter'
25
+ require 'fluent/plugin_helper/inject'
26
+ require 'fluent/plugin_helper/retry_state'
27
+ require 'fluent/plugin_helper/compat_parameters'
28
+
29
+ module Fluent
30
+ module PluginHelper
31
+ module Mixin
32
+ def self.included(mod)
33
+ mod.extend(Fluent::PluginHelper)
34
+ end
35
+ end
36
+
37
+ def helpers(*snake_case_symbols)
38
+ helper_modules = snake_case_symbols.map{|name| Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join) }
39
+ include(*helper_modules)
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,298 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/thread'
18
+ require 'fluent/plugin_helper/timer'
19
+
20
+ require 'open3'
21
+ require 'timeout'
22
+
23
+ module Fluent
24
+ module PluginHelper
25
+ module ChildProcess
26
+ include Fluent::PluginHelper::Thread
27
+ include Fluent::PluginHelper::Timer
28
+
29
+ CHILD_PROCESS_LOOP_CHECK_INTERVAL = 0.2 # sec
30
+ CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT = 10 # sec
31
+ CHILD_PROCESS_DEFAULT_KILL_TIMEOUT = 60 # sec
32
+
33
+ MODE_PARAMS = [:read, :write, :stderr, :read_with_stderr]
34
+ STDERR_OPTIONS = [:discard, :connect]
35
+
36
+ # stop : mark callback thread as stopped
37
+ # shutdown : close write IO to child processes (STDIN of child processes), send TERM (KILL for Windows) to all child processes
38
+ # close : send KILL to all child processes
39
+ # terminate: [-]
40
+
41
+ attr_reader :_child_process_processes # for tests
42
+
43
+ def child_process_running?
44
+ # checker for code in callback of child_process_execute
45
+ ::Thread.current[:_fluentd_plugin_helper_child_process_running] || false
46
+ end
47
+
48
+ def child_process_id
49
+ ::Thread.current[:_fluentd_plugin_helper_child_process_pid]
50
+ end
51
+
52
+ def child_process_exit_status
53
+ ::Thread.current[:_fluentd_plugin_helper_child_process_exit_status]
54
+ end
55
+
56
+ def child_process_execute(
57
+ title, command,
58
+ arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false,
59
+ mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,
60
+ internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,
61
+ &block
62
+ )
63
+ raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
64
+ raise ArgumentError, "BUG: arguments required if subprocess name is replaced" if subprocess_name && !arguments
65
+
66
+ raise ArgumentError, "BUG: invalid mode specification" unless mode.all?{|m| MODE_PARAMS.include?(m) }
67
+ raise ArgumentError, "BUG: read_with_stderr is exclusive with :read and :stderr" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))
68
+ raise ArgumentError, "BUG: invalid stderr handling specification" unless STDERR_OPTIONS.include?(stderr)
69
+
70
+ raise ArgumentError, "BUG: block not specified which receive i/o object" unless block_given?
71
+ raise ArgumentError, "BUG: number of block arguments are different from size of mode" unless block.arity == mode.size
72
+
73
+ running = false
74
+ callback = ->(*args) {
75
+ running = true
76
+ begin
77
+ block.call(*args)
78
+ ensure
79
+ running = false
80
+ end
81
+ }
82
+
83
+ if immediate || !interval
84
+ child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
85
+ end
86
+
87
+ if interval
88
+ timer_execute(:child_process_execute, interval, repeat: true) do
89
+ if !parallel && running
90
+ log.warn "previous child process is still running. skipped.", title: title, command: command, arguments: arguments, interval: interval, parallel: parallel
91
+ else
92
+ child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
93
+ end
94
+ end
95
+ end
96
+ end
97
+
98
+ def initialize
99
+ super
100
+ # plugins MAY configure this parameter
101
+ @_child_process_exit_timeout = CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT
102
+ @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT
103
+ @_child_process_mutex = Mutex.new
104
+ end
105
+
106
+ def start
107
+ super
108
+ @_child_process_processes = {} # pid => ProcessInfo
109
+ end
110
+
111
+ def stop
112
+ @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
113
+ process_info = @_child_process_processes[pid]
114
+ if process_info
115
+ process_info.thread[:_fluentd_plugin_helper_child_process_running] = false
116
+ end
117
+ end
118
+ end
119
+
120
+ def shutdown
121
+ @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
122
+ process_info = @_child_process_processes[pid]
123
+ next if !process_info || !process_info.writeio_in_use
124
+ begin
125
+ Timeout.timeout(@_child_process_exit_timeout) do
126
+ process_info.writeio.close
127
+ end
128
+ rescue Timeout::Error
129
+ log.debug "External process #{process_info.title} doesn't exist after STDIN close in timeout #{@_child_process_exit_timeout}sec"
130
+ end
131
+
132
+ child_process_kill(process_info)
133
+ end
134
+
135
+ super
136
+ end
137
+
138
+ def close
139
+ while (pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }).size > 0
140
+ pids.each do |pid|
141
+ process_info = @_child_process_processes[pid]
142
+ if !process_info || !process_info.alive
143
+ @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
144
+ next
145
+ end
146
+
147
+ process_info.killed_at ||= Time.now # for illegular case (e.g., created after shutdown)
148
+ next if Time.now < process_info.killed_at + @_child_process_kill_timeout
149
+
150
+ child_process_kill(process_info, force: true)
151
+ @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
152
+ end
153
+
154
+ sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
155
+ end
156
+
157
+ super
158
+ end
159
+
160
+ def child_process_kill(process_info, force: false)
161
+ if !process_info || !process_info.alive
162
+ return
163
+ end
164
+
165
+ process_info.killed_at = Time.now unless force
166
+
167
+ begin
168
+ pid, status = Process.waitpid2(process_info.pid, Process::WNOHANG)
169
+ if pid && status
170
+ process_info.thread[:_fluentd_plugin_helper_child_process_exit_status] = status
171
+ process_info.alive = false
172
+ end
173
+ rescue Errno::ECHILD, Errno::ESRCH, Errno::EPERM
174
+ process_info.alive = false
175
+ rescue
176
+ # ignore
177
+ end
178
+ if !process_info.alive
179
+ return
180
+ end
181
+
182
+ begin
183
+ signal = (Fluent.windows? || force) ? :KILL : :TERM
184
+ Process.kill(signal, process_info.pid)
185
+ if force
186
+ process_info.alive = false
187
+ end
188
+ rescue Errno::ECHILD, Errno::ESRCH
189
+ process_info.alive = false
190
+ end
191
+ end
192
+
193
+ ProcessInfo = Struct.new(:title, :thread, :pid, :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use, :wait_thread, :alive, :killed_at)
194
+
195
+ def child_process_execute_once(
196
+ title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir,
197
+ internal_encoding, external_encoding, scrub, replace_string, &block
198
+ )
199
+ spawn_args = if arguments || subprocess_name
200
+ [ env, (subprocess_name ? [command, subprocess_name] : command), *(arguments || []) ]
201
+ else
202
+ [ env, command ]
203
+ end
204
+ spawn_opts = {
205
+ unsetenv_others: unsetenv,
206
+ }
207
+ if chdir
208
+ spawn_opts[:chdir] = chdir
209
+ end
210
+
211
+ encoding_options = {}
212
+ if scrub
213
+ encoding_options[:invalid] = encoding_options[:undef] = :replace
214
+ if replace_string
215
+ encoding_options[:replace] = replace_string
216
+ end
217
+ end
218
+
219
+ log.debug "Executing command", title: title, spawn: spawn_args, mode: mode, stderr: stderr
220
+
221
+ readio = writeio = stderrio = wait_thread = nil
222
+ readio_in_use = writeio_in_use = stderrio_in_use = false
223
+
224
+ if !mode.include?(:stderr) && !mode.include?(:read_with_stderr) && stderr != :discard # connect
225
+ writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
226
+ elsif mode.include?(:read_with_stderr)
227
+ writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
228
+ else
229
+ writeio, readio, stderrio, wait_thread = *Open3.popen3(*spawn_args, spawn_opts)
230
+ if !mode.include?(:stderr) # stderr == :discard
231
+ stderrio.reopen(IO::NULL)
232
+ end
233
+ end
234
+
235
+ if mode.include?(:write)
236
+ writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
237
+ writeio_in_use = true
238
+ end
239
+ if mode.include?(:read) || mode.include?(:read_with_stderr)
240
+ readio.set_encoding(external_encoding, internal_encoding, encoding_options)
241
+ readio_in_use = true
242
+ end
243
+ if mode.include?(:stderr)
244
+ stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
245
+ stderrio_in_use = true
246
+ end
247
+
248
+ pid = wait_thread.pid # wait_thread => Process::Waiter
249
+
250
+ io_objects = []
251
+ mode.each do |m|
252
+ io_objects << case m
253
+ when :read then readio
254
+ when :write then writeio
255
+ when :read_with_stderr then readio
256
+ when :stderr then stderrio
257
+ else
258
+ raise "BUG: invalid mode must be checked before here: '#{m}'"
259
+ end
260
+ end
261
+
262
+ m = Mutex.new
263
+ m.lock
264
+ thread = thread_create :child_process_callback do
265
+ m.lock # run after plugin thread get pid, thread instance and i/o
266
+ m.unlock
267
+ begin
268
+ block.call(*io_objects)
269
+ rescue EOFError => e
270
+ log.debug "Process exit and I/O closed", title: title, pid: pid, command: command, arguments: arguments
271
+ rescue IOError => e
272
+ if e.message == 'stream closed'
273
+ log.debug "Process I/O stream closed", title: title, pid: pid, command: command, arguments: arguments
274
+ else
275
+ log.error "Unexpected I/O error for child process", title: title, pid: pid, command: command, arguments: arguments, error: e
276
+ end
277
+ rescue => e
278
+ log.warn "Unexpected error while processing I/O for child process", title: title, pid: pid, command: command, error: e
279
+ end
280
+ process_info = @_child_process_mutex.synchronize do
281
+ process_info = @_child_process_processes[pid]
282
+ @_child_process_processes.delete(pid)
283
+ process_info
284
+ end
285
+ child_process_kill(process_info, force: true) if process_info && process_info.alive && ::Thread.current[:_fluentd_plugin_helper_child_process_running]
286
+ end
287
+ thread[:_fluentd_plugin_helper_child_process_running] = true
288
+ thread[:_fluentd_plugin_helper_child_process_pid] = pid
289
+ pinfo = ProcessInfo.new(title, thread, pid, readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use, wait_thread, true, nil)
290
+ @_child_process_mutex.synchronize do
291
+ @_child_process_processes[pid] = pinfo
292
+ end
293
+ m.unlock
294
+ pid
295
+ end
296
+ end
297
+ end
298
+ end