fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,224 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/types'
18
+ require 'fluent/config/element'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module CompatParameters
23
+ # This plugin helper is to bring old-fashioned buffer/other
24
+ # configuration parameters to v0.14 plugin API configurations.
25
+ # This helper is mainly to convert plugins from v0.12 API
26
+ # to v0.14 API safely, without breaking user deployment.
27
+
28
+ BUFFER_PARAMS = {
29
+ "buffer_type" => "@type",
30
+ "buffer_path" => "path",
31
+ "num_threads" => "flush_thread_count",
32
+ "flush_interval" => "flush_interval",
33
+ "try_flush_interval" => "flush_thread_interval",
34
+ "queued_chunk_flush_interval" => "flush_thread_burst_interval",
35
+ "disable_retry_limit" => "retry_forever",
36
+ "retry_limit" => "retry_max_times",
37
+ "max_retry_wait" => "retry_max_interval",
38
+ "buffer_chunk_limit" => "chunk_limit_size",
39
+ "buffer_queue_limit" => "queue_length_limit",
40
+ "buffer_queue_full_action" => "overflow_action",
41
+ "flush_at_shutdown" => "flush_at_shutdown",
42
+ }
43
+
44
+ BUFFER_TIME_SLICED_PARAMS = {
45
+ "time_slice_format" => nil,
46
+ "time_slice_wait" => "timekey_wait",
47
+ }
48
+
49
+ PARSER_PARAMS = {
50
+ "format" => "@type",
51
+ "time_key" => "time_key",
52
+ "time_format" => "time_format",
53
+ "delimiter" => "delimiter",
54
+ "json_parser" => "json_parser", # JSONParser
55
+ "label_delimiter" => "label_delimiter", # LabeledTSVParser
56
+ "format_firstline" => "format_firstline", # MultilineParser
57
+ "message_key" => "message_key", # NoneParser
58
+ "with_priority" => "with_priority", # SyslogParser
59
+ }
60
+
61
+ INJECT_PARAMS = {
62
+ "include_time_key" => nil,
63
+ "time_key" => "time_key",
64
+ "time_format" => "time_format",
65
+ "timezone" => "timezone",
66
+ "include_tag_key" => nil,
67
+ "tag_key" => "tag_key",
68
+ "localtime" => nil,
69
+ "utc" => nil,
70
+ }
71
+
72
+ FORMATTER_PARAMS = {
73
+ "format" => "@type",
74
+ "delimiter" => "delimiter",
75
+ "force_quotes" => "force_quotes", # CsvFormatter
76
+ "fields" => "fields", # CsvFormatter
77
+ "json_parser" => "json_parser", # JSONFormatter
78
+ "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
79
+ "output_time" => "output_time", # OutFileFormatter
80
+ "output_tag" => "output_tag", # OutFileFormatter
81
+ "message_key" => "message_key", # SingleValueFormatter
82
+ "add_newline" => "add_newline", # SingleValueFormatter
83
+ "output_type" => "output_type", # StdoutFormatter
84
+ }
85
+
86
+ def compat_parameters_convert(conf, *types, **kwargs)
87
+ types.each do |type|
88
+ case type
89
+ when :buffer
90
+ compat_parameters_buffer(conf, **kwargs)
91
+ when :inject
92
+ compat_parameters_inject(conf)
93
+ when :parser
94
+ compat_parameters_parser(conf)
95
+ when :formatter
96
+ compat_parameters_formatter(conf)
97
+ else
98
+ raise "BUG: unknown compat_parameters type: #{type}"
99
+ end
100
+ end
101
+
102
+ conf
103
+ end
104
+
105
+ def compat_parameters_buffer(conf, default_chunk_key: '')
106
+ # return immediately if <buffer> section exists, or any buffer-related parameters don't exist
107
+ return unless conf.elements('buffer').empty?
108
+ return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }
109
+
110
+ # TODO: warn obsolete parameters if these are deprecated
111
+ buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
112
+ attr = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
113
+ if compat_key == 'buffer_queue_full_action' && value == 'exception'
114
+ 'throw_exception'
115
+ else
116
+ value
117
+ end
118
+ end
119
+
120
+ chunk_key = default_chunk_key
121
+
122
+ if conf.has_key?('time_slice_format')
123
+ chunk_key = 'time'
124
+ attr['timekey'] = case conf['time_slice_format']
125
+ when /\%S/ then 1
126
+ when /\%M/ then 60
127
+ when /\%H/ then 3600
128
+ when /\%d/ then 86400
129
+ else
130
+ raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
131
+ end
132
+ else
133
+ if chunk_key == 'time'
134
+ attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
135
+ end
136
+ end
137
+
138
+ e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
139
+ conf.elements << e
140
+
141
+ conf
142
+ end
143
+
144
+ def compat_parameters_inject(conf)
145
+ return unless conf.elements('inject').empty?
146
+ return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
147
+
148
+ # TODO: warn obsolete parameters if these are deprecated
149
+ attr = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
150
+
151
+ if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
152
+ attr['time_key'] ||= 'time'
153
+ attr['time_type'] ||= 'string'
154
+ end
155
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
156
+ attr['time_type'] = 'unixtime'
157
+ end
158
+ if conf.has_key?('localtime') || conf.has_key?('utc')
159
+ if conf.has_key?('localtime') && conf.has_key?('utc')
160
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
161
+ elsif conf.has_key?('localtime')
162
+ attr['localtime'] = Fluent::Config.bool_value(conf['localtime'])
163
+ elsif conf.has_key?('utc')
164
+ attr['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
165
+ # Specifying "localtime false" means using UTC in TimeFormatter
166
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
167
+ # There are difference between "Z" and "+0000" in timezone formatting.
168
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
169
+ end
170
+ end
171
+
172
+ if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
173
+ attr['tag_key'] ||= 'tag'
174
+ end
175
+
176
+ e = Fluent::Config::Element.new('inject', '', attr, [])
177
+ conf.elements << e
178
+
179
+ conf
180
+ end
181
+
182
+ def compat_parameters_parser(conf)
183
+ return unless conf.elements('parse').empty?
184
+ return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
185
+
186
+ # TODO: warn obsolete parameters if these are deprecated
187
+ attr = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
188
+
189
+ e = Fluent::Config::Element.new('parse', '', attr, [])
190
+ conf.elements << e
191
+
192
+ conf
193
+ end
194
+
195
+ def compat_parameters_formatter(conf)
196
+ return unless conf.elements('format').empty?
197
+ return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
198
+
199
+ # TODO: warn obsolete parameters if these are deprecated
200
+ attr = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
201
+
202
+ e = Fluent::Config::Element.new('format', '', attr, [])
203
+ conf.elements << e
204
+
205
+ conf
206
+ end
207
+
208
+ def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
209
+ attr = {}
210
+ params.each do |compat, current|
211
+ next unless current
212
+ if conf.has_key?(compat)
213
+ if block_given?
214
+ attr[current] = block.call(compat, conf[compat])
215
+ else
216
+ attr[current] = conf[compat]
217
+ end
218
+ end
219
+ end
220
+ attr
221
+ end
222
+ end
223
+ end
224
+ end
@@ -0,0 +1,80 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/time'
18
+
19
+ module Fluent
20
+ module PluginHelper
21
+ module EventEmitter
22
+ # stop : [-]
23
+ # shutdown : disable @router
24
+ # close : [-]
25
+ # terminate: [-]
26
+
27
+ def router
28
+ @_event_emitter_used_actually = true
29
+ @router
30
+ end
31
+
32
+ def router=(r)
33
+ @router = r
34
+ end
35
+
36
+ def has_router?
37
+ true
38
+ end
39
+
40
+ def event_emitter_used_actually?
41
+ @_event_emitter_used_actually
42
+ end
43
+
44
+ def event_emitter_router(label_name)
45
+ if label_name
46
+ Engine.root_agent.find_label(label_name).event_router
47
+ else
48
+ Engine.root_agent.event_router
49
+ end
50
+ end
51
+
52
+ def initialize
53
+ super
54
+ @_event_emitter_used_actually = false
55
+ @router = nil
56
+ end
57
+
58
+ def configure(conf)
59
+ require 'fluent/engine'
60
+ super
61
+ @router = event_emitter_router(conf['@label'])
62
+ end
63
+
64
+ def after_shutdown
65
+ @router = nil
66
+ super
67
+ end
68
+
69
+ def close # unset router many times to reduce test cost
70
+ @router = nil
71
+ super
72
+ end
73
+
74
+ def terminate
75
+ @router = nil
76
+ super
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,118 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'fluent/plugin_helper/thread'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module EventLoop
23
+ # Currently this plugin helper is only for other helpers, not plugins.
24
+ # there's no way to create customized watchers to attach event loops.
25
+ include Fluent::PluginHelper::Thread
26
+
27
+ # stop : [-]
28
+ # shutdown : detach all event watchers on event loop
29
+ # close : stop event loop
30
+ # terminate: initialize internal state
31
+
32
+ EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
33
+
34
+ attr_reader :_event_loop # for tests
35
+
36
+ def event_loop_attach(watcher)
37
+ @_event_loop_mutex.synchronize do
38
+ @_event_loop.attach(watcher)
39
+ end
40
+ end
41
+
42
+ def event_loop_wait_until_start
43
+ sleep(0.1) until event_loop_running?
44
+ end
45
+
46
+ def event_loop_wait_until_stop
47
+ sleep(0.1) while event_loop_running?
48
+ end
49
+
50
+ def event_loop_running?
51
+ @_event_loop_running
52
+ end
53
+
54
+ def initialize
55
+ super
56
+ @_event_loop = Coolio::Loop.new
57
+ @_event_loop_running = false
58
+ @_event_loop_mutex = Mutex.new
59
+ # plugin MAY configure loop run timeout in #configure
60
+ @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
61
+ end
62
+
63
+ def start
64
+ super
65
+
66
+ # event loop does not run here, so mutex lock is not required
67
+ thread_create :event_loop do
68
+ default_watcher = DefaultWatcher.new
69
+ @_event_loop.attach(default_watcher)
70
+ @_event_loop_running = true
71
+ @_event_loop.run(@_event_loop_run_timeout) # this method blocks
72
+ @_event_loop_running = false
73
+ end
74
+ end
75
+
76
+ def shutdown
77
+ @_event_loop_mutex.synchronize do
78
+ @_event_loop.watchers.each {|w| w.detach if w.attached? }
79
+ end
80
+ while @_event_loop_running
81
+ sleep 0.1
82
+ end
83
+
84
+ super
85
+ end
86
+
87
+ def close
88
+ if @_event_loop_running
89
+ begin
90
+ @_event_loop.stop # we cannot check loop is running or not
91
+ rescue RuntimeError => e
92
+ raise unless e.message == 'loop not running'
93
+ end
94
+ end
95
+
96
+ super
97
+ end
98
+
99
+ def terminate
100
+ @_event_loop = nil
101
+ @_event_loop_running = false
102
+ @_event_loop_mutex = nil
103
+ @_event_loop_run_timeout = nil
104
+
105
+ super
106
+ end
107
+
108
+ # watcher to block to run event loop until shutdown
109
+ class DefaultWatcher < Coolio::TimerWatcher
110
+ def initialize
111
+ super(1, true) # interval: 1, repeat: true
112
+ end
113
+ # do nothing
114
+ def on_timer; end
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,149 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/formatter'
19
+ require 'fluent/config/element'
20
+ require 'fluent/configurable'
21
+
22
+ module Fluent
23
+ module PluginHelper
24
+ module Formatter
25
+ def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
26
+ formatter = @_formatters[usage]
27
+ return formatter if formatter
28
+
29
+ type = if type
30
+ type
31
+ elsif conf && conf.respond_to?(:[])
32
+ raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
33
+ conf['@type']
34
+ elsif default_type
35
+ default_type
36
+ else
37
+ raise ArgumentError, "BUG: both type and conf are not specified"
38
+ end
39
+ formatter = Fluent::Plugin.new_formatter(type, parent: self)
40
+ config = case conf
41
+ when Fluent::Config::Element
42
+ conf
43
+ when Hash
44
+ # in code, programmer may use symbols as keys, but Element needs strings
45
+ conf = Hash[conf.map{|k,v| [k.to_s, v]}]
46
+ Fluent::Config::Element.new('format', usage, conf, [])
47
+ when nil
48
+ Fluent::Config::Element.new('format', usage, {}, [])
49
+ else
50
+ raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
51
+ end
52
+ formatter.configure(config)
53
+ if @_formatters_started
54
+ formatter.start
55
+ end
56
+
57
+ @_formatters[usage] = formatter
58
+ formatter
59
+ end
60
+
61
+ module FormatterParams
62
+ include Fluent::Configurable
63
+ # minimum section definition to instantiate formatter plugin instances
64
+ config_section :format, required: false, multi: true, param_name: :formatter_configs do
65
+ config_argument :usage, :string, default: ''
66
+ config_param :@type, :string
67
+ end
68
+ end
69
+
70
+ def self.included(mod)
71
+ mod.include FormatterParams
72
+ end
73
+
74
+ attr_reader :_formatters # for tests
75
+
76
+ def initialize
77
+ super
78
+ @_formatters_started = false
79
+ @_formatters = {} # usage => formatter
80
+ end
81
+
82
+ def configure(conf)
83
+ super
84
+
85
+ if @formatter_configs
86
+ @formatter_configs.each do |section|
87
+ if @_formatters[section.usage]
88
+ raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
89
+ end
90
+ formatter = Plugin.new_formatter(section[:@type], parent: self)
91
+ formatter.configure(section.corresponding_config_element)
92
+ @_formatters[section.usage] = formatter
93
+ end
94
+ end
95
+ end
96
+
97
+ def start
98
+ super
99
+ @_formatters_started = true
100
+ @_formatters.each_pair do |usage, formatter|
101
+ formatter.start
102
+ end
103
+ end
104
+
105
+ def formatter_operate(method_name, &block)
106
+ @_formatters.each_pair do |usage, formatter|
107
+ begin
108
+ formatter.send(method_name)
109
+ block.call(formatter) if block_given?
110
+ rescue => e
111
+ log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
112
+ end
113
+ end
114
+ end
115
+
116
+ def stop
117
+ super
118
+ formatter_operate(:stop)
119
+ end
120
+
121
+ def before_shutdown
122
+ formatter_operate(:before_shutdown)
123
+ super
124
+ end
125
+
126
+ def shutdown
127
+ formatter_operate(:shutdown)
128
+ super
129
+ end
130
+
131
+ def after_shutdown
132
+ formatter_operate(:after_shutdown)
133
+ super
134
+ end
135
+
136
+ def close
137
+ formatter_operate(:close)
138
+ super
139
+ end
140
+
141
+ def terminate
142
+ formatter_operate(:terminate)
143
+ @_formatters_started = false
144
+ @_formatters = {}
145
+ super
146
+ end
147
+ end
148
+ end
149
+ end