fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,677 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/output'
19
+ require 'fluent/plugin/bare_output'
20
+ require 'fluent/compat/call_super_mixin'
21
+ require 'fluent/compat/formatter_utils'
22
+ require 'fluent/compat/handle_tag_and_time_mixin'
23
+ require 'fluent/compat/parser_utils'
24
+ require 'fluent/compat/propagate_default'
25
+ require 'fluent/compat/record_filter_mixin'
26
+ require 'fluent/compat/output_chain'
27
+ require 'fluent/timezone'
28
+ require 'fluent/mixin'
29
+ require 'fluent/process' # to load Fluent::DetachProcessMixin
30
+
31
+ require 'fluent/plugin_helper/compat_parameters'
32
+
33
+ require 'time'
34
+
35
+ module Fluent
36
+ module Compat
37
+ NULL_OUTPUT_CHAIN = NullOutputChain.instance
38
+
39
+ module CompatOutputUtils
40
+ def self.buffer_section(conf)
41
+ conf.elements(name: 'buffer').first
42
+ end
43
+
44
+ def self.secondary_section(conf)
45
+ conf.elements(name: 'secondary').first
46
+ end
47
+ end
48
+
49
+ module BufferedEventStreamMixin
50
+ include Enumerable
51
+
52
+ def repeatable?
53
+ true
54
+ end
55
+
56
+ def each(&block)
57
+ msgpack_each(&block)
58
+ end
59
+
60
+ def to_msgpack_stream
61
+ read
62
+ end
63
+
64
+ def key
65
+ metadata.tag
66
+ end
67
+ end
68
+
69
+ module AddTimeSliceKeyToChunkMixin
70
+ def time_slice_format=(format)
71
+ @_time_slice_format = format
72
+ end
73
+
74
+ def timekey=(unit)
75
+ @_timekey = unit
76
+ end
77
+
78
+ def timezone=(tz)
79
+ @_timezone = tz
80
+ end
81
+
82
+ def assume_timekey!
83
+ @_formatter = Fluent::Timezone.formatter(@_timezone, @_time_slice_format)
84
+
85
+ return if self.metadata.timekey
86
+ if self.respond_to?(:path) && self.path =~ /\.(\d+)\.(?:b|q)(?:[a-z0-9]+)/
87
+ begin
88
+ self.metadata.timekey = Time.parse($1, @_time_slice_format).to_i
89
+ rescue ArgumentError
90
+ # unknown format / value as timekey
91
+ end
92
+ end
93
+ unless self.metadata.timekey
94
+ # file creation time is assumed in the time range of that time slice
95
+ # because the first record should be in that range.
96
+ time_int = self.created_at.to_i
97
+ self.metadata.timekey = time_int - (time_int % @_timekey)
98
+ end
99
+ end
100
+
101
+ def key
102
+ @_formatter.call(self.metadata.timekey)
103
+ end
104
+ end
105
+
106
+ module AddKeyToChunkMixin
107
+ def key
108
+ self.metadata.variables[:key]
109
+ end
110
+ end
111
+
112
+ module ChunkSizeCompatMixin
113
+ def size
114
+ self.bytesize
115
+ end
116
+
117
+ def size_of_events
118
+ @size + @adding_size
119
+ end
120
+ end
121
+
122
+ module BufferedChunkMixin
123
+ # prepend this module to BufferedOutput (including ObjectBufferedOutput) plugin singleton class
124
+ def write(chunk)
125
+ chunk.extend(ChunkSizeCompatMixin)
126
+ chunk.extend(AddKeyToChunkMixin) if chunk.metadata.variables && chunk.metadata.variables.has_key?(:key)
127
+ super
128
+ end
129
+ end
130
+
131
+ module TimeSliceChunkMixin
132
+ # prepend this module to TimeSlicedOutput plugin singleton class
133
+ def write(chunk)
134
+ chunk.extend(ChunkSizeCompatMixin)
135
+ chunk.extend(AddTimeSliceKeyToChunkMixin)
136
+ chunk.time_slice_format = @time_slice_format
137
+ chunk.timekey = @_timekey
138
+ chunk.timezone = @timezone
139
+ chunk.assume_timekey!
140
+ super
141
+ end
142
+ end
143
+
144
+ class Output < Fluent::Plugin::Output
145
+ # TODO: warn when deprecated
146
+
147
+ helpers :event_emitter, :inject
148
+
149
+ def support_in_v12_style?(feature)
150
+ case feature
151
+ when :synchronous then true
152
+ when :buffered then false
153
+ when :delayed_commit then false
154
+ when :custom_format then false
155
+ end
156
+ end
157
+
158
+ def process(tag, es)
159
+ emit(tag, es, NULL_OUTPUT_CHAIN)
160
+ end
161
+
162
+ def initialize
163
+ super
164
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
165
+ self.class.prepend Fluent::Compat::CallSuperMixin
166
+ end
167
+ end
168
+
169
+ def configure(conf)
170
+ ParserUtils.convert_parser_conf(conf)
171
+ FormatterUtils.convert_formatter_conf(conf)
172
+
173
+ super
174
+ end
175
+
176
+ def start
177
+ super
178
+
179
+ if instance_variable_defined?(:@formatter) && @inject_config
180
+ unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
181
+ if @formatter.respond_to?(:owner) && !@formatter.owner
182
+ @formatter.owner = self
183
+ @formatter.singleton_class.prepend FormatterUtils::InjectMixin
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
189
+
190
+ class MultiOutput < Fluent::Plugin::BareOutput
191
+ # TODO: warn when deprecated
192
+
193
+ helpers :event_emitter
194
+
195
+ def process(tag, es)
196
+ emit(tag, es, NULL_OUTPUT_CHAIN)
197
+ end
198
+ end
199
+
200
+ class BufferedOutput < Fluent::Plugin::Output
201
+ # TODO: warn when deprecated
202
+
203
+ helpers :event_emitter, :inject
204
+
205
+ def support_in_v12_style?(feature)
206
+ case feature
207
+ when :synchronous then false
208
+ when :buffered then true
209
+ when :delayed_commit then false
210
+ when :custom_format then true
211
+ end
212
+ end
213
+
214
+ desc 'The buffer type (memory, file)'
215
+ config_param :buffer_type, :string, default: 'memory'
216
+ desc 'The interval between data flushes.'
217
+ config_param :flush_interval, :time, default: 60
218
+ config_param :try_flush_interval, :float, default: 1
219
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
220
+ config_param :disable_retry_limit, :bool, default: false
221
+ desc 'The limit on the number of retries before buffered data is discarded'
222
+ config_param :retry_limit, :integer, default: 17
223
+ desc 'The initial intervals between write retries.'
224
+ config_param :retry_wait, :time, default: 1.0
225
+ desc 'The maximum intervals between write retries.'
226
+ config_param :max_retry_wait, :time, default: nil
227
+ desc 'The number of threads to flush the buffer.'
228
+ config_param :num_threads, :integer, default: 1
229
+ desc 'The interval between data flushes for queued chunk.'
230
+ config_param :queued_chunk_flush_interval, :time, default: 1
231
+
232
+ desc 'The size of each buffer chunk.'
233
+ config_param :buffer_chunk_limit, :size, default: 8*1024*1024
234
+ desc 'The length limit of the chunk queue.'
235
+ config_param :buffer_queue_limit, :integer, default: 256
236
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
237
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
238
+
239
+ config_param :flush_at_shutdown, :bool, default: true
240
+
241
+ BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS
242
+
243
+ def self.propagate_default_params
244
+ BUFFER_PARAMS
245
+ end
246
+ include PropagateDefault
247
+
248
+ def configure(conf)
249
+ bufconf = CompatOutputUtils.buffer_section(conf)
250
+ config_style = (bufconf ? :v1 : :v0)
251
+ if config_style == :v0
252
+ buf_params = {
253
+ "flush_mode" => "interval",
254
+ "retry_type" => "exponential_backoff",
255
+ }
256
+ BUFFER_PARAMS.each do |older, newer|
257
+ next unless newer
258
+ if conf.has_key?(older)
259
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
260
+ buf_params[newer] = 'throw_exception'
261
+ else
262
+ buf_params[newer] = conf[older]
263
+ end
264
+ end
265
+ end
266
+
267
+ conf.elements << Fluent::Config::Element.new('buffer', '', buf_params, [])
268
+ end
269
+
270
+ @includes_record_filter = self.class.ancestors.include?(Fluent::Compat::RecordFilterMixin)
271
+
272
+ methods_of_plugin = self.class.instance_methods(false)
273
+ @overrides_emit = methods_of_plugin.include?(:emit)
274
+ # RecordFilter mixin uses its own #format_stream method implementation
275
+ @overrides_format_stream = methods_of_plugin.include?(:format_stream) || @includes_record_filter
276
+
277
+ ParserUtils.convert_parser_conf(conf)
278
+ FormatterUtils.convert_formatter_conf(conf)
279
+
280
+ super
281
+
282
+ if config_style == :v1
283
+ unless @buffer_config.chunk_keys.empty?
284
+ raise Fluent::ConfigError, "this plugin '#{self.class}' cannot handle arguments for <buffer ...> section"
285
+ end
286
+ end
287
+
288
+ self.extend BufferedChunkMixin
289
+
290
+ if @overrides_emit
291
+ self.singleton_class.module_eval do
292
+ attr_accessor :last_emit_via_buffer
293
+ end
294
+ output_plugin = self
295
+ m = Module.new do
296
+ define_method(:emit) do |key, data, chain|
297
+ # receivers of this method are buffer instances
298
+ output_plugin.last_emit_via_buffer = [key, data]
299
+ end
300
+ end
301
+ @buffer.extend m
302
+ end
303
+ end
304
+
305
+ # original implementation of v0.12 BufferedOutput
306
+ def emit(tag, es, chain, key="")
307
+ # this method will not be used except for the case that plugin calls super
308
+ @emit_count += 1
309
+ data = format_stream(tag, es)
310
+ if @buffer.emit(key, data, chain)
311
+ submit_flush
312
+ end
313
+ end
314
+
315
+ def submit_flush
316
+ # nothing todo: blank method to be called from #emit of 3rd party plugins
317
+ end
318
+
319
+ def format_stream(tag, es)
320
+ # this method will not be used except for the case that plugin calls super
321
+ out = ''
322
+ es.each do |time, record|
323
+ out << format(tag, time, record)
324
+ end
325
+ out
326
+ end
327
+
328
+ # #format MUST be implemented in plugin
329
+ # #write is also
330
+
331
+ # This method overrides Fluent::Plugin::Output#handle_stream_simple
332
+ # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
333
+ def handle_stream_simple(tag, es, enqueue: false)
334
+ if @overrides_emit
335
+ current_emit_count = @emit_count
336
+ size = es.size
337
+ key = data = nil
338
+ begin
339
+ emit(tag, es, NULL_OUTPUT_CHAIN)
340
+ key, data = self.last_emit_via_buffer
341
+ ensure
342
+ @emit_count = current_emit_count
343
+ self.last_emit_via_buffer = nil
344
+ end
345
+ # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
346
+ meta = @buffer.metadata(variables: (key && !key.empty? ? {key: key} : nil))
347
+ write_guard do
348
+ @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
349
+ end
350
+ @counters_monitor.synchronize{ @emit_records += size }
351
+ return [meta]
352
+ end
353
+
354
+ if @overrides_format_stream
355
+ meta = metadata(nil, nil, nil)
356
+ size = es.size
357
+ bulk = format_stream(tag, es)
358
+ write_guard do
359
+ @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
360
+ end
361
+ @counters_monitor.synchronize{ @emit_records += size }
362
+ return [meta]
363
+ end
364
+
365
+ meta = metadata(nil, nil, nil)
366
+ size = es.size
367
+ data = es.map{|time,record| format(tag, time, record) }
368
+ write_guard do
369
+ @buffer.write({meta => data}, enqueue: enqueue)
370
+ end
371
+ @counters_monitor.synchronize{ @emit_records += size }
372
+ [meta]
373
+ end
374
+
375
+ def extract_placeholders(str, metadata)
376
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
377
+ end
378
+
379
+ def initialize
380
+ super
381
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
382
+ self.class.prepend Fluent::Compat::CallSuperMixin
383
+ end
384
+ end
385
+
386
+ def start
387
+ super
388
+
389
+ if instance_variable_defined?(:@formatter) && @inject_config
390
+ unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
391
+ if @formatter.respond_to?(:owner) && !@formatter.owner
392
+ @formatter.owner = self
393
+ @formatter.singleton_class.prepend FormatterUtils::InjectMixin
394
+ end
395
+ end
396
+ end
397
+ end
398
+ end
399
+
400
+ class ObjectBufferedOutput < Fluent::Plugin::Output
401
+ # TODO: warn when deprecated
402
+
403
+ helpers :event_emitter, :inject
404
+
405
+ # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag'
406
+ # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key
407
+ # in v1 style configuration
408
+
409
+ def support_in_v12_style?(feature)
410
+ case feature
411
+ when :synchronous then false
412
+ when :buffered then true
413
+ when :delayed_commit then false
414
+ when :custom_format then false
415
+ end
416
+ end
417
+
418
+ desc 'The buffer type (memory, file)'
419
+ config_param :buffer_type, :string, default: 'memory'
420
+ desc 'The interval between data flushes.'
421
+ config_param :flush_interval, :time, default: 60
422
+ config_param :try_flush_interval, :float, default: 1
423
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
424
+ config_param :disable_retry_limit, :bool, default: false
425
+ desc 'The limit on the number of retries before buffered data is discarded'
426
+ config_param :retry_limit, :integer, default: 17
427
+ desc 'The initial intervals between write retries.'
428
+ config_param :retry_wait, :time, default: 1.0
429
+ desc 'The maximum intervals between write retries.'
430
+ config_param :max_retry_wait, :time, default: nil
431
+ desc 'The number of threads to flush the buffer.'
432
+ config_param :num_threads, :integer, default: 1
433
+ desc 'The interval between data flushes for queued chunk.'
434
+ config_param :queued_chunk_flush_interval, :time, default: 1
435
+
436
+ desc 'The size of each buffer chunk.'
437
+ config_param :buffer_chunk_limit, :size, default: 8*1024*1024
438
+ desc 'The length limit of the chunk queue.'
439
+ config_param :buffer_queue_limit, :integer, default: 256
440
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
441
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
442
+
443
+ config_param :flush_at_shutdown, :bool, default: true
444
+
445
+ config_set_default :time_as_integer, true
446
+
447
+ BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS
448
+
449
+ def self.propagate_default_params
450
+ BUFFER_PARAMS
451
+ end
452
+ include PropagateDefault
453
+
454
+ def configure(conf)
455
+ bufconf = CompatOutputUtils.buffer_section(conf)
456
+ config_style = (bufconf ? :v1 : :v0)
457
+ if config_style == :v0
458
+ buf_params = {
459
+ "flush_mode" => "interval",
460
+ "retry_type" => "exponential_backoff",
461
+ }
462
+ BUFFER_PARAMS.each do |older, newer|
463
+ next unless newer
464
+ if conf.has_key?(older)
465
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
466
+ buf_params[newer] = 'throw_exception'
467
+ else
468
+ buf_params[newer] = conf[older]
469
+ end
470
+ end
471
+ end
472
+
473
+ conf.elements << Fluent::Config::Element.new('buffer', 'tag', buf_params, [])
474
+ end
475
+
476
+ ParserUtils.convert_parser_conf(conf)
477
+ FormatterUtils.convert_formatter_conf(conf)
478
+
479
+ super
480
+
481
+ if config_style == :v1
482
+ if @buffer_config.chunk_keys == ['tag']
483
+ raise Fluent::ConfigError, "this plugin '#{self.class}' allows <buffer tag> only"
484
+ end
485
+ end
486
+
487
+ self.extend BufferedChunkMixin
488
+ end
489
+
490
+ def format_stream(tag, es) # for BufferedOutputTestDriver
491
+ es.to_msgpack_stream(time_int: @time_as_integer)
492
+ end
493
+
494
+ def write(chunk)
495
+ write_objects(chunk.metadata.tag, chunk)
496
+ end
497
+
498
+ def extract_placeholders(str, metadata)
499
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
500
+ end
501
+
502
+ def initialize
503
+ super
504
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
505
+ self.class.prepend Fluent::Compat::CallSuperMixin
506
+ end
507
+ end
508
+
509
+ def start
510
+ super
511
+
512
+ if instance_variable_defined?(:@formatter) && @inject_config
513
+ unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
514
+ if @formatter.respond_to?(:owner) && !@formatter.owner
515
+ @formatter.owner = self
516
+ @formatter.singleton_class.prepend FormatterUtils::InjectMixin
517
+ end
518
+ end
519
+ end
520
+ end
521
+ end
522
+
523
+ class TimeSlicedOutput < Fluent::Plugin::Output
524
+ # TODO: warn when deprecated
525
+
526
+ helpers :event_emitter, :inject
527
+
528
+ def support_in_v12_style?(feature)
529
+ case feature
530
+ when :synchronous then false
531
+ when :buffered then true
532
+ when :delayed_commit then false
533
+ when :custom_format then true
534
+ end
535
+ end
536
+
537
+ desc 'The buffer type (memory, file)'
538
+ config_param :buffer_type, :string, default: 'file'
539
+ desc 'The interval between data flushes.'
540
+ config_param :flush_interval, :time, default: nil
541
+ config_param :try_flush_interval, :float, default: 1
542
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
543
+ config_param :disable_retry_limit, :bool, default: false
544
+ desc 'The limit on the number of retries before buffered data is discarded'
545
+ config_param :retry_limit, :integer, default: 17
546
+ desc 'The initial intervals between write retries.'
547
+ config_param :retry_wait, :time, default: 1.0
548
+ desc 'The maximum intervals between write retries.'
549
+ config_param :max_retry_wait, :time, default: nil
550
+ desc 'The number of threads to flush the buffer.'
551
+ config_param :num_threads, :integer, default: 1
552
+ desc 'The interval between data flushes for queued chunk.'
553
+ config_param :queued_chunk_flush_interval, :time, default: 1
554
+
555
+ desc 'The time format used as part of the file name.'
556
+ config_param :time_slice_format, :string, default: '%Y%m%d'
557
+ desc 'The amount of time Fluentd will wait for old logs to arrive.'
558
+ config_param :time_slice_wait, :time, default: 10*60
559
+ desc 'Parse the time value in the specified timezone'
560
+ config_param :timezone, :string, default: nil
561
+
562
+ desc 'The size of each buffer chunk.'
563
+ config_param :buffer_chunk_limit, :size, default: 256*1024*1024
564
+ desc 'The length limit of the chunk queue.'
565
+ config_param :buffer_queue_limit, :integer, default: 256
566
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
567
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
568
+
569
+ config_param :flush_at_shutdown, :bool, default: false
570
+
571
+ attr_accessor :localtime
572
+
573
+ config_section :buffer do
574
+ config_set_default :@type, 'file'
575
+ end
576
+
577
+ BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS.merge(Fluent::PluginHelper::CompatParameters::BUFFER_TIME_SLICED_PARAMS)
578
+
579
+ def initialize
580
+ super
581
+ @localtime = true
582
+
583
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
584
+ self.class.prepend Fluent::Compat::CallSuperMixin
585
+ end
586
+ end
587
+
588
+ def self.propagate_default_params
589
+ BUFFER_PARAMS
590
+ end
591
+ include PropagateDefault
592
+
593
+ def configure(conf)
594
+ bufconf = CompatOutputUtils.buffer_section(conf)
595
+ config_style = (bufconf ? :v1 : :v0)
596
+ if config_style == :v0
597
+ buf_params = {
598
+ "flush_mode" => (conf['flush_interval'] ? "interval" : "lazy"),
599
+ "retry_type" => "exponential_backoff",
600
+ }
601
+ BUFFER_PARAMS.each do |older, newer|
602
+ next unless newer
603
+ if conf.has_key?(older)
604
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
605
+ buf_params[newer] = 'throw_exception'
606
+ else
607
+ buf_params[newer] = conf[older]
608
+ end
609
+ end
610
+ end
611
+
612
+ if conf['timezone']
613
+ @timezone = conf['timezone']
614
+ Fluent::Timezone.validate!(@timezone)
615
+ elsif conf['utc']
616
+ @timezone = "+0000"
617
+ @localtime = false
618
+ elsif conf['localtime']
619
+ @timezone = Time.now.strftime('%z')
620
+ @localtime = true
621
+ else
622
+ @timezone = "+0000" # v0.12 assumes UTC without any configuration
623
+ @localtime = false
624
+ end
625
+
626
+ @_timekey = case conf['time_slice_format']
627
+ when /\%S/ then 1
628
+ when /\%M/ then 60
629
+ when /\%H/ then 3600
630
+ when /\%d/ then 86400
631
+ when nil then 86400 # default value of TimeSlicedOutput.time_slice_format is '%Y%m%d'
632
+ else
633
+ raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
634
+ end
635
+ buf_params["timekey"] = @_timekey
636
+
637
+ conf.elements << Fluent::Config::Element.new('buffer', 'time', buf_params, [])
638
+ end
639
+
640
+ ParserUtils.convert_parser_conf(conf)
641
+ FormatterUtils.convert_formatter_conf(conf)
642
+
643
+ super
644
+
645
+ if config_style == :v1
646
+ if @buffer_config.chunk_keys == ['tag']
647
+ raise Fluent::ConfigError, "this plugin '#{self.class}' allows <buffer tag> only"
648
+ end
649
+ end
650
+
651
+ self.extend TimeSliceChunkMixin
652
+ end
653
+
654
+ def start
655
+ super
656
+
657
+ if instance_variable_defined?(:@formatter) && @inject_config
658
+ unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
659
+ if @formatter.respond_to?(:owner) && !@formatter.owner
660
+ @formatter.owner = self
661
+ @formatter.singleton_class.prepend FormatterUtils::InjectMixin
662
+ end
663
+ end
664
+ end
665
+ end
666
+
667
+ # Original TimeSlicedOutput#emit doesn't call #format_stream
668
+
669
+ # #format MUST be implemented in plugin
670
+ # #write is also
671
+
672
+ def extract_placeholders(str, metadata)
673
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
674
+ end
675
+ end
676
+ end
677
+ end