fluentd 0.12.43 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (253) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +2 -0
  4. data/.travis.yml +33 -21
  5. data/CONTRIBUTING.md +1 -0
  6. data/ChangeLog +1239 -0
  7. data/README.md +0 -25
  8. data/Rakefile +2 -1
  9. data/Vagrantfile +17 -0
  10. data/appveyor.yml +35 -0
  11. data/example/filter_stdout.conf +5 -5
  12. data/example/in_forward.conf +2 -2
  13. data/example/in_http.conf +2 -2
  14. data/example/in_out_forward.conf +17 -0
  15. data/example/in_syslog.conf +2 -2
  16. data/example/in_tail.conf +2 -2
  17. data/example/in_tcp.conf +2 -2
  18. data/example/in_udp.conf +2 -2
  19. data/example/out_copy.conf +4 -4
  20. data/example/out_file.conf +2 -2
  21. data/example/out_forward.conf +2 -2
  22. data/example/out_forward_buf_file.conf +23 -0
  23. data/example/v0_12_filter.conf +8 -8
  24. data/fluent.conf +29 -0
  25. data/fluentd.gemspec +18 -11
  26. data/lib/fluent/agent.rb +60 -58
  27. data/lib/fluent/command/cat.rb +1 -1
  28. data/lib/fluent/command/debug.rb +7 -5
  29. data/lib/fluent/command/fluentd.rb +97 -2
  30. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  31. data/lib/fluent/compat/filter.rb +50 -0
  32. data/lib/fluent/compat/formatter.rb +109 -0
  33. data/lib/fluent/compat/input.rb +50 -0
  34. data/lib/fluent/compat/output.rb +617 -0
  35. data/lib/fluent/compat/output_chain.rb +60 -0
  36. data/lib/fluent/compat/parser.rb +163 -0
  37. data/lib/fluent/compat/propagate_default.rb +62 -0
  38. data/lib/fluent/config.rb +23 -20
  39. data/lib/fluent/config/configure_proxy.rb +119 -70
  40. data/lib/fluent/config/dsl.rb +5 -18
  41. data/lib/fluent/config/element.rb +72 -8
  42. data/lib/fluent/config/error.rb +0 -3
  43. data/lib/fluent/config/literal_parser.rb +0 -2
  44. data/lib/fluent/config/parser.rb +4 -4
  45. data/lib/fluent/config/section.rb +39 -28
  46. data/lib/fluent/config/types.rb +2 -13
  47. data/lib/fluent/config/v1_parser.rb +1 -3
  48. data/lib/fluent/configurable.rb +48 -16
  49. data/lib/fluent/daemon.rb +15 -0
  50. data/lib/fluent/engine.rb +26 -52
  51. data/lib/fluent/env.rb +6 -4
  52. data/lib/fluent/event.rb +58 -11
  53. data/lib/fluent/event_router.rb +5 -5
  54. data/lib/fluent/filter.rb +2 -50
  55. data/lib/fluent/formatter.rb +4 -293
  56. data/lib/fluent/input.rb +2 -32
  57. data/lib/fluent/label.rb +2 -2
  58. data/lib/fluent/load.rb +3 -2
  59. data/lib/fluent/log.rb +107 -38
  60. data/lib/fluent/match.rb +0 -36
  61. data/lib/fluent/mixin.rb +117 -7
  62. data/lib/fluent/msgpack_factory.rb +62 -0
  63. data/lib/fluent/output.rb +7 -612
  64. data/lib/fluent/output_chain.rb +23 -0
  65. data/lib/fluent/parser.rb +4 -800
  66. data/lib/fluent/plugin.rb +100 -121
  67. data/lib/fluent/plugin/bare_output.rb +63 -0
  68. data/lib/fluent/plugin/base.rb +121 -0
  69. data/lib/fluent/plugin/buf_file.rb +101 -182
  70. data/lib/fluent/plugin/buf_memory.rb +9 -92
  71. data/lib/fluent/plugin/buffer.rb +473 -0
  72. data/lib/fluent/plugin/buffer/chunk.rb +135 -0
  73. data/lib/fluent/plugin/buffer/file_chunk.rb +339 -0
  74. data/lib/fluent/plugin/buffer/memory_chunk.rb +100 -0
  75. data/lib/fluent/plugin/exec_util.rb +80 -75
  76. data/lib/fluent/plugin/file_util.rb +33 -28
  77. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  78. data/lib/fluent/plugin/filter.rb +51 -0
  79. data/lib/fluent/plugin/filter_grep.rb +13 -40
  80. data/lib/fluent/plugin/filter_record_transformer.rb +22 -18
  81. data/lib/fluent/plugin/formatter.rb +93 -0
  82. data/lib/fluent/plugin/formatter_csv.rb +48 -0
  83. data/lib/fluent/plugin/formatter_hash.rb +32 -0
  84. data/lib/fluent/plugin/formatter_json.rb +47 -0
  85. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  86. data/lib/fluent/plugin/formatter_msgpack.rb +32 -0
  87. data/lib/fluent/plugin/formatter_out_file.rb +45 -0
  88. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  89. data/lib/fluent/plugin/formatter_stdout.rb +39 -0
  90. data/lib/fluent/plugin/in_debug_agent.rb +4 -0
  91. data/lib/fluent/plugin/in_dummy.rb +22 -18
  92. data/lib/fluent/plugin/in_exec.rb +18 -8
  93. data/lib/fluent/plugin/in_forward.rb +36 -79
  94. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  95. data/lib/fluent/plugin/in_http.rb +21 -18
  96. data/lib/fluent/plugin/in_monitor_agent.rb +15 -48
  97. data/lib/fluent/plugin/in_object_space.rb +6 -1
  98. data/lib/fluent/plugin/in_stream.rb +7 -3
  99. data/lib/fluent/plugin/in_syslog.rb +46 -95
  100. data/lib/fluent/plugin/in_tail.rb +58 -640
  101. data/lib/fluent/plugin/in_tcp.rb +8 -1
  102. data/lib/fluent/plugin/in_udp.rb +8 -18
  103. data/lib/fluent/plugin/input.rb +33 -0
  104. data/lib/fluent/plugin/multi_output.rb +95 -0
  105. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  106. data/lib/fluent/plugin/out_copy.rb +11 -7
  107. data/lib/fluent/plugin/out_exec.rb +15 -11
  108. data/lib/fluent/plugin/out_exec_filter.rb +18 -10
  109. data/lib/fluent/plugin/out_file.rb +34 -5
  110. data/lib/fluent/plugin/out_forward.rb +25 -19
  111. data/lib/fluent/plugin/out_null.rb +0 -14
  112. data/lib/fluent/plugin/out_roundrobin.rb +11 -7
  113. data/lib/fluent/plugin/out_stdout.rb +5 -7
  114. data/lib/fluent/plugin/out_stream.rb +3 -1
  115. data/lib/fluent/plugin/output.rb +979 -0
  116. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  117. data/lib/fluent/plugin/parser.rb +244 -0
  118. data/lib/fluent/plugin/parser_apache.rb +24 -0
  119. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  120. data/lib/fluent/plugin/parser_apache_error.rb +21 -0
  121. data/lib/fluent/plugin/parser_csv.rb +31 -0
  122. data/lib/fluent/plugin/parser_json.rb +79 -0
  123. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  124. data/lib/fluent/plugin/parser_multiline.rb +102 -0
  125. data/lib/fluent/plugin/parser_nginx.rb +24 -0
  126. data/lib/fluent/plugin/parser_none.rb +36 -0
  127. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  128. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  129. data/lib/fluent/plugin/socket_util.rb +119 -117
  130. data/lib/fluent/plugin/storage.rb +84 -0
  131. data/lib/fluent/plugin/storage_local.rb +116 -0
  132. data/lib/fluent/plugin/string_util.rb +16 -13
  133. data/lib/fluent/plugin_helper.rb +39 -0
  134. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  135. data/lib/fluent/plugin_helper/compat_parameters.rb +99 -0
  136. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  137. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  138. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  139. data/lib/fluent/plugin_helper/storage.rb +308 -0
  140. data/lib/fluent/plugin_helper/thread.rb +147 -0
  141. data/lib/fluent/plugin_helper/timer.rb +85 -0
  142. data/lib/fluent/plugin_id.rb +63 -0
  143. data/lib/fluent/process.rb +21 -30
  144. data/lib/fluent/registry.rb +21 -9
  145. data/lib/fluent/root_agent.rb +115 -40
  146. data/lib/fluent/supervisor.rb +330 -320
  147. data/lib/fluent/system_config.rb +42 -18
  148. data/lib/fluent/test.rb +6 -1
  149. data/lib/fluent/test/base.rb +23 -3
  150. data/lib/fluent/test/driver/base.rb +247 -0
  151. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  152. data/lib/fluent/test/driver/filter.rb +35 -0
  153. data/lib/fluent/test/driver/input.rb +31 -0
  154. data/lib/fluent/test/driver/output.rb +78 -0
  155. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  156. data/lib/fluent/test/filter_test.rb +0 -1
  157. data/lib/fluent/test/formatter_test.rb +2 -1
  158. data/lib/fluent/test/input_test.rb +23 -17
  159. data/lib/fluent/test/output_test.rb +28 -39
  160. data/lib/fluent/test/parser_test.rb +1 -1
  161. data/lib/fluent/time.rb +104 -1
  162. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  163. data/lib/fluent/version.rb +1 -1
  164. data/lib/fluent/winsvc.rb +72 -0
  165. data/test/compat/test_calls_super.rb +164 -0
  166. data/test/config/test_config_parser.rb +83 -0
  167. data/test/config/test_configurable.rb +547 -274
  168. data/test/config/test_configure_proxy.rb +146 -29
  169. data/test/config/test_dsl.rb +3 -181
  170. data/test/config/test_element.rb +274 -0
  171. data/test/config/test_literal_parser.rb +1 -1
  172. data/test/config/test_section.rb +79 -7
  173. data/test/config/test_system_config.rb +21 -0
  174. data/test/config/test_types.rb +3 -26
  175. data/test/helper.rb +78 -8
  176. data/test/plugin/test_bare_output.rb +118 -0
  177. data/test/plugin/test_base.rb +75 -0
  178. data/test/plugin/test_buf_file.rb +420 -521
  179. data/test/plugin/test_buf_memory.rb +32 -194
  180. data/test/plugin/test_buffer.rb +981 -0
  181. data/test/plugin/test_buffer_chunk.rb +110 -0
  182. data/test/plugin/test_buffer_file_chunk.rb +770 -0
  183. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  184. data/test/plugin/test_filter.rb +255 -0
  185. data/test/plugin/test_filter_grep.rb +2 -73
  186. data/test/plugin/test_filter_record_transformer.rb +24 -68
  187. data/test/plugin/test_filter_stdout.rb +6 -6
  188. data/test/plugin/test_in_debug_agent.rb +2 -0
  189. data/test/plugin/test_in_dummy.rb +11 -17
  190. data/test/plugin/test_in_exec.rb +6 -25
  191. data/test/plugin/test_in_forward.rb +112 -151
  192. data/test/plugin/test_in_gc_stat.rb +2 -0
  193. data/test/plugin/test_in_http.rb +106 -157
  194. data/test/plugin/test_in_object_space.rb +21 -5
  195. data/test/plugin/test_in_stream.rb +14 -13
  196. data/test/plugin/test_in_syslog.rb +30 -275
  197. data/test/plugin/test_in_tail.rb +95 -282
  198. data/test/plugin/test_in_tcp.rb +14 -0
  199. data/test/plugin/test_in_udp.rb +21 -67
  200. data/test/plugin/test_input.rb +122 -0
  201. data/test/plugin/test_multi_output.rb +180 -0
  202. data/test/plugin/test_out_buffered_null.rb +79 -0
  203. data/test/plugin/test_out_copy.rb +15 -2
  204. data/test/plugin/test_out_exec.rb +75 -25
  205. data/test/plugin/test_out_exec_filter.rb +74 -8
  206. data/test/plugin/test_out_file.rb +61 -7
  207. data/test/plugin/test_out_forward.rb +92 -15
  208. data/test/plugin/test_out_roundrobin.rb +1 -0
  209. data/test/plugin/test_out_stdout.rb +22 -13
  210. data/test/plugin/test_out_stream.rb +18 -0
  211. data/test/plugin/test_output.rb +515 -0
  212. data/test/plugin/test_output_as_buffered.rb +1540 -0
  213. data/test/plugin/test_output_as_buffered_overflow.rb +247 -0
  214. data/test/plugin/test_output_as_buffered_retries.rb +808 -0
  215. data/test/plugin/test_output_as_buffered_secondary.rb +776 -0
  216. data/test/plugin/test_output_as_standard.rb +362 -0
  217. data/test/plugin/test_owned_by.rb +35 -0
  218. data/test/plugin/test_storage.rb +167 -0
  219. data/test/plugin/test_storage_local.rb +8 -0
  220. data/test/plugin_helper/test_child_process.rb +599 -0
  221. data/test/plugin_helper/test_compat_parameters.rb +175 -0
  222. data/test/plugin_helper/test_event_emitter.rb +51 -0
  223. data/test/plugin_helper/test_event_loop.rb +52 -0
  224. data/test/plugin_helper/test_retry_state.rb +399 -0
  225. data/test/plugin_helper/test_storage.rb +411 -0
  226. data/test/plugin_helper/test_thread.rb +164 -0
  227. data/test/plugin_helper/test_timer.rb +100 -0
  228. data/test/scripts/exec_script.rb +0 -6
  229. data/test/scripts/fluent/plugin/out_test.rb +3 -0
  230. data/test/test_config.rb +13 -4
  231. data/test/test_event.rb +24 -13
  232. data/test/test_event_router.rb +8 -7
  233. data/test/test_event_time.rb +187 -0
  234. data/test/test_formatter.rb +13 -51
  235. data/test/test_input.rb +1 -1
  236. data/test/test_log.rb +239 -16
  237. data/test/test_mixin.rb +1 -1
  238. data/test/test_output.rb +53 -66
  239. data/test/test_parser.rb +105 -323
  240. data/test/test_plugin_helper.rb +81 -0
  241. data/test/test_root_agent.rb +4 -52
  242. data/test/test_supervisor.rb +272 -0
  243. data/test/test_unique_id.rb +47 -0
  244. metadata +181 -55
  245. data/CHANGELOG.md +0 -710
  246. data/lib/fluent/buffer.rb +0 -365
  247. data/lib/fluent/plugin/filter_parser.rb +0 -107
  248. data/lib/fluent/plugin/in_status.rb +0 -76
  249. data/lib/fluent/test/helpers.rb +0 -86
  250. data/test/plugin/data/log/foo/bar2 +0 -0
  251. data/test/plugin/test_filter_parser.rb +0 -744
  252. data/test/plugin/test_in_status.rb +0 -38
  253. data/test/test_buffer.rb +0 -624
@@ -0,0 +1,109 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/formatter'
19
+
20
+ require 'fluent/plugin/formatter_out_file'
21
+ require 'fluent/plugin/formatter_stdout'
22
+ require 'fluent/plugin/formatter_json'
23
+ require 'fluent/plugin/formatter_hash'
24
+ require 'fluent/plugin/formatter_msgpack'
25
+ require 'fluent/plugin/formatter_ltsv'
26
+ require 'fluent/plugin/formatter_csv'
27
+ require 'fluent/plugin/formatter_single_value'
28
+
29
+ module Fluent
30
+ module Compat
31
+ class Formatter < Fluent::Plugin::Formatter
32
+ # TODO: warn when deprecated
33
+ end
34
+
35
+ module TextFormatter
36
+ def self.register_template(type, template)
37
+ # TODO: warn when deprecated to use Plugin.register_formatter directly
38
+ if template.is_a?(Class)
39
+ Fluent::Plugin.register_formatter(type, template)
40
+ elsif template.respond_to?(:call) && template.arity == 3 # Proc.new { |tag, time, record| }
41
+ Fluent::Plugin.register_formatter(type, Proc.new { ProcWrappedFormatter.new(template) })
42
+ elsif template.respond_to?(:call)
43
+ Fluent::Plugin.register_formatter(type, template)
44
+ else
45
+ raise ArgumentError, "Template for formatter must be a Class or callable object"
46
+ end
47
+ end
48
+
49
+ def self.lookup(type)
50
+ # TODO: warn when deprecated to use Plugin.new_formatter(type, parent: plugin)
51
+ Fluent::Plugin.new_formatter(type)
52
+ end
53
+
54
+ # Keep backward-compatibility
55
+ def self.create(conf)
56
+ # TODO: warn when deprecated
57
+ format = conf['format']
58
+ if format.nil?
59
+ raise ConfigError, "'format' parameter is required"
60
+ end
61
+
62
+ formatter = lookup(format)
63
+ if formatter.respond_to?(:configure)
64
+ formatter.configure(conf)
65
+ end
66
+ formatter
67
+ end
68
+
69
+ HandleTagAndTimeMixin = Fluent::Plugin::Formatter::HandleTagAndTimeMixin
70
+ StructuredFormatMixin = Fluent::Plugin::Formatter::StructuredFormatMixin
71
+
72
+ class ProcWrappedFormatter < Fluent::Plugin::ProcWrappedFormatter
73
+ # TODO: warn when deprecated
74
+ end
75
+
76
+ class OutFileFormatter < Fluent::Plugin::OutFileFormatter
77
+ # TODO: warn when deprecated
78
+ end
79
+
80
+ class StdoutFormatter < Fluent::Plugin::StdoutFormatter
81
+ # TODO: warn when deprecated
82
+ end
83
+
84
+ class JSONFormatter < Fluent::Plugin::JSONFormatter
85
+ # TODO: warn when deprecated
86
+ end
87
+
88
+ class HashFormatter < Fluent::Plugin::HashFormatter
89
+ # TODO: warn when deprecated
90
+ end
91
+
92
+ class MessagePackFormatter < Fluent::Plugin::MessagePackFormatter
93
+ # TODO: warn when deprecated
94
+ end
95
+
96
+ class LabeledTSVFormatter < Fluent::Plugin::LabeledTSVFormatter
97
+ # TODO: warn when deprecated
98
+ end
99
+
100
+ class CsvFormatter < Fluent::Plugin::CsvFormatter
101
+ # TODO: warn when deprecated
102
+ end
103
+
104
+ class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter
105
+ # TODO: warn when deprecated
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,50 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/input'
19
+ require 'fluent/compat/call_super_mixin'
20
+
21
+ module Fluent
22
+ module Compat
23
+ class Input < Fluent::Plugin::Input
24
+ # TODO: warn when deprecated
25
+
26
+ def initialize
27
+ super
28
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
29
+ self.class.module_eval do
30
+ prepend Fluent::Compat::CallSuperMixin
31
+ end
32
+ end
33
+ end
34
+
35
+ # These definitions are to get instance methods of superclass of 3rd party plugins
36
+ # to make it sure to call super
37
+ def start
38
+ super
39
+ end
40
+
41
+ def before_shutdown
42
+ super
43
+ end
44
+
45
+ def shutdown
46
+ super
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,617 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/output'
19
+ require 'fluent/plugin/bare_output'
20
+ require 'fluent/compat/call_super_mixin'
21
+ require 'fluent/compat/propagate_default'
22
+ require 'fluent/compat/output_chain'
23
+ require 'fluent/timezone'
24
+ require 'fluent/mixin'
25
+
26
+ require 'fluent/plugin_helper/compat_parameters'
27
+
28
+ require 'time'
29
+
30
+ module Fluent
31
+ module Compat
32
+ NULL_OUTPUT_CHAIN = NullOutputChain.instance
33
+
34
+ module CompatOutputUtils
35
+ def self.buffer_section(conf)
36
+ conf.elements(name: 'buffer').first
37
+ end
38
+
39
+ def self.secondary_section(conf)
40
+ conf.elements(name: 'secondary').first
41
+ end
42
+ end
43
+
44
+ module BufferedEventStreamMixin
45
+ include Enumerable
46
+
47
+ def repeatable?
48
+ true
49
+ end
50
+
51
+ def each(&block)
52
+ msgpack_each(&block)
53
+ end
54
+
55
+ def to_msgpack_stream
56
+ read
57
+ end
58
+
59
+ def key
60
+ metadata.tag
61
+ end
62
+ end
63
+
64
+ module AddTimeSliceKeyToChunkMixin
65
+ def time_slice_format=(format)
66
+ @_time_slice_format = format
67
+ end
68
+
69
+ def timekey=(unit)
70
+ @_timekey = unit
71
+ end
72
+
73
+ def timezone=(tz)
74
+ @_timezone = tz
75
+ end
76
+
77
+ def assume_timekey!
78
+ @_formatter = Fluent::Timezone.formatter(@_timezone, @_time_slice_format)
79
+
80
+ return if self.metadata.timekey
81
+ if self.respond_to?(:path) && self.path =~ /\.(\d+)\.(?:b|q)(?:[a-z0-9]+)/
82
+ begin
83
+ self.metadata.timekey = Time.parse($1, @_time_slice_format).to_i
84
+ rescue ArgumentError
85
+ # unknown format / value as timekey
86
+ end
87
+ end
88
+ unless self.metadata.timekey
89
+ # file creation time is assumed in the time range of that time slice
90
+ # because the first record should be in that range.
91
+ time_int = self.created_at.to_i
92
+ self.metadata.timekey = time_int - (time_int % @_timekey)
93
+ end
94
+ end
95
+
96
+ def key
97
+ @_formatter.call(self.metadata.timekey)
98
+ end
99
+ end
100
+
101
+ module AddKeyToChunkMixin
102
+ def key
103
+ self.metadata.variables[:key]
104
+ end
105
+ end
106
+
107
+ module ChunkSizeCompatMixin
108
+ def size
109
+ self.bytesize
110
+ end
111
+ end
112
+
113
+ module BufferedChunkMixin
114
+ # prepend this module to BufferedOutput (including ObjectBufferedOutput) plugin singleton class
115
+ def write(chunk)
116
+ chunk.extend(ChunkSizeCompatMixin)
117
+ chunk.extend(AddKeyToChunkMixin) if chunk.metadata.variables && chunk.metadata.variables.has_key?(:key)
118
+ super
119
+ end
120
+ end
121
+
122
+ module TimeSliceChunkMixin
123
+ # prepend this module to TimeSlicedOutput plugin singleton class
124
+ def write(chunk)
125
+ chunk.extend(ChunkSizeCompatMixin)
126
+ chunk.extend(AddTimeSliceKeyToChunkMixin)
127
+ chunk.time_slice_format = @time_slice_format
128
+ chunk.timekey = @_timekey
129
+ chunk.timezone = @timezone
130
+ chunk.assume_timekey!
131
+ super
132
+ end
133
+ end
134
+
135
+ class Output < Fluent::Plugin::Output
136
+ # TODO: warn when deprecated
137
+
138
+ helpers :event_emitter
139
+
140
+ def support_in_v12_style?(feature)
141
+ case feature
142
+ when :synchronous then true
143
+ when :buffered then false
144
+ when :delayed_commit then false
145
+ when :custom_format then false
146
+ end
147
+ end
148
+
149
+ def process(tag, es)
150
+ emit(tag, es, NULL_OUTPUT_CHAIN)
151
+ end
152
+
153
+ def initialize
154
+ super
155
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
156
+ self.class.module_eval do
157
+ prepend Fluent::Compat::CallSuperMixin
158
+ end
159
+ end
160
+ end
161
+ end
162
+
163
+ class MultiOutput < Fluent::Plugin::BareOutput
164
+ # TODO: warn when deprecated
165
+
166
+ helpers :event_emitter
167
+
168
+ def process(tag, es)
169
+ emit(tag, es, NULL_OUTPUT_CHAIN)
170
+ end
171
+ end
172
+
173
+ class BufferedOutput < Fluent::Plugin::Output
174
+ # TODO: warn when deprecated
175
+
176
+ helpers :event_emitter
177
+
178
+ def support_in_v12_style?(feature)
179
+ case feature
180
+ when :synchronous then false
181
+ when :buffered then true
182
+ when :delayed_commit then false
183
+ when :custom_format then true
184
+ end
185
+ end
186
+
187
+ desc 'The buffer type (memory, file)'
188
+ config_param :buffer_type, :string, default: 'memory'
189
+ desc 'The interval between data flushes.'
190
+ config_param :flush_interval, :time, default: 60
191
+ config_param :try_flush_interval, :float, default: 1
192
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
193
+ config_param :disable_retry_limit, :bool, default: false
194
+ desc 'The limit on the number of retries before buffered data is discarded'
195
+ config_param :retry_limit, :integer, default: 17
196
+ desc 'The initial intervals between write retries.'
197
+ config_param :retry_wait, :time, default: 1.0
198
+ desc 'The maximum intervals between write retries.'
199
+ config_param :max_retry_wait, :time, default: nil
200
+ desc 'The number of threads to flush the buffer.'
201
+ config_param :num_threads, :integer, default: 1
202
+ desc 'The interval between data flushes for queued chunk.'
203
+ config_param :queued_chunk_flush_interval, :time, default: 1
204
+
205
+ desc 'The size of each buffer chunk.'
206
+ config_param :buffer_chunk_limit, :size, default: 8*1024*1024
207
+ desc 'The length limit of the chunk queue.'
208
+ config_param :buffer_queue_limit, :integer, default: 256
209
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
210
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
211
+
212
+ config_param :flush_at_shutdown, :bool, default: true
213
+
214
+ PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
215
+
216
+ def self.propagate_default_params
217
+ PARAMS_MAP
218
+ end
219
+ include PropagateDefault
220
+
221
+ def configure(conf)
222
+ bufconf = CompatOutputUtils.buffer_section(conf)
223
+ config_style = (bufconf ? :v1 : :v0)
224
+ if config_style == :v0
225
+ buf_params = {
226
+ "flush_mode" => "interval",
227
+ "retry_type" => "exponential_backoff",
228
+ }
229
+ PARAMS_MAP.each do |older, newer|
230
+ next unless newer
231
+ if conf.has_key?(older)
232
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
233
+ buf_params[newer] = 'throw_exception'
234
+ else
235
+ buf_params[newer] = conf[older]
236
+ end
237
+ end
238
+ end
239
+
240
+ conf.elements << Fluent::Config::Element.new('buffer', '', buf_params, [])
241
+ end
242
+
243
+ @includes_record_filter = self.class.ancestors.include?(Fluent::RecordFilterMixin) # TODO rename Compat::RecordFilterMixin
244
+
245
+ methods_of_plugin = self.class.instance_methods(false)
246
+ @overrides_emit = methods_of_plugin.include?(:emit)
247
+ # RecordFilter mixin uses its own #format_stream method implementation
248
+ @overrides_format_stream = methods_of_plugin.include?(:format_stream) || @includes_record_filter
249
+
250
+ super
251
+
252
+ if config_style == :v1
253
+ unless @buffer_config.chunk_keys.empty?
254
+ raise Fluent::ConfigError, "this plugin '#{self.class}' cannot handle arguments for <buffer ...> section"
255
+ end
256
+ end
257
+
258
+ (class << self; self; end).module_eval do
259
+ prepend BufferedChunkMixin
260
+ end
261
+
262
+ if @overrides_emit
263
+ self.singleton_class.module_eval do
264
+ attr_accessor :last_emit_via_buffer
265
+ end
266
+ output_plugin = self
267
+ m = Module.new do
268
+ define_method(:emit) do |key, data, chain|
269
+ # receivers of this method are buffer instances
270
+ output_plugin.last_emit_via_buffer = [key, data]
271
+ end
272
+ end
273
+ @buffer.singleton_class.module_eval do
274
+ prepend m
275
+ end
276
+ end
277
+ end
278
+
279
+ # original implementation of v0.12 BufferedOutput
280
+ def emit(tag, es, chain, key="")
281
+ # this method will not be used except for the case that plugin calls super
282
+ @emit_count += 1
283
+ data = format_stream(tag, es)
284
+ if @buffer.emit(key, data, chain)
285
+ submit_flush
286
+ end
287
+ end
288
+
289
+ def submit_flush
290
+ # nothing todo: blank method to be called from #emit of 3rd party plugins
291
+ end
292
+
293
+ def format_stream(tag, es)
294
+ # this method will not be used except for the case that plugin calls super
295
+ out = ''
296
+ es.each do |time, record|
297
+ out << format(tag, time, record)
298
+ end
299
+ out
300
+ end
301
+
302
+ # #format MUST be implemented in plugin
303
+ # #write is also
304
+
305
+ # This method overrides Fluent::Plugin::Output#handle_stream_simple
306
+ # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
307
+ def handle_stream_simple(tag, es, enqueue: false)
308
+ if @overrides_emit
309
+ current_emit_count = @emit_count
310
+ size = es.size
311
+ key = data = nil
312
+ begin
313
+ emit(tag, es, NULL_OUTPUT_CHAIN)
314
+ key, data = self.last_emit_via_buffer
315
+ ensure
316
+ @emit_count = current_emit_count
317
+ self.last_emit_via_buffer = nil
318
+ end
319
+ # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
320
+ meta = @buffer.metadata(variables: (key && !key.empty? ? {key: key} : nil))
321
+ write_guard do
322
+ @buffer.write({meta => [data, size]}, bulk: true, enqueue: enqueue)
323
+ end
324
+ @counters_monitor.synchronize{ @emit_records += size }
325
+ return [meta]
326
+ end
327
+
328
+ if @overrides_format_stream
329
+ meta = metadata(nil, nil, nil)
330
+ size = es.size
331
+ bulk = format_stream(tag, es)
332
+ write_guard do
333
+ @buffer.write({meta => [bulk, size]}, bulk: true, enqueue: enqueue)
334
+ end
335
+ @counters_monitor.synchronize{ @emit_records += size }
336
+ return [meta]
337
+ end
338
+
339
+ meta = metadata(nil, nil, nil)
340
+ es_size = 0
341
+ es_bulk = es.map{|time,record| es_size += 1; format(tag, time, record) }.join
342
+ write_guard do
343
+ @buffer.write({meta => [es_bulk, es_size]}, bulk: true, enqueue: enqueue)
344
+ end
345
+ @counters_monitor.synchronize{ @emit_records += es_size }
346
+ [meta]
347
+ end
348
+
349
+ def extract_placeholders(str, metadata)
350
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
351
+ end
352
+
353
+ def initialize
354
+ super
355
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
356
+ self.class.module_eval do
357
+ prepend Fluent::Compat::CallSuperMixin
358
+ end
359
+ end
360
+ end
361
+ end
362
+
363
+ class ObjectBufferedOutput < Fluent::Plugin::Output
364
+ # TODO: warn when deprecated
365
+
366
+ helpers :event_emitter
367
+
368
+ # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag'
369
+ # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key
370
+ # in v1 style configuration
371
+
372
+ def support_in_v12_style?(feature)
373
+ case feature
374
+ when :synchronous then false
375
+ when :buffered then true
376
+ when :delayed_commit then false
377
+ when :custom_format then false
378
+ end
379
+ end
380
+
381
+ desc 'The buffer type (memory, file)'
382
+ config_param :buffer_type, :string, default: 'memory'
383
+ desc 'The interval between data flushes.'
384
+ config_param :flush_interval, :time, default: 60
385
+ config_param :try_flush_interval, :float, default: 1
386
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
387
+ config_param :disable_retry_limit, :bool, default: false
388
+ desc 'The limit on the number of retries before buffered data is discarded'
389
+ config_param :retry_limit, :integer, default: 17
390
+ desc 'The initial intervals between write retries.'
391
+ config_param :retry_wait, :time, default: 1.0
392
+ desc 'The maximum intervals between write retries.'
393
+ config_param :max_retry_wait, :time, default: nil
394
+ desc 'The number of threads to flush the buffer.'
395
+ config_param :num_threads, :integer, default: 1
396
+ desc 'The interval between data flushes for queued chunk.'
397
+ config_param :queued_chunk_flush_interval, :time, default: 1
398
+
399
+ desc 'The size of each buffer chunk.'
400
+ config_param :buffer_chunk_limit, :size, default: 8*1024*1024
401
+ desc 'The length limit of the chunk queue.'
402
+ config_param :buffer_queue_limit, :integer, default: 256
403
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
404
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
405
+
406
+ config_param :flush_at_shutdown, :bool, default: true
407
+
408
+ config_set_default :time_as_integer, true
409
+
410
+ PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
411
+
412
+ def self.propagate_default_params
413
+ PARAMS_MAP
414
+ end
415
+ include PropagateDefault
416
+
417
+ def configure(conf)
418
+ bufconf = CompatOutputUtils.buffer_section(conf)
419
+ config_style = (bufconf ? :v1 : :v0)
420
+ if config_style == :v0
421
+ buf_params = {
422
+ "flush_mode" => "interval",
423
+ "retry_type" => "exponential_backoff",
424
+ }
425
+ PARAMS_MAP.each do |older, newer|
426
+ next unless newer
427
+ if conf.has_key?(older)
428
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
429
+ buf_params[newer] = 'throw_exception'
430
+ else
431
+ buf_params[newer] = conf[older]
432
+ end
433
+ end
434
+ end
435
+
436
+ conf.elements << Fluent::Config::Element.new('buffer', 'tag', buf_params, [])
437
+ end
438
+
439
+ super
440
+
441
+ if config_style == :v1
442
+ if @buffer_config.chunk_keys == ['tag']
443
+ raise Fluent::ConfigError, "this plugin '#{self.class}' allows <buffer tag> only"
444
+ end
445
+ end
446
+
447
+ (class << self; self; end).module_eval do
448
+ prepend BufferedChunkMixin
449
+ end
450
+ end
451
+
452
+ def format_stream(tag, es) # for BufferedOutputTestDriver
453
+ es.to_msgpack_stream(time_int: @time_as_integer)
454
+ end
455
+
456
+ def write(chunk)
457
+ write_objects(chunk.metadata.tag, chunk)
458
+ end
459
+
460
+ def extract_placeholders(str, metadata)
461
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
462
+ end
463
+
464
+ def initialize
465
+ super
466
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
467
+ self.class.module_eval do
468
+ prepend Fluent::Compat::CallSuperMixin
469
+ end
470
+ end
471
+ end
472
+ end
473
+
474
+ class TimeSlicedOutput < Fluent::Plugin::Output
475
+ # TODO: warn when deprecated
476
+
477
+ helpers :event_emitter
478
+
479
+ def support_in_v12_style?(feature)
480
+ case feature
481
+ when :synchronous then false
482
+ when :buffered then true
483
+ when :delayed_commit then false
484
+ when :custom_format then true
485
+ end
486
+ end
487
+
488
+ desc 'The buffer type (memory, file)'
489
+ config_param :buffer_type, :string, default: 'file'
490
+ desc 'The interval between data flushes.'
491
+ config_param :flush_interval, :time, default: nil
492
+ config_param :try_flush_interval, :float, default: 1
493
+ desc 'If true, the value of `retry_value` is ignored and there is no limit'
494
+ config_param :disable_retry_limit, :bool, default: false
495
+ desc 'The limit on the number of retries before buffered data is discarded'
496
+ config_param :retry_limit, :integer, default: 17
497
+ desc 'The initial intervals between write retries.'
498
+ config_param :retry_wait, :time, default: 1.0
499
+ desc 'The maximum intervals between write retries.'
500
+ config_param :max_retry_wait, :time, default: nil
501
+ desc 'The number of threads to flush the buffer.'
502
+ config_param :num_threads, :integer, default: 1
503
+ desc 'The interval between data flushes for queued chunk.'
504
+ config_param :queued_chunk_flush_interval, :time, default: 1
505
+
506
+ desc 'The time format used as part of the file name.'
507
+ config_param :time_slice_format, :string, default: '%Y%m%d'
508
+ desc 'The amount of time Fluentd will wait for old logs to arrive.'
509
+ config_param :time_slice_wait, :time, default: 10*60
510
+ desc 'Parse the time value in the specified timezone'
511
+ config_param :timezone, :string, default: nil
512
+
513
+ desc 'The size of each buffer chunk.'
514
+ config_param :buffer_chunk_limit, :size, default: 256*1024*1024
515
+ desc 'The length limit of the chunk queue.'
516
+ config_param :buffer_queue_limit, :integer, default: 256
517
+ desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
518
+ config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
519
+
520
+ config_param :flush_at_shutdown, :bool, default: false
521
+
522
+ attr_accessor :localtime
523
+
524
+ config_section :buffer do
525
+ config_set_default :@type, 'file'
526
+ end
527
+
528
+ PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP.merge(Fluent::PluginHelper::CompatParameters::TIME_SLICED_PARAMS)
529
+
530
+ def initialize
531
+ super
532
+ @localtime = true
533
+
534
+ unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
535
+ self.class.module_eval do
536
+ prepend Fluent::Compat::CallSuperMixin
537
+ end
538
+ end
539
+ end
540
+
541
+ def self.propagate_default_params
542
+ PARAMS_MAP
543
+ end
544
+ include PropagateDefault
545
+
546
+ def configure(conf)
547
+ bufconf = CompatOutputUtils.buffer_section(conf)
548
+ config_style = (bufconf ? :v1 : :v0)
549
+ if config_style == :v0
550
+ buf_params = {
551
+ "flush_mode" => (conf['flush_interval'] ? "interval" : "lazy"),
552
+ "retry_type" => "exponential_backoff",
553
+ }
554
+ PARAMS_MAP.each do |older, newer|
555
+ next unless newer
556
+ if conf.has_key?(older)
557
+ if older == 'buffer_queue_full_action' && conf[older] == 'exception'
558
+ buf_params[newer] = 'throw_exception'
559
+ else
560
+ buf_params[newer] = conf[older]
561
+ end
562
+ end
563
+ end
564
+
565
+ if conf['timezone']
566
+ @timezone = conf['timezone']
567
+ Fluent::Timezone.validate!(@timezone)
568
+ elsif conf['utc']
569
+ @timezone = "+0000"
570
+ @localtime = false
571
+ elsif conf['localtime']
572
+ @timezone = Time.now.strftime('%z')
573
+ @localtime = true
574
+ else
575
+ @timezone = "+0000" # v0.12 assumes UTC without any configuration
576
+ @localtime = false
577
+ end
578
+
579
+ @_timekey = case conf['time_slice_format']
580
+ when /\%S/ then 1
581
+ when /\%M/ then 60
582
+ when /\%H/ then 3600
583
+ when /\%d/ then 86400
584
+ when nil then 86400 # default value of TimeSlicedOutput.time_slice_format is '%Y%m%d'
585
+ else
586
+ raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
587
+ end
588
+ buf_params["timekey"] = @_timekey
589
+
590
+ conf.elements << Fluent::Config::Element.new('buffer', 'time', buf_params, [])
591
+ end
592
+
593
+ super
594
+
595
+ if config_style == :v1
596
+ if @buffer_config.chunk_keys == ['tag']
597
+ raise Fluent::ConfigError, "this plugin '#{self.class}' allows <buffer tag> only"
598
+ end
599
+ end
600
+
601
+ (class << self; self; end).module_eval do
602
+ prepend TimeSliceChunkMixin
603
+ end
604
+ end
605
+
606
+ # Original TimeSlicedOutput#emit doesn't call #format_stream
607
+
608
+ # #format MUST be implemented in plugin
609
+ # #write is also
610
+
611
+ def extract_placeholders(str, metadata)
612
+ raise "BUG: compat plugin does not support extract_placeholders: use newer plugin API"
613
+ end
614
+ end
615
+ end
616
+ end
617
+