fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,63 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/base'
18
+
19
+ require 'fluent/log'
20
+ require 'fluent/plugin_id'
21
+ require 'fluent/plugin_helper'
22
+
23
+ module Fluent
24
+ module Plugin
25
+ class BareOutput < Base
26
+ # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
27
+ # This output plugin base class is only for meta-output plugins
28
+ # which cannot be implemented on MultiOutput.
29
+ # E.g,: forest, config-expander
30
+
31
+ include PluginId
32
+ include PluginLoggerMixin
33
+ include PluginHelper::Mixin
34
+
35
+ attr_reader :num_errors, :emit_count, :emit_records
36
+
37
+ def process(tag, es)
38
+ raise NotImplementedError, "BUG: output plugins MUST implement this method"
39
+ end
40
+
41
+ def initialize
42
+ super
43
+ @counters_monitor = Monitor.new
44
+ # TODO: well organized counters
45
+ @num_errors = 0
46
+ @emit_count = 0
47
+ @emit_records = 0
48
+ end
49
+
50
+ def emit_sync(tag, es)
51
+ @counters_monitor.synchronize{ @emit_count += 1 }
52
+ begin
53
+ process(tag, es)
54
+ @counters_monitor.synchronize{ @emit_records += es.size }
55
+ rescue
56
+ @counters_monitor.synchronize{ @num_errors += 1 }
57
+ raise
58
+ end
59
+ end
60
+ alias :emit_events :emit_sync
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,130 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/configurable'
19
+ require 'fluent/system_config'
20
+
21
+ module Fluent
22
+ module Plugin
23
+ class Base
24
+ include Configurable
25
+ include SystemConfig::Mixin
26
+
27
+ State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
28
+
29
+ def initialize
30
+ super
31
+ @_state = State.new(false, false, false, false, false, false, false, false, false)
32
+ end
33
+
34
+ def has_router?
35
+ false
36
+ end
37
+
38
+ def configure(conf)
39
+ super
40
+ @_state ||= State.new(false, false, false, false, false, false, false, false, false)
41
+ @_state.configure = true
42
+ self
43
+ end
44
+
45
+ def start
46
+ @_state.start = true
47
+ self
48
+ end
49
+
50
+ def after_start
51
+ @_state.after_start = true
52
+ self
53
+ end
54
+
55
+ def stop
56
+ @_state.stop = true
57
+ self
58
+ end
59
+
60
+ def before_shutdown
61
+ @_state.before_shutdown = true
62
+ self
63
+ end
64
+
65
+ def shutdown
66
+ @_state.shutdown = true
67
+ self
68
+ end
69
+
70
+ def after_shutdown
71
+ @_state.after_shutdown = true
72
+ self
73
+ end
74
+
75
+ def close
76
+ @_state.close = true
77
+ self
78
+ end
79
+
80
+ def terminate
81
+ @_state.terminate = true
82
+ self
83
+ end
84
+
85
+ def configured?
86
+ @_state.configure
87
+ end
88
+
89
+ def started?
90
+ @_state.start
91
+ end
92
+
93
+ def after_started?
94
+ @_state.after_start
95
+ end
96
+
97
+ def stopped?
98
+ @_state.stop
99
+ end
100
+
101
+ def before_shutdown?
102
+ @_state.before_shutdown
103
+ end
104
+
105
+ def shutdown?
106
+ @_state.shutdown
107
+ end
108
+
109
+ def after_shutdown?
110
+ @_state.after_shutdown
111
+ end
112
+
113
+ def closed?
114
+ @_state.close
115
+ end
116
+
117
+ def terminated?
118
+ @_state.terminate
119
+ end
120
+
121
+ def inspect
122
+ # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
123
+ # Original commit comment says that:
124
+ # To emulate normal inspect behavior `ruby -e'o=Object.new;p o;p (o.__id__<<1).to_s(16)'`.
125
+ # https://github.com/ruby/ruby/blob/trunk/gc.c#L788
126
+ "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
127
+ end
128
+ end
129
+ end
130
+ end
@@ -0,0 +1,154 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fileutils'
18
+
19
+ require 'fluent/plugin/buffer'
20
+ require 'fluent/plugin/buffer/file_chunk'
21
+ require 'fluent/system_config'
22
+
23
+ module Fluent
24
+ module Plugin
25
+ class FileBuffer < Fluent::Plugin::Buffer
26
+ Plugin.register_buffer('file', self)
27
+
28
+ include SystemConfig::Mixin
29
+
30
+ DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024 # 256MB
31
+ DEFAULT_TOTAL_LIMIT_SIZE = 64 * 1024 * 1024 * 1024 # 64GB, same with v0.12 (TimeSlicedOutput + buf_file)
32
+
33
+ DIR_PERMISSION = 0755
34
+
35
+ # TODO: buffer_path based on system config
36
+ desc 'The path where buffer chunks are stored.'
37
+ config_param :path, :string
38
+
39
+ config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
40
+ config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
41
+
42
+ config_param :file_permission, :string, default: nil # '0644'
43
+ config_param :dir_permission, :string, default: nil # '0755'
44
+
45
+ ##TODO: Buffer plugin cannot handle symlinks because new API @stage has many writing buffer chunks
46
+ ## re-implement this feature on out_file, w/ enqueue_chunk(or generate_chunk) hook + chunk.path
47
+ # attr_accessor :symlink_path
48
+
49
+ @@buffer_paths = {}
50
+
51
+ def initialize
52
+ super
53
+ @symlink_path = nil
54
+ end
55
+
56
+ def configure(conf)
57
+ super
58
+
59
+ type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
60
+ if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
61
+ type_using_this_path = @@buffer_paths[@path]
62
+ raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
63
+ end
64
+
65
+ @@buffer_paths[@path] = type_of_owner
66
+
67
+ # TODO: create buffer path with plugin_id, under directory specified by system config
68
+ if File.exist?(@path)
69
+ if File.directory?(@path)
70
+ @path = File.join(@path, 'buffer.*.log')
71
+ elsif File.basename(@path).include?('.*.')
72
+ # valid path (buffer.*.log will be ignored)
73
+ elsif File.basename(@path).end_with?('.*')
74
+ @path = @path + '.log'
75
+ else
76
+ # existing file will be ignored
77
+ @path = @path + '.*.log'
78
+ end
79
+ else # path doesn't exist
80
+ if File.basename(@path).include?('.*.')
81
+ # valid path
82
+ elsif File.basename(@path).end_with?('.*')
83
+ @path = @path + '.log'
84
+ else
85
+ # path is handled as directory, and it will be created at #start
86
+ @path = File.join(@path, 'buffer.*.log')
87
+ end
88
+ end
89
+
90
+ unless @dir_permission
91
+ @dir_permission = system_config.dir_permission || DIR_PERMISSION
92
+ end
93
+ end
94
+
95
+ def buffer_path_for_test?
96
+ caller_locations.each do |location|
97
+ # Thread::Backtrace::Location#path returns base filename or absolute path.
98
+ # #absolute_path returns absolute_path always.
99
+ # https://bugs.ruby-lang.org/issues/12159
100
+ if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
101
+ return true
102
+ end
103
+ end
104
+ false
105
+ end
106
+
107
+ def start
108
+ FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
109
+
110
+ super
111
+ end
112
+
113
+ def persistent?
114
+ true
115
+ end
116
+
117
+ def resume
118
+ stage = {}
119
+ queue = []
120
+
121
+ Dir.glob(@path) do |path|
122
+ m = new_metadata() # this metadata will be overwritten by resuming .meta file content
123
+ # so it should not added into @metadata_list for now
124
+ mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)
125
+ if mode == :unknown
126
+ log.debug "uknown state chunk found", path: path
127
+ next
128
+ end
129
+
130
+ chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
131
+ case chunk.state
132
+ when :staged
133
+ stage[chunk.metadata] = chunk
134
+ when :queued
135
+ queue << chunk
136
+ end
137
+ end
138
+
139
+ queue.sort_by!{ |chunk| chunk.modified_at }
140
+
141
+ return stage, queue
142
+ end
143
+
144
+ def generate_chunk(metadata)
145
+ # FileChunk generates real path with unique_id
146
+ if @file_permission
147
+ Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission)
148
+ else
149
+ Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create)
150
+ end
151
+ end
152
+ end
153
+ end
154
+ end
@@ -0,0 +1,34 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/buffer'
18
+ require 'fluent/plugin/buffer/memory_chunk'
19
+
20
+ module Fluent
21
+ module Plugin
22
+ class MemoryBuffer < Fluent::Plugin::Buffer
23
+ Plugin.register_buffer('memory', self)
24
+
25
+ def resume
26
+ return {}, []
27
+ end
28
+
29
+ def generate_chunk(metadata)
30
+ Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,603 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/base'
18
+ require 'fluent/plugin/owned_by_mixin'
19
+ require 'fluent/unique_id'
20
+
21
+ require 'monitor'
22
+
23
+ module Fluent
24
+ module Plugin
25
+ class Buffer < Base
26
+ include OwnedByMixin
27
+ include UniqueId::Mixin
28
+ include MonitorMixin
29
+
30
+ class BufferError < StandardError; end
31
+ class BufferOverflowError < BufferError; end
32
+ class BufferChunkOverflowError < BufferError; end # A record size is larger than chunk size limit
33
+
34
+ MINIMUM_APPEND_ATTEMPT_RECORDS = 10
35
+
36
+ DEFAULT_CHUNK_LIMIT_SIZE = 8 * 1024 * 1024 # 8MB
37
+ DEFAULT_TOTAL_LIMIT_SIZE = 512 * 1024 * 1024 # 512MB, same with v0.12 (BufferedOutput + buf_memory: 64 x 8MB)
38
+
39
+ DEFAULT_CHUNK_FULL_THRESHOLD = 0.95
40
+
41
+ configured_in :buffer
42
+
43
+ # TODO: system total buffer limit size in bytes by SystemConfig
44
+
45
+ config_param :chunk_limit_size, :size, default: DEFAULT_CHUNK_LIMIT_SIZE
46
+ config_param :total_limit_size, :size, default: DEFAULT_TOTAL_LIMIT_SIZE
47
+
48
+ # If user specify this value and (chunk_size * queue_length) is smaller than total_size,
49
+ # then total_size is automatically configured to that value
50
+ config_param :queue_length_limit, :integer, default: nil
51
+
52
+ # optional new limitations
53
+ config_param :chunk_records_limit, :integer, default: nil
54
+
55
+ # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
56
+ config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
57
+
58
+ Metadata = Struct.new(:timekey, :tag, :variables)
59
+
60
+ # for tests
61
+ attr_accessor :stage_size, :queue_size
62
+ attr_reader :stage, :queue, :dequeued, :queued_num
63
+
64
+ def initialize
65
+ super
66
+
67
+ @chunk_limit_size = nil
68
+ @total_limit_size = nil
69
+ @queue_length_limit = nil
70
+ @chunk_records_limit = nil
71
+
72
+ @stage = {} #=> Hash (metadata -> chunk) : not flushed yet
73
+ @queue = [] #=> Array (chunks) : already flushed (not written)
74
+ @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged)
75
+ @queued_num = {} # metadata => int (number of queued chunks)
76
+
77
+ @stage_size = @queue_size = 0
78
+ @metadata_list = [] # keys of @stage
79
+ end
80
+
81
+ def persistent?
82
+ false
83
+ end
84
+
85
+ def configure(conf)
86
+ super
87
+
88
+ unless @queue_length_limit.nil?
89
+ @total_limit_size = @chunk_limit_size * @queue_length_limit
90
+ end
91
+ end
92
+
93
+ def start
94
+ super
95
+
96
+ @stage, @queue = resume
97
+ @stage.each_pair do |metadata, chunk|
98
+ @metadata_list << metadata unless @metadata_list.include?(metadata)
99
+ @stage_size += chunk.bytesize
100
+ end
101
+ @queue.each do |chunk|
102
+ @metadata_list << chunk.metadata unless @metadata_list.include?(chunk.metadata)
103
+ @queued_num[chunk.metadata] ||= 0
104
+ @queued_num[chunk.metadata] += 1
105
+ @queue_size += chunk.bytesize
106
+ end
107
+ log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
108
+ end
109
+
110
+ def close
111
+ super
112
+ synchronize do
113
+ log.debug "closing buffer", instance: self.object_id
114
+ @dequeued.each_pair do |chunk_id, chunk|
115
+ chunk.close
116
+ end
117
+ until @queue.empty?
118
+ @queue.shift.close
119
+ end
120
+ @stage.each_pair do |metadata, chunk|
121
+ chunk.close
122
+ end
123
+ end
124
+ end
125
+
126
+ def terminate
127
+ super
128
+ @dequeued = @stage = @queue = @queued_num = @metadata_list = nil
129
+ @stage_size = @queue_size = 0
130
+ end
131
+
132
+ def storable?
133
+ @total_limit_size > @stage_size + @queue_size
134
+ end
135
+
136
+ ## TODO: for back pressure feature
137
+ # def used?(ratio)
138
+ # @total_size_limit * ratio > @stage_size + @queue_size
139
+ # end
140
+
141
+ def resume
142
+ # return {}, []
143
+ raise NotImplementedError, "Implement this method in child class"
144
+ end
145
+
146
+ def generate_chunk(metadata)
147
+ raise NotImplementedError, "Implement this method in child class"
148
+ end
149
+
150
+ def metadata_list
151
+ synchronize do
152
+ @metadata_list.dup
153
+ end
154
+ end
155
+
156
+ def new_metadata(timekey: nil, tag: nil, variables: nil)
157
+ Metadata.new(timekey, tag, variables)
158
+ end
159
+
160
+ def add_metadata(metadata)
161
+ log.trace "adding metadata", instance: self.object_id, metadata: metadata
162
+ synchronize do
163
+ if i = @metadata_list.index(metadata)
164
+ @metadata_list[i]
165
+ else
166
+ @metadata_list << metadata
167
+ metadata
168
+ end
169
+ end
170
+ end
171
+
172
+ def metadata(timekey: nil, tag: nil, variables: nil)
173
+ meta = new_metadata(timekey: timekey, tag: tag, variables: variables)
174
+ add_metadata(meta)
175
+ end
176
+
177
+ # metadata MUST have consistent object_id for each variation
178
+ # data MUST be Array of serialized events, or EventStream
179
+ # metadata_and_data MUST be a hash of { metadata => data }
180
+ def write(metadata_and_data, format: nil, size: nil, enqueue: false)
181
+ return if metadata_and_data.size < 1
182
+ raise BufferOverflowError, "buffer space has too many data" unless storable?
183
+
184
+ log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size
185
+
186
+ staged_bytesize = 0
187
+ operated_chunks = []
188
+ unstaged_chunks = {} # metadata => [chunk, chunk, ...]
189
+ chunks_to_enqueue = []
190
+
191
+ begin
192
+ metadata_and_data.each do |metadata, data|
193
+ write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
194
+ chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
195
+ operated_chunks << chunk
196
+ if chunk.staged?
197
+ staged_bytesize += adding_bytesize
198
+ elsif chunk.unstaged?
199
+ unstaged_chunks[metadata] ||= []
200
+ unstaged_chunks[metadata] << chunk
201
+ end
202
+ end
203
+ end
204
+
205
+ return if operated_chunks.empty?
206
+
207
+ # Now, this thread acquires many locks of chunks... getting buffer-global lock causes dead lock.
208
+ # Any operations needs buffer-global lock (including enqueueing) should be done after releasing locks.
209
+
210
+ first_chunk = operated_chunks.shift
211
+ # Following commits for other chunks also can finish successfully if the first commit operation
212
+ # finishes without any exceptions.
213
+ # In most cases, #commit just requires very small disk spaces, so major failure reason are
214
+ # permission errors, disk failures and other permanent(fatal) errors.
215
+ begin
216
+ first_chunk.commit
217
+ if enqueue || first_chunk.unstaged? || chunk_size_full?(first_chunk)
218
+ chunks_to_enqueue << first_chunk
219
+ end
220
+ first_chunk.mon_exit
221
+ rescue
222
+ operated_chunks.unshift(first_chunk)
223
+ raise
224
+ end
225
+
226
+ errors = []
227
+ # Buffer plugin estimates there's no serious error cause: will commit for all chunks eigher way
228
+ operated_chunks.each do |chunk|
229
+ begin
230
+ chunk.commit
231
+ if enqueue || chunk.unstaged? || chunk_size_full?(chunk)
232
+ chunks_to_enqueue << chunk
233
+ end
234
+ chunk.mon_exit
235
+ rescue => e
236
+ chunk.rollback
237
+ chunk.mon_exit
238
+ errors << e
239
+ end
240
+ end
241
+
242
+ # All locks about chunks are released.
243
+
244
+ synchronize do
245
+ # At here, staged chunks may be enqueued by other threads.
246
+ @stage_size += staged_bytesize
247
+
248
+ chunks_to_enqueue.each do |c|
249
+ if c.staged? && (enqueue || chunk_size_full?(c))
250
+ m = c.metadata
251
+ enqueue_chunk(m)
252
+ if unstaged_chunks[m]
253
+ u = unstaged_chunks[m].pop
254
+ if u.unstaged? && !chunk_size_full?(u)
255
+ @stage[m] = u.staged!
256
+ @stage_size += u.bytesize
257
+ end
258
+ end
259
+ elsif c.unstaged?
260
+ enqueue_unstaged_chunk(c)
261
+ else
262
+ # previously staged chunk is already enqueued, closed or purged.
263
+ # no problem.
264
+ end
265
+ end
266
+ end
267
+
268
+ operated_chunks.clear if errors.empty?
269
+
270
+ if errors.size > 0
271
+ log.warn "error occurs in committing chunks: only first one raised", errors: errors.map(&:class)
272
+ raise errors.first
273
+ end
274
+ ensure
275
+ operated_chunks.each do |chunk|
276
+ chunk.rollback rescue nil # nothing possible to do for #rollback failure
277
+ if chunk.unstaged?
278
+ chunk.purge rescue nil # to prevent leakage of unstaged chunks
279
+ end
280
+ chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
281
+ end
282
+ end
283
+ end
284
+
285
+ def queued_records
286
+ synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
287
+ end
288
+
289
+ def queued?(metadata=nil)
290
+ synchronize do
291
+ if metadata
292
+ n = @queued_num[metadata]
293
+ n && n.nonzero?
294
+ else
295
+ !@queue.empty?
296
+ end
297
+ end
298
+ end
299
+
300
+ def enqueue_chunk(metadata)
301
+ log.debug "enqueueing chunk", instance: self.object_id, metadata: metadata
302
+ synchronize do
303
+ chunk = @stage.delete(metadata)
304
+ return nil unless chunk
305
+
306
+ chunk.synchronize do
307
+ if chunk.empty?
308
+ chunk.close
309
+ else
310
+ @queue << chunk
311
+ @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
312
+ chunk.enqueued! if chunk.respond_to?(:enqueued!)
313
+ end
314
+ end
315
+ bytesize = chunk.bytesize
316
+ @stage_size -= bytesize
317
+ @queue_size += bytesize
318
+ end
319
+ nil
320
+ end
321
+
322
+ def enqueue_unstaged_chunk(chunk)
323
+ log.debug "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
324
+ synchronize do
325
+ chunk.synchronize do
326
+ metadata = chunk.metadata
327
+ @queue << chunk
328
+ @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
329
+ chunk.enqueued! if chunk.respond_to?(:enqueued!)
330
+ end
331
+ @queue_size += chunk.bytesize
332
+ end
333
+ end
334
+
335
+ def enqueue_all
336
+ log.debug "enqueueing all chunks in buffer", instance: self.object_id
337
+ synchronize do
338
+ if block_given?
339
+ @stage.keys.each do |metadata|
340
+ chunk = @stage[metadata]
341
+ v = yield metadata, chunk
342
+ enqueue_chunk(metadata) if v
343
+ end
344
+ else
345
+ @stage.keys.each do |metadata|
346
+ enqueue_chunk(metadata)
347
+ end
348
+ end
349
+ end
350
+ end
351
+
352
+ def dequeue_chunk
353
+ return nil if @queue.empty?
354
+ log.debug "dequeueing a chunk", instance: self.object_id
355
+ synchronize do
356
+ chunk = @queue.shift
357
+
358
+ # this buffer is dequeued by other thread just before "synchronize" in this thread
359
+ return nil unless chunk
360
+
361
+ @dequeued[chunk.unique_id] = chunk
362
+ @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
363
+ log.debug "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
364
+ chunk
365
+ end
366
+ end
367
+
368
+ def takeback_chunk(chunk_id)
369
+ log.debug "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
370
+ synchronize do
371
+ chunk = @dequeued.delete(chunk_id)
372
+ return false unless chunk # already purged by other thread
373
+ @queue.unshift(chunk)
374
+ log.debug "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
375
+ @queued_num[chunk.metadata] += 1 # BUG if nil
376
+ end
377
+ true
378
+ end
379
+
380
+ def purge_chunk(chunk_id)
381
+ synchronize do
382
+ chunk = @dequeued.delete(chunk_id)
383
+ return nil unless chunk # purged by other threads
384
+
385
+ metadata = chunk.metadata
386
+ log.debug "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
387
+ begin
388
+ bytesize = chunk.bytesize
389
+ chunk.purge
390
+ @queue_size -= bytesize
391
+ rescue => e
392
+ log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
393
+ log.error_backtrace
394
+ end
395
+
396
+ if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
397
+ @metadata_list.delete(metadata)
398
+ end
399
+ log.debug "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
400
+ end
401
+ nil
402
+ end
403
+
404
+ def clear_queue!
405
+ log.debug "clearing queue", instance: self.object_id
406
+ synchronize do
407
+ until @queue.empty?
408
+ begin
409
+ q = @queue.shift
410
+ log.debug("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
411
+ q.purge
412
+ rescue => e
413
+ log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e
414
+ log.error_backtrace
415
+ end
416
+ end
417
+ @queue_size = 0
418
+ end
419
+ end
420
+
421
+ def chunk_size_over?(chunk)
422
+ chunk.bytesize > @chunk_limit_size || (@chunk_records_limit && chunk.size > @chunk_records_limit)
423
+ end
424
+
425
+ def chunk_size_full?(chunk)
426
+ chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_records_limit && chunk.size >= @chunk_records_limit * @chunk_full_threshold)
427
+ end
428
+
429
+ class ShouldRetry < StandardError; end
430
+
431
+ # write once into a chunk
432
+ # 1. append whole data into existing chunk
433
+ # 2. commit it & return unless chunk_size_over?
434
+ # 3. enqueue existing chunk & retry whole method if chunk was not empty
435
+ # 4. go to step_by_step writing
436
+
437
+ def write_once(metadata, data, format: nil, size: nil, &block)
438
+ return if data.empty?
439
+
440
+ stored = false
441
+ adding_bytesize = nil
442
+
443
+ chunk = synchronize { @stage[metadata] ||= generate_chunk(metadata).staged! }
444
+ enqueue_chunk_before_retry = false
445
+ chunk.synchronize do
446
+ # retry this method if chunk is already queued (between getting chunk and entering critical section)
447
+ raise ShouldRetry unless chunk.staged?
448
+
449
+ empty_chunk = chunk.empty?
450
+
451
+ original_bytesize = chunk.bytesize
452
+ begin
453
+ if format
454
+ serialized = format.call(data)
455
+ chunk.concat(serialized, size ? size.call : data.size)
456
+ else
457
+ chunk.append(data)
458
+ end
459
+ adding_bytesize = chunk.bytesize - original_bytesize
460
+
461
+ if chunk_size_over?(chunk)
462
+ if format && empty_chunk
463
+ log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
464
+ end
465
+ chunk.rollback
466
+
467
+ if format && !empty_chunk
468
+ # Event streams should be appended into a chunk at once
469
+ # as far as possible, to improve performance of formatting.
470
+ # Event stream may be a MessagePackEventStream. We don't want to split it into
471
+ # 2 or more chunks (except for a case that the event stream is larger than chunk limit).
472
+ enqueue_chunk_before_retry = true
473
+ raise ShouldRetry
474
+ end
475
+ else
476
+ stored = true
477
+ end
478
+ rescue
479
+ chunk.rollback
480
+ raise
481
+ end
482
+
483
+ if stored
484
+ block.call(chunk, adding_bytesize)
485
+ end
486
+ end
487
+
488
+ unless stored
489
+ # try step-by-step appending if data can't be stored into existing a chunk in non-bulk mode
490
+ #
491
+ # 1/10 size of original event stream (splits_count == 10) seems enough small
492
+ # to try emitting events into existing chunk.
493
+ # it does not matter to split event stream into very small splits, because chunks have less
494
+ # overhead to write data many times (even about file buffer chunks).
495
+ write_step_by_step(metadata, data, format, 10, &block)
496
+ end
497
+ rescue ShouldRetry
498
+ enqueue_chunk(metadata) if enqueue_chunk_before_retry
499
+ retry
500
+ end
501
+
502
+ # EventStream can be split into many streams
503
+ # because (es1 + es2).to_msgpack_stream == es1.to_msgpack_stream + es2.to_msgpack_stream
504
+
505
+ # 1. split event streams into many (10 -> 100 -> 1000 -> ...) chunks
506
+ # 2. append splits into the staged chunks as much as possible
507
+ # 3. create unstaged chunk and append rest splits -> repeat it for all splits
508
+
509
+ def write_step_by_step(metadata, data, format, splits_count, &block)
510
+ splits = []
511
+ if splits_count > data.size
512
+ splits_count = data.size
513
+ end
514
+ slice_size = if data.size % splits_count == 0
515
+ data.size / splits_count
516
+ else
517
+ data.size / (splits_count - 1)
518
+ end
519
+ slice_origin = 0
520
+ while slice_origin < data.size
521
+ splits << data.slice(slice_origin, slice_size)
522
+ slice_origin += slice_size
523
+ end
524
+
525
+ # This method will append events into the staged chunk at first.
526
+ # Then, will generate chunks not staged (not queued) to append rest data.
527
+ staged_chunk_used = false
528
+ modified_chunks = []
529
+ get_next_chunk = ->(){
530
+ c = if staged_chunk_used
531
+ # Staging new chunk here is bad idea:
532
+ # Recovering whole state including newly staged chunks is much harder than current implementation.
533
+ generate_chunk(metadata)
534
+ else
535
+ synchronize{ @stage[metadata] ||= generate_chunk(metadata).staged! }
536
+ end
537
+ modified_chunks << c
538
+ c
539
+ }
540
+
541
+ writing_splits_index = 0
542
+ enqueue_chunk_before_retry = false
543
+
544
+ while writing_splits_index < splits.size
545
+ chunk = get_next_chunk.call
546
+ chunk.synchronize do
547
+ raise ShouldRetry unless chunk.writable?
548
+ staged_chunk_used = true if chunk.staged?
549
+
550
+ original_bytesize = chunk.bytesize
551
+ begin
552
+ while writing_splits_index < splits.size
553
+ split = splits[writing_splits_index]
554
+ if format
555
+ chunk.concat(format.call(split), split.size)
556
+ else
557
+ chunk.append(split)
558
+ end
559
+
560
+ if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
561
+ chunk.rollback
562
+
563
+ if split.size == 1 && original_bytesize == 0
564
+ big_record_size = format ? format.call(split).bytesize : split.first.bytesize
565
+ raise BufferChunkOverflowError, "a #{big_record_size}bytes record is larger than buffer chunk limit size"
566
+ end
567
+
568
+ if chunk_size_full?(chunk) || split.size == 1
569
+ enqueue_chunk_before_retry = true
570
+ else
571
+ splits_count *= 10
572
+ end
573
+
574
+ raise ShouldRetry
575
+ end
576
+
577
+ writing_splits_index += 1
578
+
579
+ if chunk_size_full?(chunk)
580
+ break
581
+ end
582
+ end
583
+ rescue
584
+ chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
585
+ raise
586
+ end
587
+
588
+ block.call(chunk, chunk.bytesize - original_bytesize)
589
+ end
590
+ end
591
+ rescue ShouldRetry
592
+ modified_chunks.each do |mc|
593
+ mc.rollback rescue nil
594
+ if mc.unstaged?
595
+ mc.purge rescue nil
596
+ end
597
+ end
598
+ enqueue_chunk(metadata) if enqueue_chunk_before_retry
599
+ retry
600
+ end
601
+ end
602
+ end
603
+ end