fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,160 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/buffer'
18
+ require 'fluent/unique_id'
19
+ require 'fluent/event'
20
+
21
+ require 'monitor'
22
+
23
+ module Fluent
24
+ module Plugin
25
+ class Buffer # fluent/plugin/buffer is alread loaded
26
+ class Chunk
27
+ include MonitorMixin
28
+ include UniqueId::Mixin
29
+ include ChunkMessagePackEventStreamer
30
+
31
+ # Chunks has 2 part:
32
+ # * metadata: contains metadata which should be restored after resume (if possible)
33
+ # v: {key=>value,key=>value,...} (optional)
34
+ # t: tag as string (optional)
35
+ # k: time slice key (optional)
36
+ #
37
+ # id: unique_id of chunk (*)
38
+ # s: size (number of events in chunk) (*)
39
+ # c: created_at as unix time (*)
40
+ # m: modified_at as unix time (*)
41
+ # (*): fields automatically injected by chunk itself
42
+ # * data: binary data, combined records represented as String, maybe compressed
43
+
44
+ # NOTE: keys of metadata are named with a single letter
45
+ # to decread bytesize of metadata I/O
46
+
47
+ # TODO: CompressedPackedMessage of forward protocol?
48
+
49
+ def initialize(metadata)
50
+ super()
51
+ @unique_id = generate_unique_id
52
+ @metadata = metadata
53
+
54
+ # state: unstaged/staged/queued/closed
55
+ @state = :unstaged
56
+
57
+ @size = 0
58
+ @created_at = Time.now
59
+ @modified_at = Time.now
60
+ end
61
+
62
+ attr_reader :unique_id, :metadata, :created_at, :modified_at, :state
63
+
64
+ # data is array of formatted record string
65
+ def append(data)
66
+ adding = ''.b
67
+ data.each do |d|
68
+ adding << d.b
69
+ end
70
+ concat(adding, data.size)
71
+ end
72
+
73
+ # for event streams which is packed or zipped (and we want not to unpack/uncompress)
74
+ def concat(bulk, records)
75
+ raise NotImplementedError, "Implement this method in child class"
76
+ end
77
+
78
+ def commit
79
+ raise NotImplementedError, "Implement this method in child class"
80
+ end
81
+
82
+ def rollback
83
+ raise NotImplementedError, "Implement this method in child class"
84
+ end
85
+
86
+ def bytesize
87
+ raise NotImplementedError, "Implement this method in child class"
88
+ end
89
+
90
+ def size
91
+ raise NotImplementedError, "Implement this method in child class"
92
+ end
93
+ alias :length :size
94
+
95
+ def empty?
96
+ size == 0
97
+ end
98
+
99
+ def writable?
100
+ @state == :staged || @state == :unstaged
101
+ end
102
+
103
+ def unstaged?
104
+ @state == :unstaged
105
+ end
106
+
107
+ def staged?
108
+ @state == :staged
109
+ end
110
+
111
+ def queued?
112
+ @state == :queued
113
+ end
114
+
115
+ def closed?
116
+ @state == :closed
117
+ end
118
+
119
+ def staged!
120
+ @state = :staged
121
+ self
122
+ end
123
+
124
+ def unstaged!
125
+ @state = :unstaged
126
+ self
127
+ end
128
+
129
+ def enqueued!
130
+ @state = :queued
131
+ self
132
+ end
133
+
134
+ def close
135
+ @state = :closed
136
+ self
137
+ end
138
+
139
+ def purge
140
+ @state = :closed
141
+ self
142
+ end
143
+
144
+ def read
145
+ raise NotImplementedError, "Implement this method in child class"
146
+ end
147
+
148
+ def open(&block)
149
+ raise NotImplementedError, "Implement this method in child class"
150
+ end
151
+
152
+ def write_to(io)
153
+ open do |i|
154
+ IO.copy_stream(i, io)
155
+ end
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,323 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/buffer/chunk'
18
+ require 'fluent/unique_id'
19
+ require 'fluent/msgpack_factory'
20
+
21
+ module Fluent
22
+ module Plugin
23
+ class Buffer
24
+ class FileChunk < Chunk
25
+ ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log
26
+ ### buffer chunk path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log
27
+ ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta
28
+
29
+ # NOTE: Old style buffer path of time sliced output plugins had a part of key: prefix.20150414.b513b61...suffix
30
+ # But this part is not used now for any purpose. (Now metadata is used instead.)
31
+
32
+ # state: b/q - 'b'(on stage, compatible with v0.12), 'q'(enqueued)
33
+ # path_prefix: path prefix string, ended with '.'
34
+ # path_suffix: path suffix string, like '.log' (or any other user specified)
35
+
36
+ include SystemConfig::Mixin
37
+ include MessagePackFactory::Mixin
38
+
39
+ FILE_PERMISSION = 0644
40
+
41
+ attr_reader :path, :permission
42
+
43
+ def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION)
44
+ super(metadata)
45
+ @permission = perm
46
+ @bytesize = @size = @adding_bytes = @adding_size = 0
47
+ @meta = nil
48
+
49
+ case mode
50
+ when :create then create_new_chunk(path, perm)
51
+ when :staged then load_existing_staged_chunk(path)
52
+ when :queued then load_existing_enqueued_chunk(path)
53
+ else
54
+ raise ArgumentError, "Invalid file chunk mode: #{mode}"
55
+ end
56
+ end
57
+
58
+ def concat(bulk, bulk_size)
59
+ raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
60
+
61
+ bulk.force_encoding(Encoding::ASCII_8BIT)
62
+ @chunk.write bulk
63
+ @adding_bytes += bulk.bytesize
64
+ @adding_size += bulk_size
65
+ true
66
+ end
67
+
68
+ def commit
69
+ write_metadata # this should be at first: of course, this operation may fail
70
+
71
+ @commit_position = @chunk.pos
72
+ @size += @adding_size
73
+ @bytesize += @adding_bytes
74
+ @adding_bytes = @adding_size = 0
75
+ @modified_at = Time.now
76
+
77
+ true
78
+ end
79
+
80
+ def rollback
81
+ if @chunk.pos != @commit_position
82
+ @chunk.seek(@commit_position, IO::SEEK_SET)
83
+ @chunk.truncate(@commit_position)
84
+ end
85
+ @adding_bytes = @adding_size = 0
86
+ true
87
+ end
88
+
89
+ def bytesize
90
+ @bytesize + @adding_bytes
91
+ end
92
+
93
+ def size
94
+ @size + @adding_size
95
+ end
96
+
97
+ def empty?
98
+ @bytesize == 0
99
+ end
100
+
101
+ def enqueued!
102
+ return unless self.staged?
103
+
104
+ new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)
105
+ new_meta_path = new_chunk_path + '.meta'
106
+
107
+ write_metadata(update: false) # re-write metadata w/ finalized records
108
+
109
+ file_rename(@chunk, @path, new_chunk_path, ->(new_io){ @chunk = new_io })
110
+ @path = new_chunk_path
111
+
112
+ file_rename(@meta, @meta_path, new_meta_path, ->(new_io){ @meta = new_io })
113
+ @meta_path = new_meta_path
114
+
115
+ super
116
+ end
117
+
118
+ def close
119
+ super
120
+ size = @chunk.size
121
+ @chunk.close
122
+ @meta.close if @meta # meta may be missing if chunk is queued at first
123
+ if size == 0
124
+ File.unlink(@path, @meta_path)
125
+ end
126
+ end
127
+
128
+ def purge
129
+ super
130
+ @chunk.close
131
+ @meta.close if @meta
132
+ @bytesize = @size = @adding_bytes = @adding_size = 0
133
+ File.unlink(@path, @meta_path)
134
+ end
135
+
136
+ def read
137
+ @chunk.seek(0, IO::SEEK_SET)
138
+ @chunk.read
139
+ end
140
+
141
+ def open(&block)
142
+ @chunk.seek(0, IO::SEEK_SET)
143
+ val = yield @chunk
144
+ @chunk.seek(0, IO::SEEK_END) if self.staged?
145
+ val
146
+ end
147
+
148
+ def self.assume_chunk_state(path)
149
+ if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
150
+ $1 == 'b' ? :staged : :queued
151
+ else
152
+ # files which matches to glob of buffer file pattern
153
+ # it includes files which are created by out_file
154
+ :unknown
155
+ end
156
+ end
157
+
158
+ def self.generate_stage_chunk_path(path, unique_id)
159
+ pos = path.index('.*.')
160
+ raise "BUG: buffer chunk path on stage MUST have '.*.'" unless pos
161
+
162
+ prefix = path[0...pos]
163
+ suffix = path[(pos+3)..-1]
164
+
165
+ chunk_id = Fluent::UniqueId.hex(unique_id)
166
+ state = 'b'
167
+ "#{prefix}.#{state}#{chunk_id}.#{suffix}"
168
+ end
169
+
170
+ def self.generate_queued_chunk_path(path, unique_id)
171
+ chunk_id = Fluent::UniqueId.hex(unique_id)
172
+ if path.index(".b#{chunk_id}.")
173
+ path.sub(".b#{chunk_id}.", ".q#{chunk_id}.")
174
+ else # for unexpected cases (ex: users rename files while opened by fluentd)
175
+ path + ".q#{chunk_id}.chunk"
176
+ end
177
+ end
178
+
179
+ # used only for queued v0.12 buffer path
180
+ def self.unique_id_from_path(path)
181
+ if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
182
+ return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')
183
+ end
184
+ nil
185
+ end
186
+
187
+ def restore_metadata(bindata)
188
+ data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
189
+
190
+ now = Time.now
191
+
192
+ @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
193
+ @size = data[:s] || 0
194
+ @created_at = Time.at(data.fetch(:c, now.to_i))
195
+ @modified_at = Time.at(data.fetch(:m, now.to_i))
196
+
197
+ @metadata.timekey = data[:timekey]
198
+ @metadata.tag = data[:tag]
199
+ @metadata.variables = data[:variables]
200
+ end
201
+
202
+ def restore_metadata_partially(chunk)
203
+ @unique_id = self.class.unique_id_from_path(chunk.path) || @unique_id
204
+ @size = 0
205
+ @created_at = chunk.ctime # birthtime isn't supported on Windows (and Travis?)
206
+ @modified_at = chunk.mtime
207
+
208
+ @metadata.timekey = nil
209
+ @metadata.tag = nil
210
+ @metadata.variables = nil
211
+ end
212
+
213
+ def write_metadata(update: true)
214
+ data = @metadata.to_h.merge({
215
+ id: @unique_id,
216
+ s: (update ? @size + @adding_size : @size),
217
+ c: @created_at.to_i,
218
+ m: (update ? Time.now : @modified_at).to_i,
219
+ })
220
+ @meta.seek(0, IO::SEEK_SET)
221
+ @meta.truncate(0)
222
+ @meta.write(msgpack_packer.pack(data))
223
+ end
224
+
225
+ def file_rename(file, old_path, new_path, callback=nil)
226
+ pos = file.pos
227
+ if Fluent.windows?
228
+ file.close
229
+ File.rename(old_path, new_path)
230
+ file = File.open(new_path, 'rb', @permission)
231
+ else
232
+ File.rename(old_path, new_path)
233
+ file.reopen(new_path, 'rb')
234
+ end
235
+ file.set_encoding(Encoding::ASCII_8BIT)
236
+ file.sync = true
237
+ file.binmode
238
+ file.pos = pos
239
+ callback.call(file) if callback
240
+ end
241
+
242
+ def create_new_chunk(path, perm)
243
+ @path = self.class.generate_stage_chunk_path(path, @unique_id)
244
+ @meta_path = @path + '.meta'
245
+ @chunk = File.open(@path, 'wb+', perm)
246
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
247
+ @chunk.sync = true
248
+ @chunk.binmode
249
+ @meta = File.open(@meta_path, 'wb', perm)
250
+ @meta.set_encoding(Encoding::ASCII_8BIT)
251
+ @meta.sync = true
252
+ @meta.binmode
253
+
254
+ @state = :unstaged
255
+ @bytesize = 0
256
+ @commit_position = @chunk.pos # must be 0
257
+ @adding_bytes = 0
258
+ @adding_size = 0
259
+ end
260
+
261
+ def load_existing_staged_chunk(path)
262
+ @path = path
263
+ @meta_path = @path + '.meta'
264
+
265
+ @meta = nil
266
+ # staging buffer chunk without metadata is classic buffer chunk file
267
+ # and it should be enqueued immediately
268
+ if File.exist?(@meta_path)
269
+ @chunk = File.open(@path, 'rb+')
270
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
271
+ @chunk.sync = true
272
+ @chunk.seek(0, IO::SEEK_END)
273
+ @chunk.binmode
274
+
275
+ @meta = File.open(@meta_path, 'rb+')
276
+ @meta.set_encoding(Encoding::ASCII_8BIT)
277
+ @meta.sync = true
278
+ @meta.binmode
279
+ restore_metadata(@meta.read)
280
+ @meta.seek(0, IO::SEEK_SET)
281
+
282
+ @state = :staged
283
+ @bytesize = @chunk.size
284
+ @commit_position = @chunk.pos
285
+ @adding_bytes = 0
286
+ @adding_size = 0
287
+ else
288
+ # classic buffer chunk - read only chunk
289
+ @chunk = File.open(@path, 'rb')
290
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
291
+ @chunk.binmode
292
+ @chunk.seek(0, IO::SEEK_SET)
293
+ @state = :queued
294
+ @bytesize = @chunk.size
295
+
296
+ restore_metadata_partially(@chunk)
297
+
298
+ @commit_position = @chunk.size
299
+ @unique_id = self.class.unique_id_from_path(@path) || @unique_id
300
+ end
301
+ end
302
+
303
+ def load_existing_enqueued_chunk(path)
304
+ @path = path
305
+ @chunk = File.open(@path, 'rb')
306
+ @chunk.set_encoding(Encoding::ASCII_8BIT)
307
+ @chunk.binmode
308
+ @chunk.seek(0, IO::SEEK_SET)
309
+ @bytesize = @chunk.size
310
+ @commit_position = @chunk.size
311
+
312
+ @meta_path = @path + '.meta'
313
+ if File.readable?(@meta_path)
314
+ restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })
315
+ else
316
+ restore_metadata_partially(@chunk)
317
+ end
318
+ @state = :queued
319
+ end
320
+ end
321
+ end
322
+ end
323
+ end