fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,168 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buffer/chunk'
3
+
4
+ class BufferChunkTest < Test::Unit::TestCase
5
+ sub_test_case 'blank buffer chunk' do
6
+ test 'has generated unique id, given metadata, created_at and modified_at' do
7
+ meta = Object.new
8
+ chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
9
+ assert{ chunk.unique_id.bytesize == 16 }
10
+ assert{ chunk.metadata.object_id == meta.object_id }
11
+ assert{ chunk.created_at.is_a? Time }
12
+ assert{ chunk.modified_at.is_a? Time }
13
+ assert chunk.unstaged?
14
+ assert !chunk.staged?
15
+ assert !chunk.queued?
16
+ assert !chunk.closed?
17
+ end
18
+
19
+ test 'has many methods for chunks, but not implemented' do
20
+ meta = Object.new
21
+ chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
22
+
23
+ assert chunk.respond_to?(:append)
24
+ assert chunk.respond_to?(:concat)
25
+ assert chunk.respond_to?(:commit)
26
+ assert chunk.respond_to?(:rollback)
27
+ assert chunk.respond_to?(:bytesize)
28
+ assert chunk.respond_to?(:size)
29
+ assert chunk.respond_to?(:length)
30
+ assert chunk.respond_to?(:empty?)
31
+ assert chunk.respond_to?(:read)
32
+ assert chunk.respond_to?(:open)
33
+ assert chunk.respond_to?(:write_to)
34
+ assert chunk.respond_to?(:msgpack_each)
35
+ assert_raise(NotImplementedError){ chunk.append([]) }
36
+ assert_raise(NotImplementedError){ chunk.concat(nil, 0) }
37
+ assert_raise(NotImplementedError){ chunk.commit }
38
+ assert_raise(NotImplementedError){ chunk.rollback }
39
+ assert_raise(NotImplementedError){ chunk.bytesize }
40
+ assert_raise(NotImplementedError){ chunk.size }
41
+ assert_raise(NotImplementedError){ chunk.length }
42
+ assert_raise(NotImplementedError){ chunk.empty? }
43
+ assert_raise(NotImplementedError){ chunk.read }
44
+ assert_raise(NotImplementedError){ chunk.open(){} }
45
+ assert_raise(NotImplementedError){ chunk.write_to(nil) }
46
+ assert_raise(NotImplementedError){ chunk.msgpack_each(){|v| v} }
47
+ end
48
+ end
49
+
50
+ class TestChunk < Fluent::Plugin::Buffer::Chunk
51
+ attr_accessor :data
52
+ def initialize(meta)
53
+ super
54
+ @data = ''
55
+ end
56
+ def size
57
+ @data.size
58
+ end
59
+ def open
60
+ require 'stringio'
61
+ io = StringIO.new(@data)
62
+ yield io
63
+ end
64
+ end
65
+
66
+ sub_test_case 'minimum chunk implements #size and #open' do
67
+ test 'chunk lifecycle' do
68
+ c = TestChunk.new(Object.new)
69
+ assert c.unstaged?
70
+ assert !c.staged?
71
+ assert !c.queued?
72
+ assert !c.closed?
73
+ assert c.writable?
74
+
75
+ c.staged!
76
+
77
+ assert !c.unstaged?
78
+ assert c.staged?
79
+ assert !c.queued?
80
+ assert !c.closed?
81
+ assert c.writable?
82
+
83
+ c.enqueued!
84
+
85
+ assert !c.unstaged?
86
+ assert !c.staged?
87
+ assert c.queued?
88
+ assert !c.closed?
89
+ assert !c.writable?
90
+
91
+ c.close
92
+
93
+ assert !c.unstaged?
94
+ assert !c.staged?
95
+ assert !c.queued?
96
+ assert c.closed?
97
+ assert !c.writable?
98
+ end
99
+
100
+ test 'chunk can be unstaged' do
101
+ c = TestChunk.new(Object.new)
102
+ assert c.unstaged?
103
+ assert !c.staged?
104
+ assert !c.queued?
105
+ assert !c.closed?
106
+ assert c.writable?
107
+
108
+ c.staged!
109
+
110
+ assert !c.unstaged?
111
+ assert c.staged?
112
+ assert !c.queued?
113
+ assert !c.closed?
114
+ assert c.writable?
115
+
116
+ c.unstaged!
117
+
118
+ assert c.unstaged?
119
+ assert !c.staged?
120
+ assert !c.queued?
121
+ assert !c.closed?
122
+ assert c.writable?
123
+
124
+ c.enqueued!
125
+
126
+ assert !c.unstaged?
127
+ assert !c.staged?
128
+ assert c.queued?
129
+ assert !c.closed?
130
+ assert !c.writable?
131
+
132
+ c.close
133
+
134
+ assert !c.unstaged?
135
+ assert !c.staged?
136
+ assert !c.queued?
137
+ assert c.closed?
138
+ assert !c.writable?
139
+ end
140
+
141
+ test 'can respond to #empty? correctly' do
142
+ c = TestChunk.new(Object.new)
143
+ assert_equal 0, c.size
144
+ assert c.empty?
145
+ end
146
+
147
+ test 'can write its contents to io object' do
148
+ c = TestChunk.new(Object.new)
149
+ c.data << "my data\nyour data\n"
150
+ io = StringIO.new
151
+ c.write_to(io)
152
+ assert "my data\nyour data\n", io.to_s
153
+ end
154
+
155
+ test 'can feed objects into blocks with unpacking msgpack' do
156
+ require 'msgpack'
157
+ c = TestChunk.new(Object.new)
158
+ c.data << MessagePack.pack(['my data', 1])
159
+ c.data << MessagePack.pack(['your data', 2])
160
+ ary = []
161
+ c.msgpack_each do |obj|
162
+ ary << obj
163
+ end
164
+ assert_equal ['my data', 1], ary[0]
165
+ assert_equal ['your data', 2], ary[1]
166
+ end
167
+ end
168
+ end
@@ -0,0 +1,771 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buffer/file_chunk'
3
+ require 'fluent/unique_id'
4
+
5
+ require 'fileutils'
6
+ require 'msgpack'
7
+ require 'time'
8
+ require 'timecop'
9
+
10
+ class BufferFileChunkTest < Test::Unit::TestCase
11
+ setup do
12
+ @klass = Fluent::Plugin::Buffer::FileChunk
13
+ @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__)
14
+ FileUtils.rm_r @chunkdir rescue nil
15
+ FileUtils.mkdir_p @chunkdir
16
+ end
17
+ teardown do
18
+ Timecop.return
19
+ end
20
+
21
+ Metadata = Struct.new(:timekey, :tag, :variables)
22
+ def gen_metadata(timekey: nil, tag: nil, variables: nil)
23
+ Metadata.new(timekey, tag, variables)
24
+ end
25
+
26
+ def read_metadata_file(path)
27
+ File.open(path, 'rb'){|f| MessagePack.unpack(f.read, symbolize_keys: true) }
28
+ end
29
+
30
+ def gen_path(path)
31
+ File.join(@chunkdir, path)
32
+ end
33
+
34
+ def gen_test_chunk_id
35
+ require 'time'
36
+ now = Time.parse('2016-04-07 14:31:33 +0900')
37
+ u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`
38
+ u3 = 2979763054 # one of rand(0xffffffff)
39
+ u4 = 438020492 # ditto
40
+ [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')
41
+ # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c"
42
+ end
43
+
44
+ def hex_id(id)
45
+ id.unpack('N*').map{|n| n.to_s(16)}.join
46
+ end
47
+
48
+ sub_test_case 'classmethods' do
49
+ data(
50
+ correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged],
51
+ correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued],
52
+ incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :unknown],
53
+ incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :unknown],
54
+ output_file: ['/mydir/mypath/myfile.20160716.log', :unknown],
55
+ )
56
+ test 'can .assume_chunk_state' do |data|
57
+ path, expected = data
58
+ assert_equal expected, @klass.assume_chunk_state(path)
59
+ end
60
+
61
+ test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do
62
+ assert_equal gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_stage_chunk_path(gen_path("mychunk.*.log"), gen_test_chunk_id)
63
+ assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
64
+ @klass.generate_stage_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id)
65
+ end
66
+ assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
67
+ @klass.generate_stage_chunk_path(gen_path("mychunk.*"), gen_test_chunk_id)
68
+ end
69
+ assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
70
+ @klass.generate_stage_chunk_path(gen_path("*.log"), gen_test_chunk_id)
71
+ end
72
+ end
73
+
74
+ test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do
75
+ assert_equal(
76
+ gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"),
77
+ @klass.generate_queued_chunk_path(gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), gen_test_chunk_id)
78
+ )
79
+ end
80
+
81
+ test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do
82
+ assert_equal(
83
+ gen_path("mychunk.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
84
+ @klass.generate_queued_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id)
85
+ )
86
+ assert_equal(
87
+ gen_path("mychunk.q55555555555555555555555555555555.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
88
+ @klass.generate_queued_chunk_path(gen_path("mychunk.q55555555555555555555555555555555.log"), gen_test_chunk_id)
89
+ )
90
+ end
91
+
92
+ test '.unique_id_from_path recreates unique_id from file path to assume unique_id for v0.12 chunks' do
93
+ assert_equal gen_test_chunk_id, @klass.unique_id_from_path(gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"))
94
+ end
95
+ end
96
+
97
+ sub_test_case 'newly created chunk' do
98
+ setup do
99
+ @chunk_path = File.join(@chunkdir, 'test.*.log')
100
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :create)
101
+ end
102
+
103
+ def gen_chunk_path(prefix, unique_id)
104
+ File.join(@chunkdir, "test.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.log")
105
+ end
106
+
107
+ teardown do
108
+ if @c
109
+ @c.purge rescue nil
110
+ end
111
+ if File.exist? @chunk_path
112
+ File.unlink @chunk_path
113
+ end
114
+ end
115
+
116
+ test 'creates new files for chunk and metadata with specified path & permission' do
117
+ assert{ @c.unique_id.size == 16 }
118
+ assert_equal gen_chunk_path('b', @c.unique_id), @c.path
119
+
120
+ assert File.exist?(gen_chunk_path('b', @c.unique_id))
121
+ assert{ File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(@klass.const_get('FILE_PERMISSION').to_s(8)) }
122
+
123
+ assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta')
124
+ assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(@klass.const_get('FILE_PERMISSION').to_s(8)) }
125
+
126
+ assert_equal :unstaged, @c.state
127
+ assert @c.empty?
128
+ end
129
+
130
+ test 'can #append, #commit and #read it' do
131
+ assert @c.empty?
132
+
133
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
134
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
135
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
136
+ @c.append(data)
137
+ @c.commit
138
+
139
+ content = @c.read
140
+ ds = content.split("\n").select{|d| !d.empty? }
141
+
142
+ assert_equal 2, ds.size
143
+ assert_equal d1, JSON.parse(ds[0])
144
+ assert_equal d2, JSON.parse(ds[1])
145
+
146
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
147
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
148
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
149
+ @c.commit
150
+
151
+ content = @c.read
152
+ ds = content.split("\n").select{|d| !d.empty? }
153
+
154
+ assert_equal 4, ds.size
155
+ assert_equal d1, JSON.parse(ds[0])
156
+ assert_equal d2, JSON.parse(ds[1])
157
+ assert_equal d3, JSON.parse(ds[2])
158
+ assert_equal d4, JSON.parse(ds[3])
159
+ end
160
+
161
+ test 'can #concat, #commit and #read it' do
162
+ assert @c.empty?
163
+
164
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
165
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
166
+ data = [d1.to_json + "\n", d2.to_json + "\n"].join
167
+ @c.concat(data, 2)
168
+ @c.commit
169
+
170
+ content = @c.read
171
+ ds = content.split("\n").select{|d| !d.empty? }
172
+
173
+ assert_equal 2, ds.size
174
+ assert_equal d1, JSON.parse(ds[0])
175
+ assert_equal d2, JSON.parse(ds[1])
176
+
177
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
178
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
179
+ @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
180
+ @c.commit
181
+
182
+ content = @c.read
183
+ ds = content.split("\n").select{|d| !d.empty? }
184
+
185
+ assert_equal 4, ds.size
186
+ assert_equal d1, JSON.parse(ds[0])
187
+ assert_equal d2, JSON.parse(ds[1])
188
+ assert_equal d3, JSON.parse(ds[2])
189
+ assert_equal d4, JSON.parse(ds[3])
190
+ end
191
+
192
+ test 'has its contents in binary (ascii-8bit)' do
193
+ data1 = "aaa bbb ccc".force_encoding('utf-8')
194
+ @c.append([data1])
195
+ @c.commit
196
+ assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }
197
+
198
+ content = @c.read
199
+ assert_equal Encoding::ASCII_8BIT, content.encoding
200
+ end
201
+
202
+ test 'has #bytesize and #size' do
203
+ assert @c.empty?
204
+
205
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
206
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
207
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
208
+ @c.append(data)
209
+
210
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
211
+ assert_equal 2, @c.size
212
+
213
+ @c.commit
214
+
215
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
216
+ assert_equal 2, @c.size
217
+
218
+ first_bytesize = @c.bytesize
219
+
220
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
221
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
222
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
223
+
224
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
225
+ assert_equal 4, @c.size
226
+
227
+ @c.commit
228
+
229
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
230
+ assert_equal 4, @c.size
231
+ end
232
+
233
+ test 'can #rollback to revert non-committed data' do
234
+ assert @c.empty?
235
+
236
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
237
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
238
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
239
+ @c.append(data)
240
+
241
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
242
+ assert_equal 2, @c.size
243
+
244
+ @c.rollback
245
+
246
+ assert @c.empty?
247
+
248
+ assert_equal '', File.open(@c.path, 'rb'){|f| f.read }
249
+
250
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
251
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
252
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
253
+ @c.append(data)
254
+ @c.commit
255
+
256
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
257
+ assert_equal 2, @c.size
258
+
259
+ first_bytesize = @c.bytesize
260
+
261
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
262
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
263
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
264
+
265
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
266
+ assert_equal 4, @c.size
267
+
268
+ @c.rollback
269
+
270
+ assert_equal first_bytesize, @c.bytesize
271
+ assert_equal 2, @c.size
272
+
273
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read }
274
+ end
275
+
276
+ test 'can #rollback to revert non-committed data from #concat' do
277
+ assert @c.empty?
278
+
279
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
280
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
281
+ data = [d1.to_json + "\n", d2.to_json + "\n"].join
282
+ @c.concat(data, 2)
283
+
284
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
285
+ assert_equal 2, @c.size
286
+
287
+ @c.rollback
288
+
289
+ assert @c.empty?
290
+
291
+ assert_equal '', File.open(@c.path, 'rb'){|f| f.read }
292
+
293
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
294
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
295
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
296
+ @c.append(data)
297
+ @c.commit
298
+
299
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
300
+ assert_equal 2, @c.size
301
+
302
+ first_bytesize = @c.bytesize
303
+
304
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
305
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
306
+ @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
307
+
308
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
309
+ assert_equal 4, @c.size
310
+
311
+ @c.rollback
312
+
313
+ assert_equal first_bytesize, @c.bytesize
314
+ assert_equal 2, @c.size
315
+
316
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read }
317
+ end
318
+
319
+ test 'can store its data by #close' do
320
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
321
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
322
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
323
+ @c.append(data)
324
+ @c.commit
325
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
326
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
327
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
328
+ @c.commit
329
+
330
+ content = @c.read
331
+
332
+ unique_id = @c.unique_id
333
+ size = @c.size
334
+ created_at = @c.created_at
335
+ modified_at = @c.modified_at
336
+
337
+ @c.close
338
+
339
+ assert_equal content, File.open(@c.path, 'rb'){|f| f.read }
340
+
341
+ stored_meta = {
342
+ timekey: nil, tag: nil, variables: nil,
343
+ id: unique_id,
344
+ s: size,
345
+ c: created_at.to_i,
346
+ m: modified_at.to_i,
347
+ }
348
+
349
+ assert_equal stored_meta, read_metadata_file(@c.path + '.meta')
350
+ end
351
+
352
+ test 'deletes all data by #purge' do
353
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
354
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
355
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
356
+ @c.append(data)
357
+ @c.commit
358
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
359
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
360
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
361
+ @c.commit
362
+
363
+ @c.purge
364
+
365
+ assert @c.empty?
366
+ assert_equal 0, @c.bytesize
367
+ assert_equal 0, @c.size
368
+
369
+ assert !File.exist?(@c.path)
370
+ assert !File.exist?(@c.path + '.meta')
371
+ end
372
+
373
+ test 'can #open its contents as io' do
374
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
375
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
376
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
377
+ @c.append(data)
378
+ @c.commit
379
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
380
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
381
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
382
+ @c.commit
383
+
384
+ lines = []
385
+ @c.open do |io|
386
+ assert io
387
+ io.readlines.each do |l|
388
+ lines << l
389
+ end
390
+ end
391
+
392
+ assert_equal d1.to_json + "\n", lines[0]
393
+ assert_equal d2.to_json + "\n", lines[1]
394
+ assert_equal d3.to_json + "\n", lines[2]
395
+ assert_equal d4.to_json + "\n", lines[3]
396
+ end
397
+
398
+ test 'can refer system config for file permission' do
399
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
400
+
401
+ chunk_path = File.join(@chunkdir, 'testperm.*.log')
402
+ Fluent::SystemConfig.overwrite_system_config("file_permission" => "600") do
403
+ c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
404
+ assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
405
+ assert{ File.stat(c.path + '.meta').mode.to_s(8).end_with?('600') }
406
+ end
407
+ end
408
+
409
+ test '#write_metadata tries to store metadata on file' do
410
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
411
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
412
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
413
+ @c.append(data)
414
+ @c.commit
415
+
416
+ expected = {
417
+ timekey: nil, tag: nil, variables: nil,
418
+ id: @c.unique_id,
419
+ s: @c.size,
420
+ c: @c.created_at.to_i,
421
+ m: @c.modified_at.to_i,
422
+ }
423
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
424
+
425
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
426
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
427
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
428
+ # append does write_metadata
429
+
430
+ dummy_now = Time.parse('2016-04-07 16:59:59 +0900')
431
+ Timecop.freeze(dummy_now)
432
+ @c.write_metadata
433
+
434
+ expected = {
435
+ timekey: nil, tag: nil, variables: nil,
436
+ id: @c.unique_id,
437
+ s: @c.size,
438
+ c: @c.created_at.to_i,
439
+ m: dummy_now.to_i,
440
+ }
441
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
442
+
443
+ @c.commit
444
+
445
+ expected = {
446
+ timekey: nil, tag: nil, variables: nil,
447
+ id: @c.unique_id,
448
+ s: @c.size,
449
+ c: @c.created_at.to_i,
450
+ m: @c.modified_at.to_i,
451
+ }
452
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
453
+
454
+ content = @c.read
455
+
456
+ unique_id = @c.unique_id
457
+ size = @c.size
458
+ created_at = @c.created_at
459
+ modified_at = @c.modified_at
460
+
461
+ @c.close
462
+
463
+ assert_equal content, File.open(@c.path, 'rb'){|f| f.read }
464
+
465
+ stored_meta = {
466
+ timekey: nil, tag: nil, variables: nil,
467
+ id: unique_id,
468
+ s: size,
469
+ c: created_at.to_i,
470
+ m: modified_at.to_i,
471
+ }
472
+
473
+ assert_equal stored_meta, read_metadata_file(@c.path + '.meta')
474
+ end
475
+ end
476
+
477
+ sub_test_case 'chunk with file for staged chunk' do
478
+ setup do
479
+ @chunk_id = gen_test_chunk_id
480
+ @chunk_path = File.join(@chunkdir, "test_staged.b#{hex_id(@chunk_id)}.log")
481
+ @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log")
482
+
483
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
484
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
485
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
486
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
487
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
488
+ File.open(@chunk_path, 'wb') do |f|
489
+ f.write @d
490
+ end
491
+
492
+ @metadata = {
493
+ timekey: nil, tag: 'testing', variables: {k: "x"},
494
+ id: @chunk_id,
495
+ s: 4,
496
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
497
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
498
+ }
499
+ File.open(@chunk_path + '.meta', 'wb') do |f|
500
+ f.write @metadata.to_msgpack
501
+ end
502
+
503
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)
504
+ end
505
+
506
+ teardown do
507
+ if @c
508
+ @c.purge rescue nil
509
+ end
510
+ [@chunk_path, @chunk_path + '.meta', @enqueued_path, @enqueued_path + '.meta'].each do |path|
511
+ File.unlink path if File.exist? path
512
+ end
513
+ end
514
+
515
+ test 'can load as staged chunk from file with metadata' do
516
+ assert_equal @chunk_path, @c.path
517
+ assert_equal :staged, @c.state
518
+
519
+ assert_nil @c.metadata.timekey
520
+ assert_equal 'testing', @c.metadata.tag
521
+ assert_equal({k: "x"}, @c.metadata.variables)
522
+
523
+ assert_equal 4, @c.size
524
+ assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at
525
+ assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at
526
+
527
+ content = @c.read
528
+ assert_equal @d, content
529
+ end
530
+
531
+ test 'can be enqueued' do
532
+ stage_path = @c.path
533
+ queue_path = @enqueued_path
534
+ assert File.exist?(stage_path)
535
+ assert File.exist?(stage_path + '.meta')
536
+ assert !File.exist?(queue_path)
537
+ assert !File.exist?(queue_path + '.meta')
538
+
539
+ @c.enqueued!
540
+
541
+ assert_equal queue_path, @c.path
542
+
543
+ assert !File.exist?(stage_path)
544
+ assert !File.exist?(stage_path + '.meta')
545
+ assert File.exist?(queue_path)
546
+ assert File.exist?(queue_path + '.meta')
547
+
548
+ assert_nil @c.metadata.timekey
549
+ assert_equal 'testing', @c.metadata.tag
550
+ assert_equal({k: "x"}, @c.metadata.variables)
551
+
552
+ assert_equal 4, @c.size
553
+ assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at
554
+ assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at
555
+
556
+ assert_equal @d, File.open(@c.path, 'rb'){|f| f.read }
557
+ assert_equal @metadata, read_metadata_file(@c.path + '.meta')
558
+ end
559
+
560
+ test '#write_metadata tries to store metadata on file with non-committed data' do
561
+ d5 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
562
+ d5s = d5.to_json + "\n"
563
+ @c.append([d5s])
564
+
565
+ metadata = {
566
+ timekey: nil, tag: 'testing', variables: {k: "x"},
567
+ id: @chunk_id,
568
+ s: 4,
569
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
570
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
571
+ }
572
+ assert_equal metadata, read_metadata_file(@c.path + '.meta')
573
+
574
+ @c.write_metadata
575
+
576
+ metadata = {
577
+ timekey: nil, tag: 'testing', variables: {k: "x"},
578
+ id: @chunk_id,
579
+ s: 5,
580
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
581
+ m: Time.parse('2016-04-07 17:44:38 +0900').to_i,
582
+ }
583
+
584
+ dummy_now = Time.parse('2016-04-07 17:44:38 +0900')
585
+ Timecop.freeze(dummy_now)
586
+ @c.write_metadata
587
+
588
+ assert_equal metadata, read_metadata_file(@c.path + '.meta')
589
+ end
590
+
591
+ test '#file_rename can rename chunk files even in windows, and call callback with file size' do
592
+ data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc"
593
+
594
+ testing_file1 = gen_path('rename1.test')
595
+ testing_file2 = gen_path('rename2.test')
596
+ f = File.open(testing_file1, 'wb', @c.permission)
597
+ f.set_encoding(Encoding::ASCII_8BIT)
598
+ f.sync = true
599
+ f.binmode
600
+ f.write data
601
+ pos = f.pos
602
+
603
+ assert f.binmode?
604
+ assert f.sync
605
+ assert_equal data.bytesize, f.size
606
+
607
+ io = nil
608
+ @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })
609
+ assert io
610
+ if Fluent.windows?
611
+ assert{ f != io }
612
+ else
613
+ assert_equal f, io
614
+ end
615
+ assert_equal Encoding::ASCII_8BIT, io.external_encoding
616
+ assert io.sync
617
+ assert io.binmode?
618
+ assert_equal data.bytesize, io.size
619
+
620
+ assert_equal pos, io.pos
621
+
622
+ assert_equal '', io.read
623
+
624
+ io.rewind
625
+ assert_equal data, io.read
626
+ end
627
+ end
628
+
629
+ sub_test_case 'chunk with file for enqueued chunk' do
630
+ setup do
631
+ @chunk_id = gen_test_chunk_id
632
+ @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log")
633
+
634
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
635
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
636
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
637
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
638
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
639
+ File.open(@enqueued_path, 'wb') do |f|
640
+ f.write @d
641
+ end
642
+
643
+ @dummy_timekey = Time.parse('2016-04-07 17:40:00 +0900').to_i
644
+
645
+ @metadata = {
646
+ timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"},
647
+ id: @chunk_id,
648
+ s: 4,
649
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
650
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
651
+ }
652
+ File.open(@enqueued_path + '.meta', 'wb') do |f|
653
+ f.write @metadata.to_msgpack
654
+ end
655
+
656
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @enqueued_path, :queued)
657
+ end
658
+
659
+ teardown do
660
+ if @c
661
+ @c.purge rescue nil
662
+ end
663
+ [@enqueued_path, @enqueued_path + '.meta'].each do |path|
664
+ File.unlink path if File.exist? path
665
+ end
666
+ end
667
+
668
+ test 'can load as queued chunk (read only) with metadata' do
669
+ assert @c
670
+ assert_equal @chunk_id, @c.unique_id
671
+ assert_equal :queued, @c.state
672
+ assert_equal gen_metadata(timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}), @c.metadata
673
+ assert_equal Time.at(@metadata[:c]), @c.created_at
674
+ assert_equal Time.at(@metadata[:m]), @c.modified_at
675
+ assert_equal @metadata[:s], @c.size
676
+ assert_equal @d.bytesize, @c.bytesize
677
+ assert_equal @d, @c.read
678
+
679
+ assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
680
+ @c.append(["queued chunk is read only"])
681
+ end
682
+ assert_raise IOError do
683
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
684
+ end
685
+ end
686
+ end
687
+
688
+ sub_test_case 'chunk with queued chunk file of v0.12, without metadata' do
689
+ setup do
690
+ @chunk_id = gen_test_chunk_id
691
+ @chunk_path = File.join(@chunkdir, "test_v12.2016040811.q#{hex_id(@chunk_id)}.log")
692
+
693
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
694
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
695
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
696
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
697
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
698
+ File.open(@chunk_path, 'wb') do |f|
699
+ f.write @d
700
+ end
701
+
702
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :queued)
703
+ end
704
+
705
+ teardown do
706
+ if @c
707
+ @c.purge rescue nil
708
+ end
709
+ File.unlink @chunk_path if File.exist? @chunk_path
710
+ end
711
+
712
+ test 'can load as queued chunk from file without metadata' do
713
+ assert @c
714
+ assert_equal :queued, @c.state
715
+ assert_equal @chunk_id, @c.unique_id
716
+ assert_equal gen_metadata, @c.metadata
717
+ assert_equal @d.bytesize, @c.bytesize
718
+ assert_equal 0, @c.size
719
+ assert_equal @d, @c.read
720
+
721
+ assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
722
+ @c.append(["queued chunk is read only"])
723
+ end
724
+ assert_raise IOError do
725
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
726
+ end
727
+ end
728
+ end
729
+
730
+ sub_test_case 'chunk with staged chunk file of v0.12, without metadata' do
731
+ setup do
732
+ @chunk_id = gen_test_chunk_id
733
+ @chunk_path = File.join(@chunkdir, "test_v12.2016040811.b#{hex_id(@chunk_id)}.log")
734
+
735
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
736
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
737
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
738
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
739
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
740
+ File.open(@chunk_path, 'wb') do |f|
741
+ f.write @d
742
+ end
743
+
744
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)
745
+ end
746
+
747
+ teardown do
748
+ if @c
749
+ @c.purge rescue nil
750
+ end
751
+ File.unlink @chunk_path if File.exist? @chunk_path
752
+ end
753
+
754
+ test 'can load as queued chunk from file without metadata even if it was loaded as staged chunk' do
755
+ assert @c
756
+ assert_equal :queued, @c.state
757
+ assert_equal @chunk_id, @c.unique_id
758
+ assert_equal gen_metadata, @c.metadata
759
+ assert_equal @d.bytesize, @c.bytesize
760
+ assert_equal 0, @c.size
761
+ assert_equal @d, @c.read
762
+
763
+ assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
764
+ @c.append(["queued chunk is read only"])
765
+ end
766
+ assert_raise IOError do
767
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
768
+ end
769
+ end
770
+ end
771
+ end