fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,871 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buffer/file_chunk'
3
+ require 'fluent/plugin/compressable'
4
+ require 'fluent/unique_id'
5
+
6
+ require 'fileutils'
7
+ require 'msgpack'
8
+ require 'time'
9
+ require 'timecop'
10
+
11
+ class BufferFileChunkTest < Test::Unit::TestCase
12
+ include Fluent::Plugin::Compressable
13
+
14
+ setup do
15
+ @klass = Fluent::Plugin::Buffer::FileChunk
16
+ @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__)
17
+ FileUtils.rm_r @chunkdir rescue nil
18
+ FileUtils.mkdir_p @chunkdir
19
+ end
20
+ teardown do
21
+ Timecop.return
22
+ end
23
+
24
+ Metadata = Fluent::Plugin::Buffer::Metadata
25
+
26
+ def gen_metadata(timekey: nil, tag: nil, variables: nil)
27
+ Metadata.new(timekey, tag, variables)
28
+ end
29
+
30
+ def read_metadata_file(path)
31
+ File.open(path, 'rb') do |f|
32
+ chunk = f.read
33
+ if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data(4)
34
+ return nil
35
+ end
36
+
37
+ data = nil
38
+ if chunk.slice(0, 2) == Fluent::Plugin::Buffer::FileChunk::BUFFER_HEADER
39
+ size = chunk.slice(2, 4).unpack('N').first
40
+ if size
41
+ data = MessagePack.unpack(chunk.slice(6, size), symbolize_keys: true)
42
+ end
43
+ else
44
+ # old type
45
+ data = MessagePack.unpack(chunk, symbolize_keys: true)
46
+ end
47
+
48
+ data
49
+ end
50
+ end
51
+
52
+ def gen_path(path)
53
+ File.join(@chunkdir, path)
54
+ end
55
+
56
+ def gen_test_chunk_id
57
+ require 'time'
58
+ now = Time.parse('2016-04-07 14:31:33 +0900')
59
+ u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`
60
+ u3 = 2979763054 # one of rand(0xffffffff)
61
+ u4 = 438020492 # ditto
62
+ [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')
63
+ # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c"
64
+ end
65
+
66
+ def hex_id(id)
67
+ id.unpack('N*').map{|n| n.to_s(16)}.join
68
+ end
69
+
70
+ sub_test_case 'classmethods' do
71
+ data(
72
+ correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged],
73
+ correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued],
74
+ incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :unknown],
75
+ incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :unknown],
76
+ output_file: ['/mydir/mypath/myfile.20160716.log', :unknown],
77
+ )
78
+ test 'can .assume_chunk_state' do |data|
79
+ path, expected = data
80
+ assert_equal expected, @klass.assume_chunk_state(path)
81
+ end
82
+
83
+ test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do
84
+ assert_equal gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_stage_chunk_path(gen_path("mychunk.*.log"), gen_test_chunk_id)
85
+ assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
86
+ @klass.generate_stage_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id)
87
+ end
88
+ assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
89
+ @klass.generate_stage_chunk_path(gen_path("mychunk.*"), gen_test_chunk_id)
90
+ end
91
+ assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
92
+ @klass.generate_stage_chunk_path(gen_path("*.log"), gen_test_chunk_id)
93
+ end
94
+ end
95
+
96
+ test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do
97
+ assert_equal(
98
+ gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"),
99
+ @klass.generate_queued_chunk_path(gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), gen_test_chunk_id)
100
+ )
101
+ end
102
+
103
+ test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do
104
+ assert_equal(
105
+ gen_path("mychunk.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
106
+ @klass.generate_queued_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id)
107
+ )
108
+ assert_equal(
109
+ gen_path("mychunk.q55555555555555555555555555555555.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
110
+ @klass.generate_queued_chunk_path(gen_path("mychunk.q55555555555555555555555555555555.log"), gen_test_chunk_id)
111
+ )
112
+ end
113
+
114
+ test '.unique_id_from_path recreates unique_id from file path to assume unique_id for v0.12 chunks' do
115
+ assert_equal gen_test_chunk_id, @klass.unique_id_from_path(gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"))
116
+ end
117
+ end
118
+
119
+ sub_test_case 'newly created chunk' do
120
+ setup do
121
+ @chunk_path = File.join(@chunkdir, 'test.*.log')
122
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :create)
123
+ end
124
+
125
+ def gen_chunk_path(prefix, unique_id)
126
+ File.join(@chunkdir, "test.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.log")
127
+ end
128
+
129
+ teardown do
130
+ if @c
131
+ @c.purge rescue nil
132
+ end
133
+ if File.exist? @chunk_path
134
+ File.unlink @chunk_path
135
+ end
136
+ end
137
+
138
+ test 'creates new files for chunk and metadata with specified path & permission' do
139
+ assert{ @c.unique_id.size == 16 }
140
+ assert_equal gen_chunk_path('b', @c.unique_id), @c.path
141
+
142
+ assert File.exist?(gen_chunk_path('b', @c.unique_id))
143
+ assert{ File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }
144
+
145
+ assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta')
146
+ assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }
147
+
148
+ assert_equal :unstaged, @c.state
149
+ assert @c.empty?
150
+ end
151
+
152
+ test 'can #append, #commit and #read it' do
153
+ assert @c.empty?
154
+
155
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
156
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
157
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
158
+ @c.append(data)
159
+ @c.commit
160
+
161
+ content = @c.read
162
+ ds = content.split("\n").select{|d| !d.empty? }
163
+
164
+ assert_equal 2, ds.size
165
+ assert_equal d1, JSON.parse(ds[0])
166
+ assert_equal d2, JSON.parse(ds[1])
167
+
168
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
169
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
170
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
171
+ @c.commit
172
+
173
+ content = @c.read
174
+ ds = content.split("\n").select{|d| !d.empty? }
175
+
176
+ assert_equal 4, ds.size
177
+ assert_equal d1, JSON.parse(ds[0])
178
+ assert_equal d2, JSON.parse(ds[1])
179
+ assert_equal d3, JSON.parse(ds[2])
180
+ assert_equal d4, JSON.parse(ds[3])
181
+ end
182
+
183
+ test 'can #concat, #commit and #read it' do
184
+ assert @c.empty?
185
+
186
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
187
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
188
+ data = [d1.to_json + "\n", d2.to_json + "\n"].join
189
+ @c.concat(data, 2)
190
+ @c.commit
191
+
192
+ content = @c.read
193
+ ds = content.split("\n").select{|d| !d.empty? }
194
+
195
+ assert_equal 2, ds.size
196
+ assert_equal d1, JSON.parse(ds[0])
197
+ assert_equal d2, JSON.parse(ds[1])
198
+
199
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
200
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
201
+ @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
202
+ @c.commit
203
+
204
+ content = @c.read
205
+ ds = content.split("\n").select{|d| !d.empty? }
206
+
207
+ assert_equal 4, ds.size
208
+ assert_equal d1, JSON.parse(ds[0])
209
+ assert_equal d2, JSON.parse(ds[1])
210
+ assert_equal d3, JSON.parse(ds[2])
211
+ assert_equal d4, JSON.parse(ds[3])
212
+ end
213
+
214
+ test 'has its contents in binary (ascii-8bit)' do
215
+ data1 = "aaa bbb ccc".force_encoding('utf-8')
216
+ @c.append([data1])
217
+ @c.commit
218
+ assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }
219
+
220
+ content = @c.read
221
+ assert_equal Encoding::ASCII_8BIT, content.encoding
222
+ end
223
+
224
+ test 'has #bytesize and #size' do
225
+ assert @c.empty?
226
+
227
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
228
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
229
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
230
+ @c.append(data)
231
+
232
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
233
+ assert_equal 2, @c.size
234
+
235
+ @c.commit
236
+
237
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
238
+ assert_equal 2, @c.size
239
+
240
+ first_bytesize = @c.bytesize
241
+
242
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
243
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
244
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
245
+
246
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
247
+ assert_equal 4, @c.size
248
+
249
+ @c.commit
250
+
251
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
252
+ assert_equal 4, @c.size
253
+ end
254
+
255
+ test 'can #rollback to revert non-committed data' do
256
+ assert @c.empty?
257
+
258
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
259
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
260
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
261
+ @c.append(data)
262
+
263
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
264
+ assert_equal 2, @c.size
265
+
266
+ @c.rollback
267
+
268
+ assert @c.empty?
269
+
270
+ assert_equal '', File.open(@c.path, 'rb'){|f| f.read }
271
+
272
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
273
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
274
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
275
+ @c.append(data)
276
+ @c.commit
277
+
278
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
279
+ assert_equal 2, @c.size
280
+
281
+ first_bytesize = @c.bytesize
282
+
283
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
284
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
285
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
286
+
287
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
288
+ assert_equal 4, @c.size
289
+
290
+ @c.rollback
291
+
292
+ assert_equal first_bytesize, @c.bytesize
293
+ assert_equal 2, @c.size
294
+
295
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read }
296
+ end
297
+
298
+ test 'can #rollback to revert non-committed data from #concat' do
299
+ assert @c.empty?
300
+
301
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
302
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
303
+ data = [d1.to_json + "\n", d2.to_json + "\n"].join
304
+ @c.concat(data, 2)
305
+
306
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
307
+ assert_equal 2, @c.size
308
+
309
+ @c.rollback
310
+
311
+ assert @c.empty?
312
+
313
+ assert_equal '', File.open(@c.path, 'rb'){|f| f.read }
314
+
315
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
316
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
317
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
318
+ @c.append(data)
319
+ @c.commit
320
+
321
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
322
+ assert_equal 2, @c.size
323
+
324
+ first_bytesize = @c.bytesize
325
+
326
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
327
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
328
+ @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
329
+
330
+ assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
331
+ assert_equal 4, @c.size
332
+
333
+ @c.rollback
334
+
335
+ assert_equal first_bytesize, @c.bytesize
336
+ assert_equal 2, @c.size
337
+
338
+ assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read }
339
+ end
340
+
341
+ test 'can store its data by #close' do
342
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
343
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
344
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
345
+ @c.append(data)
346
+ @c.commit
347
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
348
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
349
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
350
+ @c.commit
351
+
352
+ content = @c.read
353
+
354
+ unique_id = @c.unique_id
355
+ size = @c.size
356
+ created_at = @c.created_at
357
+ modified_at = @c.modified_at
358
+
359
+ @c.close
360
+
361
+ assert_equal content, File.open(@c.path, 'rb'){|f| f.read }
362
+
363
+ stored_meta = {
364
+ timekey: nil, tag: nil, variables: nil, seq: 0,
365
+ id: unique_id,
366
+ s: size,
367
+ c: created_at.to_i,
368
+ m: modified_at.to_i,
369
+ }
370
+
371
+ assert_equal stored_meta, read_metadata_file(@c.path + '.meta')
372
+ end
373
+
374
+ test 'deletes all data by #purge' do
375
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
376
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
377
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
378
+ @c.append(data)
379
+ @c.commit
380
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
381
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
382
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
383
+ @c.commit
384
+
385
+ @c.purge
386
+
387
+ assert @c.empty?
388
+ assert_equal 0, @c.bytesize
389
+ assert_equal 0, @c.size
390
+
391
+ assert !File.exist?(@c.path)
392
+ assert !File.exist?(@c.path + '.meta')
393
+ end
394
+
395
+ test 'can #open its contents as io' do
396
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
397
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
398
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
399
+ @c.append(data)
400
+ @c.commit
401
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
402
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
403
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
404
+ @c.commit
405
+
406
+ lines = []
407
+ @c.open do |io|
408
+ assert io
409
+ io.readlines.each do |l|
410
+ lines << l
411
+ end
412
+ end
413
+
414
+ assert_equal d1.to_json + "\n", lines[0]
415
+ assert_equal d2.to_json + "\n", lines[1]
416
+ assert_equal d3.to_json + "\n", lines[2]
417
+ assert_equal d4.to_json + "\n", lines[3]
418
+ end
419
+
420
+ test '#write_metadata tries to store metadata on file' do
421
+ d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
422
+ d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
423
+ data = [d1.to_json + "\n", d2.to_json + "\n"]
424
+ @c.append(data)
425
+ @c.commit
426
+
427
+ expected = {
428
+ timekey: nil, tag: nil, variables: nil, seq: 0,
429
+ id: @c.unique_id,
430
+ s: @c.size,
431
+ c: @c.created_at.to_i,
432
+ m: @c.modified_at.to_i,
433
+ }
434
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
435
+
436
+ d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
437
+ d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
438
+ @c.append([d3.to_json + "\n", d4.to_json + "\n"])
439
+ # append does write_metadata
440
+
441
+ dummy_now = Time.parse('2016-04-07 16:59:59 +0900')
442
+ Timecop.freeze(dummy_now)
443
+ @c.write_metadata
444
+
445
+ expected = {
446
+ timekey: nil, tag: nil, variables: nil, seq: 0,
447
+ id: @c.unique_id,
448
+ s: @c.size,
449
+ c: @c.created_at.to_i,
450
+ m: dummy_now.to_i,
451
+ }
452
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
453
+
454
+ @c.commit
455
+
456
+ expected = {
457
+ timekey: nil, tag: nil, variables: nil, seq: 0,
458
+ id: @c.unique_id,
459
+ s: @c.size,
460
+ c: @c.created_at.to_i,
461
+ m: @c.modified_at.to_i,
462
+ }
463
+ assert_equal expected, read_metadata_file(@c.path + '.meta')
464
+
465
+ content = @c.read
466
+
467
+ unique_id = @c.unique_id
468
+ size = @c.size
469
+ created_at = @c.created_at
470
+ modified_at = @c.modified_at
471
+
472
+ @c.close
473
+
474
+ assert_equal content, File.open(@c.path, 'rb'){|f| f.read }
475
+
476
+ stored_meta = {
477
+ timekey: nil, tag: nil, variables: nil, seq: 0,
478
+ id: unique_id,
479
+ s: size,
480
+ c: created_at.to_i,
481
+ m: modified_at.to_i,
482
+ }
483
+
484
+ assert_equal stored_meta, read_metadata_file(@c.path + '.meta')
485
+ end
486
+ end
487
+
488
+ test 'ensure to remove metadata file if #write_metadata raise an error because of disk full' do
489
+ chunk_path = File.join(@chunkdir, 'test.*.log')
490
+ stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
491
+
492
+ any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass|
493
+ stub(klass).write_metadata(anything) do |v|
494
+ raise 'disk full'
495
+ end
496
+ end
497
+
498
+ err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do
499
+ Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
500
+ end
501
+
502
+ assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta'))
503
+ assert_match(/create buffer metadata/, err.message)
504
+ end
505
+
506
+ sub_test_case 'chunk with file for staged chunk' do
507
+ setup do
508
+ @chunk_id = gen_test_chunk_id
509
+ @chunk_path = File.join(@chunkdir, "test_staged.b#{hex_id(@chunk_id)}.log")
510
+ @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log")
511
+
512
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
513
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
514
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
515
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
516
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
517
+ File.open(@chunk_path, 'wb') do |f|
518
+ f.write @d
519
+ end
520
+
521
+ @metadata = {
522
+ timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0,
523
+ id: @chunk_id,
524
+ s: 4,
525
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
526
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
527
+ }
528
+ File.open(@chunk_path + '.meta', 'wb') do |f|
529
+ f.write @metadata.to_msgpack
530
+ end
531
+
532
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)
533
+ end
534
+
535
+ teardown do
536
+ if @c
537
+ @c.purge rescue nil
538
+ end
539
+ [@chunk_path, @chunk_path + '.meta', @enqueued_path, @enqueued_path + '.meta'].each do |path|
540
+ File.unlink path if File.exist? path
541
+ end
542
+ end
543
+
544
+ test 'can load as staged chunk from file with metadata' do
545
+ assert_equal @chunk_path, @c.path
546
+ assert_equal :staged, @c.state
547
+
548
+ assert_nil @c.metadata.timekey
549
+ assert_equal 'testing', @c.metadata.tag
550
+ assert_equal({k: "x"}, @c.metadata.variables)
551
+
552
+ assert_equal 4, @c.size
553
+ assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at
554
+ assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at
555
+
556
+ content = @c.read
557
+ assert_equal @d, content
558
+ end
559
+
560
+ test 'can be enqueued' do
561
+ stage_path = @c.path
562
+ queue_path = @enqueued_path
563
+ assert File.exist?(stage_path)
564
+ assert File.exist?(stage_path + '.meta')
565
+ assert !File.exist?(queue_path)
566
+ assert !File.exist?(queue_path + '.meta')
567
+
568
+ @c.enqueued!
569
+
570
+ assert_equal queue_path, @c.path
571
+
572
+ assert !File.exist?(stage_path)
573
+ assert !File.exist?(stage_path + '.meta')
574
+ assert File.exist?(queue_path)
575
+ assert File.exist?(queue_path + '.meta')
576
+
577
+ assert_nil @c.metadata.timekey
578
+ assert_equal 'testing', @c.metadata.tag
579
+ assert_equal({k: "x"}, @c.metadata.variables)
580
+
581
+ assert_equal 4, @c.size
582
+ assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at
583
+ assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at
584
+
585
+ assert_equal @d, File.open(@c.path, 'rb'){|f| f.read }
586
+ assert_equal @metadata, read_metadata_file(@c.path + '.meta')
587
+ end
588
+
589
+ test '#write_metadata tries to store metadata on file with non-committed data' do
590
+ d5 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
591
+ d5s = d5.to_json + "\n"
592
+ @c.append([d5s])
593
+
594
+ metadata = {
595
+ timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0,
596
+ id: @chunk_id,
597
+ s: 4,
598
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
599
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
600
+ }
601
+ assert_equal metadata, read_metadata_file(@c.path + '.meta')
602
+
603
+ @c.write_metadata
604
+
605
+ metadata = {
606
+ timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0,
607
+ id: @chunk_id,
608
+ s: 5,
609
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
610
+ m: Time.parse('2016-04-07 17:44:38 +0900').to_i,
611
+ }
612
+
613
+ dummy_now = Time.parse('2016-04-07 17:44:38 +0900')
614
+ Timecop.freeze(dummy_now)
615
+ @c.write_metadata
616
+
617
+ assert_equal metadata, read_metadata_file(@c.path + '.meta')
618
+ end
619
+
620
+ test '#file_rename can rename chunk files even in windows, and call callback with file size' do
621
+ data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc"
622
+
623
+ testing_file1 = gen_path('rename1.test')
624
+ testing_file2 = gen_path('rename2.test')
625
+ f = File.open(testing_file1, 'wb', @c.permission)
626
+ f.set_encoding(Encoding::ASCII_8BIT)
627
+ f.sync = true
628
+ f.binmode
629
+ f.write data
630
+ pos = f.pos
631
+
632
+ assert f.binmode?
633
+ assert f.sync
634
+ assert_equal data.bytesize, f.size
635
+
636
+ io = nil
637
+ @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })
638
+ assert io
639
+ if Fluent.windows?
640
+ assert{ f != io }
641
+ else
642
+ assert_equal f, io
643
+ end
644
+ assert_equal Encoding::ASCII_8BIT, io.external_encoding
645
+ assert io.sync
646
+ assert io.binmode?
647
+ assert_equal data.bytesize, io.size
648
+
649
+ assert_equal pos, io.pos
650
+
651
+ assert_equal '', io.read
652
+
653
+ io.rewind
654
+ assert_equal data, io.read
655
+ end
656
+ end
657
+
658
+ sub_test_case 'chunk with file for enqueued chunk' do
659
+ setup do
660
+ @chunk_id = gen_test_chunk_id
661
+ @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log")
662
+
663
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
664
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
665
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
666
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
667
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
668
+ File.open(@enqueued_path, 'wb') do |f|
669
+ f.write @d
670
+ end
671
+
672
+ @dummy_timekey = Time.parse('2016-04-07 17:40:00 +0900').to_i
673
+
674
+ @metadata = {
675
+ timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}, seq: 0,
676
+ id: @chunk_id,
677
+ s: 4,
678
+ c: Time.parse('2016-04-07 17:44:00 +0900').to_i,
679
+ m: Time.parse('2016-04-07 17:44:13 +0900').to_i,
680
+ }
681
+ File.open(@enqueued_path + '.meta', 'wb') do |f|
682
+ f.write @metadata.to_msgpack
683
+ end
684
+
685
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @enqueued_path, :queued)
686
+ end
687
+
688
+ teardown do
689
+ if @c
690
+ @c.purge rescue nil
691
+ end
692
+ [@enqueued_path, @enqueued_path + '.meta'].each do |path|
693
+ File.unlink path if File.exist? path
694
+ end
695
+ end
696
+
697
+ test 'can load as queued chunk (read only) with metadata' do
698
+ assert @c
699
+ assert_equal @chunk_id, @c.unique_id
700
+ assert_equal :queued, @c.state
701
+ assert_equal gen_metadata(timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}), @c.metadata
702
+ assert_equal Time.at(@metadata[:c]), @c.created_at
703
+ assert_equal Time.at(@metadata[:m]), @c.modified_at
704
+ assert_equal @metadata[:s], @c.size
705
+ assert_equal @d.bytesize, @c.bytesize
706
+ assert_equal @d, @c.read
707
+
708
+ assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
709
+ @c.append(["queued chunk is read only"])
710
+ end
711
+ assert_raise IOError do
712
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
713
+ end
714
+ end
715
+ end
716
+
717
+ sub_test_case 'chunk with queued chunk file of v0.12, without metadata' do
718
+ setup do
719
+ @chunk_id = gen_test_chunk_id
720
+ @chunk_path = File.join(@chunkdir, "test_v12.2016040811.q#{hex_id(@chunk_id)}.log")
721
+
722
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
723
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
724
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
725
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
726
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
727
+ File.open(@chunk_path, 'wb') do |f|
728
+ f.write @d
729
+ end
730
+
731
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :queued)
732
+ end
733
+
734
+ teardown do
735
+ if @c
736
+ @c.purge rescue nil
737
+ end
738
+ File.unlink @chunk_path if File.exist? @chunk_path
739
+ end
740
+
741
+ test 'can load as queued chunk from file without metadata' do
742
+ assert @c
743
+ assert_equal :queued, @c.state
744
+ assert_equal @chunk_id, @c.unique_id
745
+ assert_equal gen_metadata, @c.metadata
746
+ assert_equal @d.bytesize, @c.bytesize
747
+ assert_equal 0, @c.size
748
+ assert_equal @d, @c.read
749
+
750
+ assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
751
+ @c.append(["queued chunk is read only"])
752
+ end
753
+ assert_raise IOError do
754
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
755
+ end
756
+ end
757
+ end
758
+
759
+ sub_test_case 'chunk with staged chunk file of v0.12, without metadata' do
760
+ setup do
761
+ @chunk_id = gen_test_chunk_id
762
+ @chunk_path = File.join(@chunkdir, "test_v12.2016040811.b#{hex_id(@chunk_id)}.log")
763
+
764
+ @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
765
+ @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
766
+ @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
767
+ @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
768
+ @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join
769
+ File.open(@chunk_path, 'wb') do |f|
770
+ f.write @d
771
+ end
772
+
773
+ @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged)
774
+ end
775
+
776
+ teardown do
777
+ if @c
778
+ @c.purge rescue nil
779
+ end
780
+ File.unlink @chunk_path if File.exist? @chunk_path
781
+ end
782
+
783
+ test 'can load as queued chunk from file without metadata even if it was loaded as staged chunk' do
784
+ assert @c
785
+ assert_equal :queued, @c.state
786
+ assert_equal @chunk_id, @c.unique_id
787
+ assert_equal gen_metadata, @c.metadata
788
+ assert_equal @d.bytesize, @c.bytesize
789
+ assert_equal 0, @c.size
790
+ assert_equal @d, @c.read
791
+
792
+ assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
793
+ @c.append(["queued chunk is read only"])
794
+ end
795
+ assert_raise IOError do
796
+ @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
797
+ end
798
+ end
799
+ end
800
+
801
+ sub_test_case 'compressed buffer' do
802
+ setup do
803
+ @src = 'text data for compressing' * 5
804
+ @gzipped_src = compress(@src)
805
+ end
806
+
807
+ test '#append with compress option writes compressed data to chunk when compress is gzip' do
808
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
809
+ c.append([@src, @src], compress: :gzip)
810
+ c.commit
811
+
812
+ # check chunk is compressed
813
+ assert c.read(compressed: :gzip).size < [@src, @src].join("").size
814
+
815
+ assert_equal @src + @src, c.read
816
+ end
817
+
818
+ test '#open passes io object having decompressed data to a block when compress is gzip' do
819
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
820
+ c.concat(@gzipped_src, @src.size)
821
+ c.commit
822
+
823
+ decomressed_data = c.open do |io|
824
+ v = io.read
825
+ assert_equal @src, v
826
+ v
827
+ end
828
+ assert_equal @src, decomressed_data
829
+ end
830
+
831
+ test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
832
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
833
+ c.concat(@gzipped_src, @src.size)
834
+ c.commit
835
+
836
+ comressed_data = c.open(compressed: :gzip) do |io|
837
+ v = io.read
838
+ assert_equal @gzipped_src, v
839
+ v
840
+ end
841
+ assert_equal @gzipped_src, comressed_data
842
+ end
843
+
844
+ test '#write_to writes decompressed data when compress is gzip' do
845
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
846
+ c.concat(@gzipped_src, @src.size)
847
+ c.commit
848
+
849
+ assert_equal @src, c.read
850
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
851
+
852
+ io = StringIO.new
853
+ c.write_to(io)
854
+ assert_equal @src, io.string
855
+ end
856
+
857
+ test '#write_to with compressed option writes compressed data when compress is gzip' do
858
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
859
+ c.concat(@gzipped_src, @src.size)
860
+ c.commit
861
+
862
+ assert_equal @src, c.read
863
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
864
+
865
+ io = StringIO.new
866
+ io.set_encoding(Encoding::ASCII_8BIT)
867
+ c.write_to(io, compressed: :gzip)
868
+ assert_equal @gzipped_src, io.string
869
+ end
870
+ end
871
+ end