fluentd 0.14.4-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,119 @@
1
+ # simplecov must be loaded before any of target code
2
+ if ENV['SIMPLE_COV']
3
+ require 'simplecov'
4
+ if defined?(SimpleCov::SourceFile)
5
+ mod = SimpleCov::SourceFile
6
+ def mod.new(*args, &block)
7
+ m = allocate
8
+ m.instance_eval do
9
+ begin
10
+ initialize(*args, &block)
11
+ rescue Encoding::UndefinedConversionError
12
+ @src = "".force_encoding('UTF-8')
13
+ end
14
+ end
15
+ m
16
+ end
17
+ end
18
+ unless SimpleCov.running
19
+ SimpleCov.start do
20
+ add_filter '/test/'
21
+ add_filter '/gems/'
22
+ end
23
+ end
24
+ end
25
+
26
+ # Some tests use Hash instead of Element for configure.
27
+ # We should rewrite these tests in the future and remove this ad-hoc code
28
+ class Hash
29
+ def corresponding_proxies
30
+ @corresponding_proxies ||= []
31
+ end
32
+
33
+ def to_masked_element
34
+ self
35
+ end
36
+ end
37
+
38
+ require 'rr'
39
+ require 'test/unit'
40
+ require 'test/unit/rr'
41
+ require 'fileutils'
42
+ require 'fluent/config/element'
43
+ require 'fluent/log'
44
+ require 'fluent/test'
45
+ require 'fluent/test/helpers'
46
+ require 'fluent/plugin/base'
47
+ require 'fluent/log'
48
+ require 'fluent/plugin_id'
49
+ require 'fluent/plugin_helper'
50
+ require 'fluent/msgpack_factory'
51
+ require 'fluent/time'
52
+ require 'serverengine'
53
+
54
+ module Fluent
55
+ module Plugin
56
+ class TestBase < Base
57
+ # a base plugin class, but not input nor output
58
+ # mainly for helpers and owned plugins
59
+ include PluginId
60
+ include PluginLoggerMixin
61
+ include PluginHelper::Mixin
62
+ end
63
+ end
64
+ end
65
+
66
+ unless defined?(Test::Unit::AssertionFailedError)
67
+ class Test::Unit::AssertionFailedError < StandardError
68
+ end
69
+ end
70
+
71
+ include Fluent::Test::Helpers
72
+
73
+ def unused_port(num = 1)
74
+ ports = []
75
+ sockets = []
76
+ num.times do
77
+ s = TCPServer.open(0)
78
+ sockets << s
79
+ ports << s.addr[1]
80
+ end
81
+ sockets.each{|s| s.close }
82
+ if num == 1
83
+ return ports.first
84
+ else
85
+ return *ports
86
+ end
87
+ end
88
+
89
+ def waiting(seconds, logs: nil, plugin: nil)
90
+ begin
91
+ Timeout.timeout(seconds) do
92
+ yield
93
+ end
94
+ rescue Timeout::Error
95
+ if logs
96
+ STDERR.print(*logs)
97
+ elsif plugin
98
+ STDERR.print(*plugin.log.out.logs)
99
+ end
100
+ raise
101
+ end
102
+ end
103
+
104
+ def ipv6_enabled?
105
+ require 'socket'
106
+
107
+ begin
108
+ TCPServer.open("::1", 0)
109
+ true
110
+ rescue
111
+ false
112
+ end
113
+ end
114
+
115
+ dl_opts = {}
116
+ dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
117
+ logdev = Fluent::Test::DummyLogDevice.new
118
+ logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
119
+ $log ||= Fluent::Log.new(logger)
File without changes
File without changes
File without changes
File without changes
@@ -0,0 +1,118 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/bare_output'
3
+ require 'fluent/event'
4
+
5
+ module FluentPluginBareOutputTest
6
+ class DummyPlugin < Fluent::Plugin::BareOutput
7
+ attr_reader :store
8
+ def initialize
9
+ super
10
+ @store = []
11
+ end
12
+ def process(tag, es)
13
+ es.each do |time, record|
14
+ @store << [tag, time, record]
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ class BareOutputTest < Test::Unit::TestCase
21
+ setup do
22
+ Fluent::Test.setup
23
+ @p = FluentPluginBareOutputTest::DummyPlugin.new
24
+ end
25
+
26
+ test 'has healthy lifecycle' do
27
+ assert !@p.configured?
28
+ @p.configure(config_element())
29
+ assert @p.configured?
30
+
31
+ assert !@p.started?
32
+ @p.start
33
+ assert @p.start
34
+
35
+ assert !@p.stopped?
36
+ @p.stop
37
+ assert @p.stopped?
38
+
39
+ assert !@p.before_shutdown?
40
+ @p.before_shutdown
41
+ assert @p.before_shutdown?
42
+
43
+ assert !@p.shutdown?
44
+ @p.shutdown
45
+ assert @p.shutdown?
46
+
47
+ assert !@p.after_shutdown?
48
+ @p.after_shutdown
49
+ assert @p.after_shutdown?
50
+
51
+ assert !@p.closed?
52
+ @p.close
53
+ assert @p.closed?
54
+
55
+ assert !@p.terminated?
56
+ @p.terminate
57
+ assert @p.terminated?
58
+ end
59
+
60
+ test 'has plugin_id automatically generated' do
61
+ assert @p.respond_to?(:plugin_id_configured?)
62
+ assert @p.respond_to?(:plugin_id)
63
+
64
+ @p.configure(config_element())
65
+
66
+ assert !@p.plugin_id_configured?
67
+ assert @p.plugin_id
68
+ assert{ @p.plugin_id != 'mytest' }
69
+ end
70
+
71
+ test 'has plugin_id manually configured' do
72
+ @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
73
+ assert @p.plugin_id_configured?
74
+ assert_equal 'mytest', @p.plugin_id
75
+ end
76
+
77
+ test 'has plugin logger' do
78
+ assert @p.respond_to?(:log)
79
+ assert @p.log
80
+
81
+ # default logger
82
+ original_logger = @p.log
83
+
84
+ @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
85
+
86
+ assert{ @p.log.object_id != original_logger.object_id }
87
+ assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
88
+ end
89
+
90
+ test 'can load plugin helpers' do
91
+ assert_nothing_raised do
92
+ class FluentPluginBareOutputTest::DummyPlugin2 < Fluent::Plugin::BareOutput
93
+ helpers :storage
94
+ end
95
+ end
96
+ end
97
+
98
+ test 'can get input event stream to write' do
99
+ @p.configure(config_element('ROOT'))
100
+ @p.start
101
+
102
+ es1 = Fluent::OneEventStream.new(event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'})
103
+ es2 = Fluent::ArrayEventStream.new([
104
+ [event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
105
+ [event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
106
+ ])
107
+ @p.emit_events('mytest1', es1)
108
+ @p.emit_events('mytest2', es2)
109
+
110
+ all_events = [
111
+ ['mytest1', event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}],
112
+ ['mytest2', event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
113
+ ['mytest2', event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
114
+ ]
115
+
116
+ assert_equal all_events, @p.store
117
+ end
118
+ end
@@ -0,0 +1,75 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/base'
3
+
4
+ module FluentPluginBaseTest
5
+ class DummyPlugin < Fluent::Plugin::Base
6
+ end
7
+ end
8
+
9
+ class BaseTest < Test::Unit::TestCase
10
+ setup do
11
+ @p = FluentPluginBaseTest::DummyPlugin.new
12
+ end
13
+
14
+ test 'has methods for phases of plugin life cycle, and methods to know "super"s were correctly called or not' do
15
+ assert !@p.configured?
16
+ @p.configure(config_element())
17
+ assert @p.configured?
18
+
19
+ assert !@p.started?
20
+ @p.start
21
+ assert @p.start
22
+
23
+ assert !@p.stopped?
24
+ @p.stop
25
+ assert @p.stopped?
26
+
27
+ assert !@p.before_shutdown?
28
+ @p.before_shutdown
29
+ assert @p.before_shutdown?
30
+
31
+ assert !@p.shutdown?
32
+ @p.shutdown
33
+ assert @p.shutdown?
34
+
35
+ assert !@p.after_shutdown?
36
+ @p.after_shutdown
37
+ assert @p.after_shutdown?
38
+
39
+ assert !@p.closed?
40
+ @p.close
41
+ assert @p.closed?
42
+
43
+ assert !@p.terminated?
44
+ @p.terminate
45
+ assert @p.terminated?
46
+ end
47
+
48
+ test 'can access system config' do
49
+ assert @p.system_config
50
+
51
+ @p.system_config_override({'process_name' => 'mytest'})
52
+ assert_equal 'mytest', @p.system_config.process_name
53
+ end
54
+
55
+ test 'does not have router in default' do
56
+ assert !@p.has_router?
57
+ end
58
+
59
+ test 'is configurable by config_param and config_section' do
60
+ assert_nothing_raised do
61
+ class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase
62
+ config_param :myparam1, :string
63
+ config_section :mysection, multi: false do
64
+ config_param :myparam2, :integer
65
+ end
66
+ end
67
+ end
68
+ p2 = FluentPluginBaseTest::DummyPlugin2.new
69
+ assert_nothing_raised do
70
+ p2.configure(config_element('ROOT', '', {'myparam1' => 'myvalue1'}, [config_element('mysection', '', {'myparam2' => 99})]))
71
+ end
72
+ assert_equal 'myvalue1', p2.myparam1
73
+ assert_equal 99, p2.mysection.myparam2
74
+ end
75
+ end
@@ -0,0 +1,571 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buf_file'
3
+ require 'fluent/plugin/output'
4
+ require 'fluent/unique_id'
5
+ require 'fluent/system_config'
6
+ require 'fluent/env'
7
+
8
+ require 'msgpack'
9
+
10
+ module FluentPluginFileBufferTest
11
+ class DummyOutputPlugin < Fluent::Plugin::Output
12
+ Fluent::Plugin.register_output('buffer_file_test_output', self)
13
+ end
14
+ end
15
+
16
+ class FileBufferTest < Test::Unit::TestCase
17
+ def metadata(timekey: nil, tag: nil, variables: nil)
18
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
19
+ end
20
+
21
+ def write_metadata(path, chunk_id, metadata, size, ctime, mtime)
22
+ metadata = {
23
+ timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables,
24
+ id: chunk_id,
25
+ s: size,
26
+ c: ctime,
27
+ m: mtime,
28
+ }
29
+ File.open(path, 'wb') do |f|
30
+ f.write metadata.to_msgpack
31
+ end
32
+ end
33
+
34
+ sub_test_case 'non configured buffer plugin instance' do
35
+ setup do
36
+ Fluent::Test.setup
37
+
38
+ @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__)
39
+ unless File.exist?(@dir)
40
+ FileUtils.mkdir_p @dir
41
+ end
42
+ Dir.glob(File.join(@dir, '*')).each do |path|
43
+ next if ['.', '..'].include?(File.basename(path))
44
+ File.delete(path)
45
+ end
46
+ end
47
+
48
+ test 'path should include * normally' do
49
+ d = FluentPluginFileBufferTest::DummyOutputPlugin.new
50
+ p = Fluent::Plugin::FileBuffer.new
51
+ p.owner = d
52
+ p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer.*.file')}))
53
+ assert_equal File.join(@dir, 'buffer.*.file'), p.path
54
+ end
55
+
56
+ test 'existing directory will be used with additional default file name' do
57
+ d = FluentPluginFileBufferTest::DummyOutputPlugin.new
58
+ p = Fluent::Plugin::FileBuffer.new
59
+ p.owner = d
60
+ p.configure(config_element('buffer', '', {'path' => @dir}))
61
+ assert_equal File.join(@dir, 'buffer.*.log'), p.path
62
+ end
63
+
64
+ test 'unexisting path without * handled as directory' do
65
+ d = FluentPluginFileBufferTest::DummyOutputPlugin.new
66
+ p = Fluent::Plugin::FileBuffer.new
67
+ p.owner = d
68
+ p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
69
+ assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
70
+ end
71
+ end
72
+
73
+ sub_test_case 'buffer plugin configured only with path' do
74
+ setup do
75
+ @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
76
+ @bufpath = File.join(@bufdir, 'testbuf.*.log')
77
+ FileUtils.rm_r @bufdir if File.exist?(@bufdir)
78
+
79
+ Fluent::Test.setup
80
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
81
+ @p = Fluent::Plugin::FileBuffer.new
82
+ @p.owner = @d
83
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
84
+ @p.start
85
+ end
86
+
87
+ teardown do
88
+ if @p
89
+ @p.stop unless @p.stopped?
90
+ @p.before_shutdown unless @p.before_shutdown?
91
+ @p.shutdown unless @p.shutdown?
92
+ @p.after_shutdown unless @p.after_shutdown?
93
+ @p.close unless @p.closed?
94
+ @p.terminate unless @p.terminated?
95
+ end
96
+ if @bufdir
97
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
98
+ next if ['.', '..'].include?(File.basename(path))
99
+ File.delete(path)
100
+ end
101
+ end
102
+ end
103
+
104
+ test 'this is persistent plugin' do
105
+ assert @p.persistent?
106
+ end
107
+
108
+ test '#start creates directory for buffer chunks' do
109
+ plugin = Fluent::Plugin::FileBuffer.new
110
+ plugin.owner = @d
111
+ rand_num = rand(0..100)
112
+ bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
113
+ bufdir = File.dirname(bufpath)
114
+
115
+ FileUtils.rm_r bufdir if File.exist?(bufdir)
116
+ assert !File.exist?(bufdir)
117
+
118
+ plugin.configure(config_element('buffer', '', {'path' => bufpath}))
119
+ assert !File.exist?(bufdir)
120
+
121
+ plugin.start
122
+ assert File.exist?(bufdir)
123
+ assert{ File.stat(bufdir).mode.to_s(8).end_with?('755') }
124
+
125
+ plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
126
+ FileUtils.rm_r bufdir
127
+ end
128
+
129
+ test '#start creates directory for buffer chunks with specified permission' do
130
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
131
+
132
+ plugin = Fluent::Plugin::FileBuffer.new
133
+ plugin.owner = @d
134
+ rand_num = rand(0..100)
135
+ bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
136
+ bufdir = File.dirname(bufpath)
137
+
138
+ FileUtils.rm_r bufdir if File.exist?(bufdir)
139
+ assert !File.exist?(bufdir)
140
+
141
+ plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => 0700}))
142
+ assert !File.exist?(bufdir)
143
+
144
+ plugin.start
145
+ assert File.exist?(bufdir)
146
+ assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }
147
+
148
+ plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
149
+ FileUtils.rm_r bufdir
150
+ end
151
+
152
+ test '#start creates directory for buffer chunks with specified permission via system config' do
153
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
154
+
155
+ sysconf = {'dir_permission' => '700'}
156
+ Fluent::SystemConfig.overwrite_system_config(sysconf) do
157
+ plugin = Fluent::Plugin::FileBuffer.new
158
+ plugin.owner = @d
159
+ rand_num = rand(0..100)
160
+ bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
161
+ bufdir = File.dirname(bufpath)
162
+
163
+ FileUtils.rm_r bufdir if File.exist?(bufdir)
164
+ assert !File.exist?(bufdir)
165
+
166
+ plugin.configure(config_element('buffer', '', {'path' => bufpath}))
167
+ assert !File.exist?(bufdir)
168
+
169
+ plugin.start
170
+ assert File.exist?(bufdir)
171
+ assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }
172
+
173
+ plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
174
+ FileUtils.rm_r bufdir
175
+ end
176
+ end
177
+
178
+ test '#generate_chunk generates blank file chunk on path from unique_id of metadata' do
179
+ m1 = metadata()
180
+ c1 = @p.generate_chunk(m1)
181
+ assert c1.is_a? Fluent::Plugin::Buffer::FileChunk
182
+ assert_equal m1, c1.metadata
183
+ assert c1.empty?
184
+ assert_equal :unstaged, c1.state
185
+ assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c1.permission
186
+ assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path
187
+ assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
188
+
189
+ m2 = metadata(timekey: event_time('2016-04-17 11:15:00 -0700').to_i)
190
+ c2 = @p.generate_chunk(m2)
191
+ assert c2.is_a? Fluent::Plugin::Buffer::FileChunk
192
+ assert_equal m2, c2.metadata
193
+ assert c2.empty?
194
+ assert_equal :unstaged, c2.state
195
+ assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission
196
+ assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path
197
+ assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') }
198
+
199
+ c1.purge
200
+ c2.purge
201
+ end
202
+
203
+ test '#generate_chunk generates blank file chunk with specified permission' do
204
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
205
+
206
+ plugin = Fluent::Plugin::FileBuffer.new
207
+ plugin.owner = @d
208
+ rand_num = rand(0..100)
209
+ bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
210
+ bufdir = File.dirname(bufpath)
211
+
212
+ FileUtils.rm_r bufdir if File.exist?(bufdir)
213
+ assert !File.exist?(bufdir)
214
+
215
+ plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => 0600}))
216
+ assert !File.exist?(bufdir)
217
+ plugin.start
218
+
219
+ m = metadata()
220
+ c = plugin.generate_chunk(m)
221
+ assert c.is_a? Fluent::Plugin::Buffer::FileChunk
222
+ assert_equal m, c.metadata
223
+ assert c.empty?
224
+ assert_equal :unstaged, c.state
225
+ assert_equal 0600, c.permission
226
+ assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path
227
+ assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
228
+
229
+ c.purge
230
+
231
+ plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
232
+ FileUtils.rm_r bufdir
233
+ end
234
+ end
235
+
236
+
237
+ sub_test_case 'there are no existing file chunks' do
238
+ setup do
239
+ @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
240
+ @bufpath = File.join(@bufdir, 'testbuf.*.log')
241
+ FileUtils.rm_r @bufdir if File.exist?(@bufdir)
242
+
243
+ Fluent::Test.setup
244
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
245
+ @p = Fluent::Plugin::FileBuffer.new
246
+ @p.owner = @d
247
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
248
+ @p.start
249
+ end
250
+ teardown do
251
+ if @p
252
+ @p.stop unless @p.stopped?
253
+ @p.before_shutdown unless @p.before_shutdown?
254
+ @p.shutdown unless @p.shutdown?
255
+ @p.after_shutdown unless @p.after_shutdown?
256
+ @p.close unless @p.closed?
257
+ @p.terminate unless @p.terminated?
258
+ end
259
+ if @bufdir
260
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
261
+ next if ['.', '..'].include?(File.basename(path))
262
+ File.delete(path)
263
+ end
264
+ end
265
+ end
266
+
267
+ test '#resume returns empty buffer state' do
268
+ ary = @p.resume
269
+ assert_equal({}, ary[0])
270
+ assert_equal([], ary[1])
271
+ end
272
+ end
273
+
274
+ sub_test_case 'there are some existing file chunks' do
275
+ setup do
276
+ @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
277
+ FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
278
+
279
+ @c1id = Fluent::UniqueId.generate
280
+ p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log")
281
+ File.open(p1, 'wb') do |f|
282
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
283
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
284
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
285
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
286
+ end
287
+ write_metadata(
288
+ p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
289
+ 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
290
+ )
291
+
292
+ @c2id = Fluent::UniqueId.generate
293
+ p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log")
294
+ File.open(p2, 'wb') do |f|
295
+ f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
296
+ f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
297
+ f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
298
+ end
299
+ write_metadata(
300
+ p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),
301
+ 3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i
302
+ )
303
+
304
+ @c3id = Fluent::UniqueId.generate
305
+ p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log")
306
+ File.open(p3, 'wb') do |f|
307
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
308
+ f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
309
+ f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
310
+ f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
311
+ end
312
+ write_metadata(
313
+ p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
314
+ 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
315
+ )
316
+
317
+ @c4id = Fluent::UniqueId.generate
318
+ p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log")
319
+ File.open(p4, 'wb') do |f|
320
+ f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
321
+ f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
322
+ f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
323
+ end
324
+ write_metadata(
325
+ p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
326
+ 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
327
+ )
328
+
329
+ @bufpath = File.join(@bufdir, 'etest.*.log')
330
+
331
+ Fluent::Test.setup
332
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
333
+ @p = Fluent::Plugin::FileBuffer.new
334
+ @p.owner = @d
335
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
336
+ @p.start
337
+ end
338
+
339
+ teardown do
340
+ if @p
341
+ @p.stop unless @p.stopped?
342
+ @p.before_shutdown unless @p.before_shutdown?
343
+ @p.shutdown unless @p.shutdown?
344
+ @p.after_shutdown unless @p.after_shutdown?
345
+ @p.close unless @p.closed?
346
+ @p.terminate unless @p.terminated?
347
+ end
348
+ if @bufdir
349
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
350
+ next if ['.', '..'].include?(File.basename(path))
351
+ File.delete(path)
352
+ end
353
+ end
354
+ end
355
+
356
+ test '#resume returns staged/queued chunks with metadata' do
357
+ assert_equal 2, @p.stage.size
358
+ assert_equal 2, @p.queue.size
359
+
360
+ stage = @p.stage
361
+
362
+ m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
363
+ assert_equal @c3id, stage[m3].unique_id
364
+ assert_equal 4, stage[m3].size
365
+ assert_equal :staged, stage[m3].state
366
+
367
+ m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
368
+ assert_equal @c4id, stage[m4].unique_id
369
+ assert_equal 3, stage[m4].size
370
+ assert_equal :staged, stage[m4].state
371
+ end
372
+
373
+ test '#resume returns queued chunks ordered by last modified time (FIFO)' do
374
+ assert_equal 2, @p.stage.size
375
+ assert_equal 2, @p.queue.size
376
+
377
+ queue = @p.queue
378
+
379
+ assert{ queue[0].modified_at < queue[1].modified_at }
380
+
381
+ assert_equal @c1id, queue[0].unique_id
382
+ assert_equal :queued, queue[0].state
383
+ assert_equal event_time('2016-04-17 13:58:00 -0700').to_i, queue[0].metadata.timekey
384
+ assert_nil queue[0].metadata.tag
385
+ assert_nil queue[0].metadata.variables
386
+ assert_equal Time.parse('2016-04-17 13:58:00 -0700').localtime, queue[0].created_at
387
+ assert_equal Time.parse('2016-04-17 13:58:22 -0700').localtime, queue[0].modified_at
388
+ assert_equal 4, queue[0].size
389
+
390
+ assert_equal @c2id, queue[1].unique_id
391
+ assert_equal :queued, queue[1].state
392
+ assert_equal event_time('2016-04-17 13:59:00 -0700').to_i, queue[1].metadata.timekey
393
+ assert_nil queue[1].metadata.tag
394
+ assert_nil queue[1].metadata.variables
395
+ assert_equal Time.parse('2016-04-17 13:59:00 -0700').localtime, queue[1].created_at
396
+ assert_equal Time.parse('2016-04-17 13:59:23 -0700').localtime, queue[1].modified_at
397
+ assert_equal 3, queue[1].size
398
+ end
399
+ end
400
+
401
+ sub_test_case 'there are some existing file chunks without metadata file' do
402
+ setup do
403
+ @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
404
+
405
+ @c1id = Fluent::UniqueId.generate
406
+ p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
407
+ File.open(p1, 'wb') do |f|
408
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
409
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
410
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
411
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
412
+ end
413
+ FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))
414
+
415
+ @c2id = Fluent::UniqueId.generate
416
+ p2 = File.join(@bufdir, "etest.201604171359.q#{Fluent::UniqueId.hex(@c2id)}.log")
417
+ File.open(p2, 'wb') do |f|
418
+ f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
419
+ f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
420
+ f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
421
+ end
422
+ FileUtils.touch(p2, mtime: Time.parse('2016-04-17 13:59:30 -0700'))
423
+
424
+ @c3id = Fluent::UniqueId.generate
425
+ p3 = File.join(@bufdir, "etest.201604171400.b#{Fluent::UniqueId.hex(@c3id)}.log")
426
+ File.open(p3, 'wb') do |f|
427
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
428
+ f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
429
+ f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
430
+ f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
431
+ end
432
+ FileUtils.touch(p3, mtime: Time.parse('2016-04-17 14:00:29 -0700'))
433
+
434
+ @c4id = Fluent::UniqueId.generate
435
+ p4 = File.join(@bufdir, "etest.201604171401.b#{Fluent::UniqueId.hex(@c4id)}.log")
436
+ File.open(p4, 'wb') do |f|
437
+ f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
438
+ f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
439
+ f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
440
+ end
441
+ FileUtils.touch(p4, mtime: Time.parse('2016-04-17 14:01:22 -0700'))
442
+
443
+ @bufpath = File.join(@bufdir, 'etest.*.log')
444
+
445
+ Fluent::Test.setup
446
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
447
+ @p = Fluent::Plugin::FileBuffer.new
448
+ @p.owner = @d
449
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
450
+ @p.start
451
+ end
452
+
453
+ teardown do
454
+ if @p
455
+ @p.stop unless @p.stopped?
456
+ @p.before_shutdown unless @p.before_shutdown?
457
+ @p.shutdown unless @p.shutdown?
458
+ @p.after_shutdown unless @p.after_shutdown?
459
+ @p.close unless @p.closed?
460
+ @p.terminate unless @p.terminated?
461
+ end
462
+ if @bufdir
463
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
464
+ next if ['.', '..'].include?(File.basename(path))
465
+ File.delete(path)
466
+ end
467
+ end
468
+ end
469
+
470
+ test '#resume returns queued chunks for files without metadata' do
471
+ assert_equal 0, @p.stage.size
472
+ assert_equal 4, @p.queue.size
473
+
474
+ queue = @p.queue
475
+
476
+ m = metadata()
477
+
478
+ assert_equal @c1id, queue[0].unique_id
479
+ assert_equal m, queue[0].metadata
480
+ assert_equal 0, queue[0].size
481
+ assert_equal :queued, queue[0].state
482
+ assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at
483
+
484
+ assert_equal @c2id, queue[1].unique_id
485
+ assert_equal m, queue[1].metadata
486
+ assert_equal 0, queue[1].size
487
+ assert_equal :queued, queue[1].state
488
+ assert_equal Time.parse('2016-04-17 13:59:30 -0700'), queue[1].modified_at
489
+
490
+ assert_equal @c3id, queue[2].unique_id
491
+ assert_equal m, queue[2].metadata
492
+ assert_equal 0, queue[2].size
493
+ assert_equal :queued, queue[2].state
494
+ assert_equal Time.parse('2016-04-17 14:00:29 -0700'), queue[2].modified_at
495
+
496
+ assert_equal @c4id, queue[3].unique_id
497
+ assert_equal m, queue[3].metadata
498
+ assert_equal 0, queue[3].size
499
+ assert_equal :queued, queue[3].state
500
+ assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at
501
+ end
502
+ end
503
+
504
+ sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do
505
+ setup do
506
+ @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
507
+
508
+ @c1id = Fluent::UniqueId.generate
509
+ p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
510
+ File.open(p1, 'wb') do |f|
511
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
512
+ f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
513
+ f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
514
+ f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
515
+ end
516
+ FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))
517
+
518
+ @not_chunk = File.join(@bufdir, 'etest.20160416.log')
519
+ File.open(@not_chunk, 'wb') do |f|
520
+ f.write ["t1.test", event_time('2016-04-16 23:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
521
+ f.write ["t2.test", event_time('2016-04-16 23:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
522
+ f.write ["t3.test", event_time('2016-04-16 23:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
523
+ f.write ["t4.test", event_time('2016-04-16 23:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
524
+ end
525
+ FileUtils.touch(@not_chunk, mtime: Time.parse('2016-04-17 00:00:00 -0700'))
526
+
527
+ @bufpath = File.join(@bufdir, 'etest.*.log')
528
+
529
+ Fluent::Test.setup
530
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
531
+ @p = Fluent::Plugin::FileBuffer.new
532
+ @p.owner = @d
533
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
534
+ @p.start
535
+ end
536
+
537
+ teardown do
538
+ if @p
539
+ @p.stop unless @p.stopped?
540
+ @p.before_shutdown unless @p.before_shutdown?
541
+ @p.shutdown unless @p.shutdown?
542
+ @p.after_shutdown unless @p.after_shutdown?
543
+ @p.close unless @p.closed?
544
+ @p.terminate unless @p.terminated?
545
+ end
546
+ if @bufdir
547
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
548
+ next if ['.', '..'].include?(File.basename(path))
549
+ File.delete(path)
550
+ end
551
+ end
552
+ end
553
+
554
+ test '#resume returns queued chunks for files without metadata, while ignoring non-chunk looking files' do
555
+ assert_equal 0, @p.stage.size
556
+ assert_equal 1, @p.queue.size
557
+
558
+ queue = @p.queue
559
+
560
+ m = metadata()
561
+
562
+ assert_equal @c1id, queue[0].unique_id
563
+ assert_equal m, queue[0].metadata
564
+ assert_equal 0, queue[0].size
565
+ assert_equal :queued, queue[0].state
566
+ assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at
567
+
568
+ assert File.exist?(@not_chunk)
569
+ end
570
+ end
571
+ end