fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,1001 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test'
3
+ require 'fluent/plugin/in_tail'
4
+ require 'fluent/plugin/buffer'
5
+ require 'fluent/system_config'
6
+ require 'net/http'
7
+ require 'flexmock/test_unit'
8
+
9
+ class TailInputTest < Test::Unit::TestCase
10
+ include FlexMock::TestCase
11
+
12
+ def setup
13
+ Fluent::Test.setup
14
+ FileUtils.rm_rf(TMP_DIR, secure: true)
15
+ if File.exist?(TMP_DIR)
16
+ # ensure files are closed for Windows, on which deleted files
17
+ # are still visible from filesystem
18
+ GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
19
+ FileUtils.remove_entry_secure(TMP_DIR)
20
+ end
21
+ FileUtils.mkdir_p(TMP_DIR)
22
+ end
23
+
24
+ def teardown
25
+ super
26
+ Fluent::Engine.stop
27
+ end
28
+
29
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
30
+
31
+ CONFIG = %[
32
+ path #{TMP_DIR}/tail.txt
33
+ tag t1
34
+ rotate_wait 2s
35
+ ]
36
+ COMMON_CONFIG = CONFIG + %[
37
+ pos_file #{TMP_DIR}/tail.pos
38
+ ]
39
+ CONFIG_READ_FROM_HEAD = %[
40
+ read_from_head true
41
+ ]
42
+ CONFIG_ENABLE_WATCH_TIMER = %[
43
+ enable_watch_timer false
44
+ ]
45
+ SINGLE_LINE_CONFIG = %[
46
+ format /(?<message>.*)/
47
+ ]
48
+
49
+ def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
50
+ config = use_common_conf ? COMMON_CONFIG + conf : conf
51
+ Fluent::Test::InputTestDriver.new(Fluent::NewTailInput).configure(config)
52
+ end
53
+
54
+ def test_configure
55
+ d = create_driver
56
+ assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
57
+ assert_equal "t1", d.instance.tag
58
+ assert_equal 2, d.instance.rotate_wait
59
+ assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
60
+ assert_equal 1000, d.instance.read_lines_limit
61
+ end
62
+
63
+ def test_configure_encoding
64
+ # valid encoding
65
+ d = create_driver(SINGLE_LINE_CONFIG + 'encoding utf-8')
66
+ assert_equal Encoding::UTF_8, d.instance.encoding
67
+
68
+ # invalid encoding
69
+ assert_raise(Fluent::ConfigError) do
70
+ create_driver(SINGLE_LINE_CONFIG + 'encoding no-such-encoding')
71
+ end
72
+ end
73
+
74
+ def test_configure_from_encoding
75
+ # If only specified from_encoding raise ConfigError
76
+ assert_raise(Fluent::ConfigError) do
77
+ create_driver(SINGLE_LINE_CONFIG + 'from_encoding utf-8')
78
+ end
79
+
80
+ # valid setting
81
+ d = create_driver %[
82
+ format /(?<message>.*)/
83
+ read_from_head true
84
+ from_encoding utf-8
85
+ encoding utf-8
86
+ ]
87
+ assert_equal Encoding::UTF_8, d.instance.from_encoding
88
+
89
+ # invalid from_encoding
90
+ assert_raise(Fluent::ConfigError) do
91
+ d = create_driver %[
92
+ format /(?<message>.*)/
93
+ read_from_head true
94
+ from_encoding no-such-encoding
95
+ encoding utf-8
96
+ ]
97
+ end
98
+ end
99
+
100
+ # TODO: Should using more better approach instead of sleep wait
101
+
102
+ def test_emit
103
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
104
+ f.puts "test1"
105
+ f.puts "test2"
106
+ }
107
+
108
+ d = create_driver
109
+
110
+ d.run do
111
+ sleep 1
112
+
113
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
114
+ f.puts "test3"
115
+ f.puts "test4"
116
+ }
117
+ sleep 1
118
+ end
119
+
120
+ emits = d.emits
121
+ assert_equal(true, emits.length > 0)
122
+ assert_equal({"message" => "test3"}, emits[0][2])
123
+ assert_equal({"message" => "test4"}, emits[1][2])
124
+ assert(emits[0][1].is_a?(Fluent::EventTime))
125
+ assert(emits[1][1].is_a?(Fluent::EventTime))
126
+ assert_equal(1, d.emit_streams.size)
127
+ end
128
+
129
+ class TestWithSystem < self
130
+ include Fluent::SystemConfig::Mixin
131
+
132
+ OVERRIDE_FILE_PERMISSION = 0620
133
+ CONFIG_SYSTEM = %[
134
+ <system>
135
+ file_permission #{OVERRIDE_FILE_PERMISSION}
136
+ </system>
137
+ ]
138
+
139
+ def setup
140
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
141
+ # Store default permission
142
+ @default_permission = system_config.instance_variable_get(:@file_permission)
143
+ end
144
+
145
+ def teardown
146
+ # Restore default permission
147
+ system_config.instance_variable_set(:@file_permission, @default_permission)
148
+ end
149
+
150
+ def parse_system(text)
151
+ basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
152
+ Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
153
+ end
154
+
155
+ def test_emit_with_system
156
+ system_conf = parse_system(CONFIG_SYSTEM)
157
+ sc = Fluent::SystemConfig.new(system_conf)
158
+ Fluent::Engine.init(sc)
159
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
160
+ f.puts "test1"
161
+ f.puts "test2"
162
+ }
163
+
164
+ d = create_driver
165
+
166
+ d.run do
167
+ sleep 1
168
+
169
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
170
+ f.puts "test3"
171
+ f.puts "test4"
172
+ }
173
+ sleep 1
174
+ end
175
+
176
+ emits = d.emits
177
+ assert_equal(true, emits.length > 0)
178
+ assert_equal({"message" => "test3"}, emits[0][2])
179
+ assert_equal({"message" => "test4"}, emits[1][2])
180
+ assert(emits[0][1].is_a?(Fluent::EventTime))
181
+ assert(emits[1][1].is_a?(Fluent::EventTime))
182
+ assert_equal(1, d.emit_streams.size)
183
+ pos = d.instance.instance_variable_get(:@pf_file)
184
+ mode = "%o" % File.stat(pos).mode
185
+ assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
186
+ end
187
+ end
188
+
189
+ data('1' => [1, 2], '10' => [10, 1])
190
+ def test_emit_with_read_lines_limit(data)
191
+ limit, num_emits = data
192
+ d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
193
+ msg = 'test' * 500 # in_tail reads 2048 bytes at once.
194
+
195
+ d.run do
196
+ sleep 1
197
+
198
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
199
+ f.puts msg
200
+ f.puts msg
201
+ }
202
+ sleep 1
203
+ end
204
+
205
+ emits = d.emits
206
+ assert_equal(true, emits.length > 0)
207
+ assert_equal({"message" => msg}, emits[0][2])
208
+ assert_equal({"message" => msg}, emits[1][2])
209
+ assert_equal(num_emits, d.emit_streams.size)
210
+ end
211
+
212
+ def test_emit_with_read_from_head
213
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
214
+ f.puts "test1"
215
+ f.puts "test2"
216
+ }
217
+
218
+ d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
219
+
220
+ d.run do
221
+ sleep 1
222
+
223
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
224
+ f.puts "test3"
225
+ f.puts "test4"
226
+ }
227
+ sleep 1
228
+ end
229
+
230
+ emits = d.emits
231
+ assert(emits.length > 0)
232
+ assert_equal({"message" => "test1"}, emits[0][2])
233
+ assert_equal({"message" => "test2"}, emits[1][2])
234
+ assert_equal({"message" => "test3"}, emits[2][2])
235
+ assert_equal({"message" => "test4"}, emits[3][2])
236
+ end
237
+
238
+ def test_emit_with_enable_watch_timer
239
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
240
+ f.puts "test1"
241
+ f.puts "test2"
242
+ }
243
+
244
+ d = create_driver(CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG)
245
+
246
+ d.run do
247
+ sleep 1
248
+
249
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
250
+ f.puts "test3"
251
+ f.puts "test4"
252
+ }
253
+ # according to cool.io's stat_watcher.c, systems without inotify will use
254
+ # an "automatic" value, typically around 5 seconds
255
+ sleep 10
256
+ end
257
+
258
+ emits = d.emits
259
+ assert(emits.length > 0)
260
+ assert_equal({"message" => "test3"}, emits[0][2])
261
+ assert_equal({"message" => "test4"}, emits[1][2])
262
+ end
263
+
264
+ def test_rotate_file
265
+ emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
266
+ assert_equal(4, emits.length)
267
+ assert_equal({"message" => "test3"}, emits[0][2])
268
+ assert_equal({"message" => "test4"}, emits[1][2])
269
+ assert_equal({"message" => "test5"}, emits[2][2])
270
+ assert_equal({"message" => "test6"}, emits[3][2])
271
+ end
272
+
273
+ def test_rotate_file_with_read_from_head
274
+ emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
275
+ assert_equal(6, emits.length)
276
+ assert_equal({"message" => "test1"}, emits[0][2])
277
+ assert_equal({"message" => "test2"}, emits[1][2])
278
+ assert_equal({"message" => "test3"}, emits[2][2])
279
+ assert_equal({"message" => "test4"}, emits[3][2])
280
+ assert_equal({"message" => "test5"}, emits[4][2])
281
+ assert_equal({"message" => "test6"}, emits[5][2])
282
+ end
283
+
284
+ def test_rotate_file_with_write_old
285
+ emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
286
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
287
+ rotated_file.puts "test7"
288
+ rotated_file.puts "test8"
289
+ rotated_file.flush
290
+
291
+ sleep 1
292
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
293
+ f.puts "test5"
294
+ f.puts "test6"
295
+ }
296
+ }
297
+ assert_equal(6, emits.length)
298
+ assert_equal({"message" => "test3"}, emits[0][2])
299
+ assert_equal({"message" => "test4"}, emits[1][2])
300
+ assert_equal({"message" => "test7"}, emits[2][2])
301
+ assert_equal({"message" => "test8"}, emits[3][2])
302
+ assert_equal({"message" => "test5"}, emits[4][2])
303
+ assert_equal({"message" => "test6"}, emits[5][2])
304
+ end
305
+
306
+ def test_rotate_file_with_write_old_and_no_new_file
307
+ emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
308
+ rotated_file.puts "test7"
309
+ rotated_file.puts "test8"
310
+ rotated_file.flush
311
+ }
312
+ assert_equal(4, emits.length)
313
+ assert_equal({"message" => "test3"}, emits[0][2])
314
+ assert_equal({"message" => "test4"}, emits[1][2])
315
+ assert_equal({"message" => "test7"}, emits[2][2])
316
+ assert_equal({"message" => "test8"}, emits[3][2])
317
+ end
318
+
319
+ def sub_test_rotate_file(config = nil)
320
+ file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
321
+ file.puts "test1"
322
+ file.puts "test2"
323
+ file.flush
324
+
325
+ d = create_driver(config)
326
+ d.run do
327
+ sleep 1
328
+
329
+ file.puts "test3"
330
+ file.puts "test4"
331
+ file.flush
332
+ sleep 1
333
+
334
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
335
+ if block_given?
336
+ yield file
337
+ sleep 1
338
+ else
339
+ sleep 1
340
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
341
+ sleep 1
342
+
343
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
344
+ f.puts "test5"
345
+ f.puts "test6"
346
+ }
347
+ sleep 1
348
+ end
349
+ end
350
+
351
+ d.run do
352
+ sleep 1
353
+ end
354
+
355
+ d.emits
356
+ ensure
357
+ file.close if file
358
+ end
359
+
360
+ def test_lf
361
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
362
+
363
+ d = create_driver
364
+
365
+ d.run do
366
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
367
+ f.print "test3"
368
+ }
369
+ sleep 1
370
+
371
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
372
+ f.puts "test4"
373
+ }
374
+ sleep 1
375
+ end
376
+
377
+ emits = d.emits
378
+ assert_equal(true, emits.length > 0)
379
+ assert_equal({"message" => "test3test4"}, emits[0][2])
380
+ end
381
+
382
+ def test_whitespace
383
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
384
+
385
+ d = create_driver
386
+
387
+ d.run do
388
+ sleep 1
389
+
390
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
391
+ f.puts " " # 4 spaces
392
+ f.puts " 4 spaces"
393
+ f.puts "4 spaces "
394
+ f.puts " " # tab
395
+ f.puts " tab"
396
+ f.puts "tab "
397
+ }
398
+ sleep 1
399
+ end
400
+
401
+ emits = d.emits
402
+ assert_equal(true, emits.length > 0)
403
+ assert_equal({"message" => " "}, emits[0][2])
404
+ assert_equal({"message" => " 4 spaces"}, emits[1][2])
405
+ assert_equal({"message" => "4 spaces "}, emits[2][2])
406
+ assert_equal({"message" => " "}, emits[3][2])
407
+ assert_equal({"message" => " tab"}, emits[4][2])
408
+ assert_equal({"message" => "tab "}, emits[5][2])
409
+ end
410
+
411
+ data(
412
+ 'default encoding' => ['', Encoding::ASCII_8BIT],
413
+ 'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
414
+ def test_encoding(data)
415
+ encoding_config, encoding = data
416
+
417
+ d = create_driver(SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD + encoding_config)
418
+
419
+ d.run do
420
+ sleep 1
421
+
422
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
423
+ f.puts "test"
424
+ }
425
+ sleep 1
426
+ end
427
+
428
+ emits = d.emits
429
+ assert_equal(encoding, emits[0][2]['message'].encoding)
430
+ end
431
+
432
+ def test_from_encoding
433
+ d = create_driver %[
434
+ format /(?<message>.*)/
435
+ read_from_head true
436
+ from_encoding cp932
437
+ encoding utf-8
438
+ ]
439
+
440
+ d.run do
441
+ sleep 1
442
+
443
+ File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
444
+ f.puts "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
445
+ }
446
+ sleep 1
447
+ end
448
+
449
+ emits = d.emits
450
+ assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message'])
451
+ assert_equal(Encoding::UTF_8, emits[0][2]['message'].encoding)
452
+ end
453
+
454
+ # multiline mode test
455
+
456
+ def test_multiline
457
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
458
+
459
+ d = create_driver %[
460
+ format multiline
461
+ format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
462
+ format_firstline /^[s]/
463
+ ]
464
+ d.run do
465
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
466
+ f.puts "f test1"
467
+ f.puts "s test2"
468
+ f.puts "f test3"
469
+ f.puts "f test4"
470
+ f.puts "s test5"
471
+ f.puts "s test6"
472
+ f.puts "f test7"
473
+ f.puts "s test8"
474
+ }
475
+ sleep 1
476
+
477
+ emits = d.emits
478
+ assert(emits.length == 3)
479
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
480
+ assert_equal({"message1" => "test5"}, emits[1][2])
481
+ assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
482
+
483
+ sleep 3
484
+ emits = d.emits
485
+ assert(emits.length == 3)
486
+ end
487
+
488
+ emits = d.emits
489
+ assert(emits.length == 4)
490
+ assert_equal({"message1" => "test8"}, emits[3][2])
491
+ end
492
+
493
+ def test_multiline_with_flush_interval
494
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
495
+
496
+ d = create_driver %[
497
+ format multiline
498
+ format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
499
+ format_firstline /^[s]/
500
+ multiline_flush_interval 2s
501
+ ]
502
+
503
+ assert_equal 2, d.instance.multiline_flush_interval
504
+
505
+ d.run do
506
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
507
+ f.puts "f test1"
508
+ f.puts "s test2"
509
+ f.puts "f test3"
510
+ f.puts "f test4"
511
+ f.puts "s test5"
512
+ f.puts "s test6"
513
+ f.puts "f test7"
514
+ f.puts "s test8"
515
+ }
516
+ sleep 1
517
+
518
+ emits = d.emits
519
+ assert(emits.length == 3)
520
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
521
+ assert_equal({"message1" => "test5"}, emits[1][2])
522
+ assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
523
+
524
+ sleep 3
525
+ emits = d.emits
526
+ assert(emits.length == 4)
527
+ assert_equal({"message1" => "test8"}, emits[3][2])
528
+ end
529
+ end
530
+
531
+ data(
532
+ 'default encoding' => ['', Encoding::ASCII_8BIT],
533
+ 'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
534
+ def test_multiline_encoding_of_flushed_record(data)
535
+ encoding_config, encoding = data
536
+
537
+ d = create_driver %[
538
+ format multiline
539
+ format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
540
+ format_firstline /^[s]/
541
+ multiline_flush_interval 2s
542
+ read_from_head true
543
+ #{encoding_config}
544
+ ]
545
+
546
+ d.run do
547
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
548
+ f.puts "s test"
549
+ }
550
+
551
+ sleep 4
552
+ emits = d.emits
553
+ assert_equal(1, emits.length)
554
+ assert_equal(encoding, emits[0][2]['message1'].encoding)
555
+ end
556
+ end
557
+
558
+ def test_multiline_from_encoding_of_flushed_record
559
+ d = create_driver %[
560
+ format multiline
561
+ format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
562
+ format_firstline /^[s]/
563
+ multiline_flush_interval 2s
564
+ read_from_head true
565
+ from_encoding cp932
566
+ encoding utf-8
567
+ ]
568
+
569
+ d.run do
570
+ sleep 1
571
+ File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
572
+ f.puts "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
573
+ }
574
+
575
+ sleep 4
576
+ emits = d.emits
577
+ assert_equal(1, emits.length)
578
+ assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message1'])
579
+ assert_equal(Encoding::UTF_8, emits[0][2]['message1'].encoding)
580
+ end
581
+ end
582
+
583
+ def test_multiline_with_multiple_formats
584
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
585
+
586
+ d = create_driver %[
587
+ format multiline
588
+ format1 /^s (?<message1>[^\\n]+)\\n?/
589
+ format2 /(f (?<message2>[^\\n]+)\\n?)?/
590
+ format3 /(f (?<message3>.*))?/
591
+ format_firstline /^[s]/
592
+ ]
593
+ d.run do
594
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
595
+ f.puts "f test1"
596
+ f.puts "s test2"
597
+ f.puts "f test3"
598
+ f.puts "f test4"
599
+ f.puts "s test5"
600
+ f.puts "s test6"
601
+ f.puts "f test7"
602
+ f.puts "s test8"
603
+ }
604
+ sleep 1
605
+ end
606
+
607
+ emits = d.emits
608
+ assert(emits.length > 0)
609
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
610
+ assert_equal({"message1" => "test5"}, emits[1][2])
611
+ assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
612
+ assert_equal({"message1" => "test8"}, emits[3][2])
613
+ end
614
+
615
+ def test_multilinelog_with_multiple_paths
616
+ files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
617
+ files.each { |file| File.open(file, "wb") { |f| } }
618
+
619
+ d = create_driver(%[
620
+ path #{files[0]},#{files[1]}
621
+ tag t1
622
+ format multiline
623
+ format1 /^[s|f] (?<message>.*)/
624
+ format_firstline /^[s]/
625
+ ], false)
626
+ d.run do
627
+ files.each do |file|
628
+ File.open(file, 'ab') { |f|
629
+ f.puts "f #{file} line should be ignored"
630
+ f.puts "s test1"
631
+ f.puts "f test2"
632
+ f.puts "f test3"
633
+ f.puts "s test4"
634
+ }
635
+ end
636
+ sleep 1
637
+ end
638
+
639
+ emits = d.emits
640
+ assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
641
+ assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
642
+ # "test4" events are here because these events are flushed at shutdown phase
643
+ assert_equal({"message" => "test4"}, emits[2][2])
644
+ assert_equal({"message" => "test4"}, emits[3][2])
645
+ end
646
+
647
+ def test_multiline_without_firstline
648
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
649
+
650
+ d = create_driver %[
651
+ format multiline
652
+ format1 /(?<var1>foo \\d)\\n/
653
+ format2 /(?<var2>bar \\d)\\n/
654
+ format3 /(?<var3>baz \\d)/
655
+ ]
656
+ d.run do
657
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
658
+ f.puts "foo 1"
659
+ f.puts "bar 1"
660
+ f.puts "baz 1"
661
+ f.puts "foo 2"
662
+ f.puts "bar 2"
663
+ f.puts "baz 2"
664
+ }
665
+ sleep 1
666
+ end
667
+
668
+ emits = d.emits
669
+ assert_equal(2, emits.length)
670
+ assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, emits[0][2])
671
+ assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
672
+ end
673
+
674
+ # * path test
675
+ # TODO: Clean up tests
676
+ EX_RORATE_WAIT = 0
677
+
678
+ EX_CONFIG = %[
679
+ tag tail
680
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
681
+ format none
682
+ pos_file #{TMP_DIR}/tail.pos
683
+ read_from_head true
684
+ refresh_interval 30
685
+ rotate_wait #{EX_RORATE_WAIT}s
686
+ ]
687
+ EX_PATHS = [
688
+ 'test/plugin/data/2010/01/20100102-030405.log',
689
+ 'test/plugin/data/log/foo/bar.log',
690
+ 'test/plugin/data/log/test.log'
691
+ ]
692
+
693
+ def test_expand_paths
694
+ plugin = create_driver(EX_CONFIG, false).instance
695
+ flexstub(Time) do |timeclass|
696
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
697
+ assert_equal EX_PATHS, plugin.expand_paths.sort
698
+ end
699
+
700
+ # Test exclusion
701
+ exclude_config = EX_CONFIG + " exclude_path [\"#{EX_PATHS.last}\"]"
702
+ plugin = create_driver(exclude_config, false).instance
703
+ assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
704
+ end
705
+
706
+ def test_z_refresh_watchers
707
+ plugin = create_driver(EX_CONFIG, false).instance
708
+ sio = StringIO.new
709
+ plugin.instance_eval do
710
+ @pf = Fluent::NewTailInput::PositionFile.parse(sio)
711
+ @loop = Coolio::Loop.new
712
+ end
713
+
714
+ flexstub(Time) do |timeclass|
715
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
716
+
717
+ flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
718
+ EX_PATHS.each do |path|
719
+ watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
720
+ flexmock('TailWatcher') { |watcher|
721
+ watcher.should_receive(:attach).once
722
+ watcher.should_receive(:unwatched=).zero_or_more_times
723
+ watcher.should_receive(:line_buffer).zero_or_more_times
724
+ }
725
+ end
726
+ end
727
+ plugin.refresh_watchers
728
+ end
729
+
730
+ plugin.instance_eval do
731
+ @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
732
+ end
733
+
734
+ flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
735
+ watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
736
+ flexmock('TailWatcher') do |watcher|
737
+ watcher.should_receive(:attach).once
738
+ watcher.should_receive(:unwatched=).zero_or_more_times
739
+ watcher.should_receive(:line_buffer).zero_or_more_times
740
+ end
741
+ end
742
+ plugin.refresh_watchers
743
+ end
744
+
745
+ flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
746
+ watcherclass.should_receive(:new).never
747
+ plugin.refresh_watchers
748
+ end
749
+ end
750
+ end
751
+
752
+ DummyWatcher = Struct.new("DummyWatcher", :tag)
753
+
754
+ def test_receive_lines
755
+ plugin = create_driver(EX_CONFIG, false).instance
756
+ flexstub(plugin.router) do |engineclass|
757
+ engineclass.should_receive(:emit_stream).with('tail', any).once
758
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
759
+ end
760
+
761
+ config = %[
762
+ tag pre.*
763
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
764
+ format none
765
+ read_from_head true
766
+ ]
767
+ plugin = create_driver(config, false).instance
768
+ flexstub(plugin.router) do |engineclass|
769
+ engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
770
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
771
+ end
772
+
773
+ config = %[
774
+ tag *.post
775
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
776
+ format none
777
+ read_from_head true
778
+ ]
779
+ plugin = create_driver(config, false).instance
780
+ flexstub(plugin.router) do |engineclass|
781
+ engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
782
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
783
+ end
784
+
785
+ config = %[
786
+ tag pre.*.post
787
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
788
+ format none
789
+ read_from_head true
790
+ ]
791
+ plugin = create_driver(config, false).instance
792
+ flexstub(plugin.router) do |engineclass|
793
+ engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
794
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
795
+ end
796
+
797
+ config = %[
798
+ tag pre.*.post*ignore
799
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
800
+ format none
801
+ read_from_head true
802
+ ]
803
+ plugin = create_driver(config, false).instance
804
+ flexstub(plugin.router) do |engineclass|
805
+ engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
806
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
807
+ end
808
+ end
809
+
810
+ # Ensure that no fatal exception is raised when a file is missing and that
811
+ # files that do exist are still tailed as expected.
812
+ def test_missing_file
813
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
814
+ f.puts "test1"
815
+ f.puts "test2"
816
+ }
817
+
818
+ # Try two different configs - one with read_from_head and one without,
819
+ # since their interactions with the filesystem differ.
820
+ config1 = %[
821
+ tag t1
822
+ path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
823
+ format none
824
+ rotate_wait 2s
825
+ pos_file #{TMP_DIR}/tail.pos
826
+ ]
827
+ config2 = config1 + ' read_from_head true'
828
+ [config1, config2].each do |config|
829
+ d = create_driver(config, false)
830
+ d.run do
831
+ sleep 1
832
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
833
+ f.puts "test3"
834
+ f.puts "test4"
835
+ }
836
+ sleep 1
837
+ end
838
+ emits = d.emits
839
+ assert_equal(2, emits.length)
840
+ assert_equal({"message" => "test3"}, emits[0][2])
841
+ assert_equal({"message" => "test4"}, emits[1][2])
842
+ end
843
+ end
844
+
845
+ sub_test_case 'emit error cases' do
846
+ def test_emit_error_with_buffer_queue_limit_error
847
+ emits = execute_test(Fluent::Plugin::Buffer::BufferOverflowError, "buffer space has too many data")
848
+ assert_equal(10, emits.length)
849
+ 10.times { |i|
850
+ assert_equal({"message" => "test#{i}"}, emits[i][2])
851
+ }
852
+ end
853
+
854
+ def test_emit_error_with_non_buffer_queue_limit_error
855
+ emits = execute_test(StandardError, "non BufferQueueLimitError error")
856
+ assert_true(emits.size > 0 && emits.size != 10)
857
+ emits.size.times { |i|
858
+ assert_equal({"message" => "test#{10 - emits.size + i}"}, emits[i][2])
859
+ }
860
+ end
861
+
862
+ def execute_test(error_class, error_message)
863
+ d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
864
+ # Use define_singleton_method instead of d.emit_stream to capture local variable
865
+ d.define_singleton_method(:emit_stream) do |tag, es|
866
+ @test_num_errors ||= 0
867
+ if @test_num_errors < 5
868
+ @test_num_errors += 1
869
+ raise error_class, error_message
870
+ else
871
+ @emit_streams << [tag, es.to_a]
872
+ end
873
+ end
874
+
875
+ d.run do
876
+ 10.times { |i|
877
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test#{i}" }
878
+ sleep 0.5
879
+ }
880
+ sleep 1
881
+ end
882
+
883
+ d.emits
884
+ end
885
+ end
886
+
887
+ sub_test_case "tail_path" do
888
+ def test_tail_path_with_singleline
889
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
890
+ f.puts "test1"
891
+ f.puts "test2"
892
+ }
893
+
894
+ d = create_driver(%[path_key path] + SINGLE_LINE_CONFIG)
895
+
896
+ d.run do
897
+ sleep 1
898
+
899
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
900
+ f.puts "test3"
901
+ f.puts "test4"
902
+ }
903
+ sleep 1
904
+ end
905
+
906
+ emits = d.emits
907
+ assert_equal(true, emits.length > 0)
908
+ emits.each do |emit|
909
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
910
+ end
911
+ end
912
+
913
+ def test_tail_path_with_multiline_with_firstline
914
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
915
+
916
+ d = create_driver %[
917
+ path_key path
918
+ format multiline
919
+ format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
920
+ format_firstline /^[s]/
921
+ ]
922
+ d.run do
923
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
924
+ f.puts "f test1"
925
+ f.puts "s test2"
926
+ f.puts "f test3"
927
+ f.puts "f test4"
928
+ f.puts "s test5"
929
+ f.puts "s test6"
930
+ f.puts "f test7"
931
+ f.puts "s test8"
932
+ }
933
+ sleep 1
934
+ end
935
+
936
+ emits = d.emits
937
+ assert(emits.length == 4)
938
+ emits.each do |emit|
939
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
940
+ end
941
+ end
942
+
943
+ def test_tail_path_with_multiline_without_firstline
944
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
945
+
946
+ d = create_driver %[
947
+ path_key path
948
+ format multiline
949
+ format1 /(?<var1>foo \\d)\\n/
950
+ format2 /(?<var2>bar \\d)\\n/
951
+ format3 /(?<var3>baz \\d)/
952
+ ]
953
+ d.run do
954
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
955
+ f.puts "foo 1"
956
+ f.puts "bar 1"
957
+ f.puts "baz 1"
958
+ }
959
+ sleep 1
960
+ end
961
+
962
+ emits = d.emits
963
+ assert(emits.length > 0)
964
+ emits.each do |emit|
965
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
966
+ end
967
+ end
968
+
969
+ def test_tail_path_with_multiline_with_multiple_paths
970
+ files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
971
+ files.each { |file| File.open(file, "wb") { |f| } }
972
+
973
+ d = create_driver(%[
974
+ path #{files[0]},#{files[1]}
975
+ path_key path
976
+ tag t1
977
+ format multiline
978
+ format1 /^[s|f] (?<message>.*)/
979
+ format_firstline /^[s]/
980
+ ], false)
981
+ d.run do
982
+ files.each do |file|
983
+ File.open(file, 'ab') { |f|
984
+ f.puts "f #{file} line should be ignored"
985
+ f.puts "s test1"
986
+ f.puts "f test2"
987
+ f.puts "f test3"
988
+ f.puts "s test4"
989
+ }
990
+ end
991
+ sleep 1
992
+ end
993
+
994
+ emits = d.emits
995
+ assert(emits.length == 4)
996
+ assert_equal(files, [emits[0][2]["path"], emits[1][2]["path"]].sort)
997
+ # "test4" events are here because these events are flushed at shutdown phase
998
+ assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
999
+ end
1000
+ end
1001
+ end