fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,66 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+
5
+ class SyslogParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::SyslogParser)
9
+ @expected = {
10
+ 'host' => '192.168.0.1',
11
+ 'ident' => 'fluentd',
12
+ 'pid' => '11111',
13
+ 'message' => '[error] Syslog test'
14
+ }
15
+ end
16
+
17
+ def test_parse
18
+ @parser.configure({})
19
+ @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
20
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
21
+ assert_equal(@expected, record)
22
+ }
23
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
24
+ assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
25
+ end
26
+
27
+ def test_parse_with_time_format
28
+ @parser.configure('time_format' => '%b %d %M:%S:%H')
29
+ @parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
30
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
31
+ assert_equal(@expected, record)
32
+ }
33
+ assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
34
+ end
35
+
36
+ def test_parse_with_priority
37
+ @parser.configure('with_priority' => true)
38
+ @parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
39
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
40
+ assert_equal(@expected.merge('pri' => 6), record)
41
+ }
42
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
43
+ assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
44
+ end
45
+
46
+ def test_parse_without_colon
47
+ @parser.configure({})
48
+ @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
49
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
50
+ assert_equal(@expected, record)
51
+ }
52
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
53
+ assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
54
+ end
55
+
56
+ def test_parse_with_keep_time_key
57
+ @parser.configure(
58
+ 'time_format' => '%b %d %M:%S:%H',
59
+ 'keep_time_key'=>'true',
60
+ )
61
+ text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
62
+ @parser.instance.parse(text) do |time, record|
63
+ assert_equal "Feb 28 00:00:12", record['time']
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,46 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+
5
+ class TimeParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def test_call_with_parse
11
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
12
+
13
+ assert(parser.instance.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))
14
+
15
+ time = event_time('2013-09-18 12:00:00 +0900')
16
+ assert_equal(time, parser.instance.parse('2013-09-18 12:00:00 +0900'))
17
+ end
18
+
19
+ def test_parse_with_strptime
20
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z'))
21
+
22
+ assert(parser.instance.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))
23
+
24
+ time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
25
+ assert_equal(time, parser.instance.parse('28/Feb/2013:12:00:00 +0900'))
26
+ end
27
+
28
+ def test_parse_nsec_with_strptime
29
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z'))
30
+
31
+ assert(parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))
32
+
33
+ time = event_time('28/Feb/2013:12:00:00:123456789 +0900', format: '%d/%b/%Y:%H:%M:%S:%N %z')
34
+ assert_equal_event_time(time, parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900'))
35
+ end
36
+
37
+ def test_parse_with_invalid_argument
38
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
39
+
40
+ [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
41
+ assert_raise Fluent::ParserError do
42
+ parser.instance.parse(v)
43
+ end
44
+ }
45
+ end
46
+ end
@@ -0,0 +1,121 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser_tsv'
4
+
5
+ class TSVParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def create_driver(conf={})
11
+ Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser).configure(conf)
12
+ end
13
+
14
+ data('array param' => '["a","b"]', 'string param' => 'a,b')
15
+ def test_config_params(param)
16
+ d = create_driver
17
+
18
+ assert_equal "\t", d.instance.delimiter
19
+
20
+ d = create_driver(
21
+ 'keys' => param,
22
+ 'delimiter' => ',',
23
+ )
24
+
25
+ assert_equal ['a', 'b'], d.instance.keys
26
+ assert_equal ",", d.instance.delimiter
27
+ end
28
+
29
+ data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
30
+ def test_parse(param)
31
+ d = create_driver('keys' => param, 'time_key' => 'time')
32
+ d.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
33
+ assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
34
+ assert_equal({
35
+ 'a' => '192.168.0.1',
36
+ 'b' => '111',
37
+ }, record)
38
+ }
39
+ end
40
+
41
+ def test_parse_with_time
42
+ time_at_start = Time.now.to_i
43
+
44
+ d = create_driver('keys' => 'a,b')
45
+ d.instance.parse("192.168.0.1\t111") { |time, record|
46
+ assert time && time >= time_at_start, "parser puts current time without time input"
47
+ assert_equal({
48
+ 'a' => '192.168.0.1',
49
+ 'b' => '111',
50
+ }, record)
51
+ }
52
+
53
+ d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
54
+ d.instance.estimate_current_event = false
55
+ d.configure('keys' => 'a,b', 'time_key' => 'time')
56
+ d.instance.parse("192.168.0.1\t111") { |time, record|
57
+ assert_equal({
58
+ 'a' => '192.168.0.1',
59
+ 'b' => '111',
60
+ }, record)
61
+ assert_nil time, "parser returns nil w/o time and if configured so"
62
+ }
63
+ end
64
+
65
+ data(
66
+ 'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
67
+ 'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
68
+ 'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
69
+ '2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
70
+ 'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
71
+ 'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
72
+ def test_black_column(data)
73
+ line, expected = data
74
+
75
+ d = create_driver('keys' => '1,2,3')
76
+ d.instance.parse(line) { |time, record|
77
+ assert_equal(expected, record)
78
+ }
79
+ end
80
+
81
+ def test_parse_with_keep_time_key
82
+ d = create_driver(
83
+ 'keys'=>'time',
84
+ 'time_key'=>'time',
85
+ 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
86
+ 'keep_time_key'=>'true',
87
+ )
88
+ text = '28/Feb/2013:12:00:00 +0900'
89
+ d.instance.parse(text) do |time, record|
90
+ assert_equal text, record['time']
91
+ end
92
+ end
93
+
94
+ data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
95
+ def test_parse_with_null_value_pattern(param)
96
+ d = create_driver(
97
+ 'keys'=>param,
98
+ 'null_value_pattern'=>'^(-|null|NULL)$'
99
+ )
100
+ d.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
101
+ assert_nil record['a']
102
+ assert_nil record['b']
103
+ assert_nil record['c']
104
+ assert_equal record['d'], ''
105
+ assert_equal record['e'], '--'
106
+ assert_equal record['f'], 'nuLL'
107
+ end
108
+ end
109
+
110
+ data('array param' => '["a","b"]', 'string param' => 'a,b')
111
+ def test_parse_with_null_empty_string(param)
112
+ d = create_driver(
113
+ 'keys'=>param,
114
+ 'null_empty_string'=>true
115
+ )
116
+ d.instance.parse("\t ") do |time, record|
117
+ assert_nil record['a']
118
+ assert_equal record['b'], ' '
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,167 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/storage'
3
+ require 'fluent/plugin/base'
4
+
5
+ class DummyPlugin < Fluent::Plugin::TestBase
6
+ end
7
+
8
+ class BareStorage < Fluent::Plugin::Storage
9
+ Fluent::Plugin.register_storage('bare', self)
10
+ end
11
+
12
+ class BasicStorage < Fluent::Plugin::Storage
13
+ Fluent::Plugin.register_storage('example', self)
14
+
15
+ attr_reader :data, :saved
16
+
17
+ def initialize
18
+ super
19
+ @data = @saved = nil
20
+ end
21
+ def load
22
+ @data = {}
23
+ end
24
+ def save
25
+ @saved = @data.dup
26
+ end
27
+ def get(key)
28
+ @data[key]
29
+ end
30
+ def fetch(key, defval)
31
+ @data.fetch(key, defval)
32
+ end
33
+ def put(key, value)
34
+ @data[key] = value
35
+ end
36
+ def delete(key)
37
+ @data.delete(key)
38
+ end
39
+ def update(key, &block)
40
+ @data[key] = block.call(@data[key])
41
+ end
42
+ def close
43
+ @data = {}
44
+ super
45
+ end
46
+ def terminate
47
+ @saved = {}
48
+ super
49
+ end
50
+ end
51
+
52
+ class StorageTest < Test::Unit::TestCase
53
+ sub_test_case 'BareStorage' do
54
+ setup do
55
+ plugin = DummyPlugin.new
56
+ @s = BareStorage.new
57
+ @s.configure(config_element())
58
+ @s.owner = plugin
59
+ end
60
+
61
+ test 'is configured with plugin information and system config' do
62
+ plugin = DummyPlugin.new
63
+ plugin.system_config_override({'process_name' => 'mytest'})
64
+ plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
65
+ s = BareStorage.new
66
+ s.configure(config_element())
67
+ s.owner = plugin
68
+
69
+ assert_equal 'mytest', s.owner.system_config.process_name
70
+ assert_equal '1', s.instance_eval{ @_plugin_id }
71
+ assert_equal true, s.instance_eval{ @_plugin_id_configured }
72
+ end
73
+
74
+ test 'does NOT have features for high-performance/high-consistent storages' do
75
+ assert_equal false, @s.persistent_always?
76
+ assert_equal false, @s.synchronized?
77
+ end
78
+
79
+ test 'does have default values which is conservative for almost all users' do
80
+ assert_equal false, @s.persistent
81
+ assert_equal true, @s.autosave
82
+ assert_equal 10, @s.autosave_interval
83
+ assert_equal true, @s.save_at_shutdown
84
+ end
85
+
86
+ test 'load/save doesn NOT anything: just as memory storage' do
87
+ assert_nothing_raised{ @s.load }
88
+ assert_nothing_raised{ @s.save }
89
+ end
90
+
91
+ test 'all operations are not defined yet' do
92
+ assert_raise NotImplementedError do
93
+ @s.get('key')
94
+ end
95
+ assert_raise NotImplementedError do
96
+ @s.fetch('key', 'value')
97
+ end
98
+ assert_raise NotImplementedError do
99
+ @s.put('key', 'value')
100
+ end
101
+ assert_raise NotImplementedError do
102
+ @s.delete('key')
103
+ end
104
+ assert_raise NotImplementedError do
105
+ @s.update('key'){ |v| v + '2' }
106
+ end
107
+ end
108
+ end
109
+
110
+ sub_test_case 'ExampleStorage' do
111
+ setup do
112
+ plugin = DummyPlugin.new
113
+ plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
114
+ @s = BasicStorage.new
115
+ @s.configure(config_element())
116
+ @s.owner = plugin
117
+ end
118
+
119
+ test 'load/save works well as plugin internal state operations' do
120
+ plugin = DummyPlugin.new
121
+ plugin.configure(config_element('ROOT', '', {'@id' => '0'}))
122
+ s = BasicStorage.new
123
+ s.owner = plugin
124
+
125
+ assert_nothing_raised{ s.load }
126
+ assert s.data
127
+ assert_nil s.saved
128
+
129
+ assert_nothing_raised{ s.save }
130
+ assert s.saved
131
+ assert{ s.data == s.saved }
132
+ assert{ s.data.object_id != s.saved.object_id }
133
+ end
134
+
135
+ test 'all operations work well' do
136
+ @s.load
137
+
138
+ assert_nil @s.get('key')
139
+ assert_equal 'value', @s.fetch('key', 'value')
140
+ assert_nil @s.get('key')
141
+
142
+ assert_equal 'value', @s.put('key', 'value')
143
+ assert_equal 'value', @s.get('key')
144
+
145
+ assert_equal 'valuevalue', @s.update('key'){|v| v * 2 }
146
+
147
+ assert_equal 'valuevalue', @s.delete('key')
148
+ end
149
+
150
+ test 'close and terminate work to operate internal states' do
151
+ @s.load
152
+ @s.put('k1', 'v1')
153
+ @s.put('k2', 'v2')
154
+ assert_equal 2, @s.data.size
155
+ @s.save
156
+ assert_equal @s.data.size, @s.saved.size
157
+
158
+ assert_nothing_raised{ @s.close }
159
+ assert @s.data.empty?
160
+ assert !@s.saved.empty?
161
+
162
+ assert_nothing_raised{ @s.terminate }
163
+ assert @s.data.empty?
164
+ assert @s.saved.empty?
165
+ end
166
+ end
167
+ end
@@ -0,0 +1,8 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/storage_local'
3
+
4
+ class LocalStorageTest < Test::Unit::TestCase
5
+ test 'syntax' do
6
+ assert true
7
+ end
8
+ end
@@ -0,0 +1,26 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/string_util'
3
+
4
+ class StringUtilTest < Test::Unit::TestCase
5
+ def setup
6
+ @null_value_pattern = Regexp.new("^(-|null|NULL)$")
7
+ end
8
+
9
+ sub_test_case 'valid string' do
10
+ test 'null string' do
11
+ assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "null").to_s, "null"
12
+ assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "NULL").to_s, "NULL"
13
+ assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "-").to_s, "-"
14
+ end
15
+
16
+ test 'normal string' do
17
+ assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "fluentd"), nil
18
+ end
19
+ end
20
+
21
+ sub_test_case 'invalid string' do
22
+ test 'normal string' do
23
+ assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "\xff"), nil
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,608 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/child_process'
3
+ require 'fluent/plugin/base'
4
+ require 'timeout'
5
+
6
+ class ChildProcessTest < Test::Unit::TestCase
7
+ TEST_DEADLOCK_TIMEOUT = 30
8
+ TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING = 0.1 # This may be shorter than ruby's threading timer, but work well
9
+ # @nalsh says that ruby's cpu assignments for threads are almost 200ms or so.
10
+ # Loop interval (expected that it work as specified) should be longer than it.
11
+ TEST_WAIT_INTERVAL_FOR_LOOP = 0.5
12
+
13
+ setup do
14
+ @d = Dummy.new
15
+ @d.configure(config_element())
16
+ @d.start
17
+ end
18
+
19
+ teardown do
20
+ if @d
21
+ @d.stop unless @d.stopped?
22
+ @d.shutdown unless @d.shutdown?
23
+ @d.close unless @d.closed?
24
+ @d.terminate unless @d.terminated?
25
+ end
26
+ end
27
+
28
+ class Dummy < Fluent::Plugin::TestBase
29
+ helpers :child_process
30
+ def configure(conf)
31
+ super
32
+ @_child_process_kill_timeout = 1
33
+ end
34
+ end
35
+
36
+ test 'can be instantiated' do
37
+ d1 = Dummy.new
38
+ assert d1.respond_to?(:_child_process_processes)
39
+ end
40
+
41
+ test 'can be configured and started' do
42
+ d1 = Dummy.new
43
+ assert_nothing_raised do
44
+ d1.configure(config_element())
45
+ end
46
+ assert d1.plugin_id
47
+ assert d1.log
48
+
49
+ d1.start
50
+ end
51
+
52
+ test 'can execute external command asyncronously' do
53
+ m = Mutex.new
54
+ m.lock
55
+ ary = []
56
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
57
+ ran = false
58
+ @d.child_process_execute(:t0, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
59
+ m.lock
60
+ ran = true
61
+ io.read # discard
62
+ ary << 2
63
+ m.unlock
64
+ end
65
+ ary << 1
66
+ m.unlock
67
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
68
+ m.lock
69
+ m.unlock
70
+ end
71
+ assert_equal [1,2], ary
72
+ end
73
+
74
+ test 'can execute external command at just once, which finishes immediately' do
75
+ m = Mutex.new
76
+ t1 = Time.now
77
+ ary = []
78
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
79
+ ran = false
80
+ @d.child_process_execute(:t1, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
81
+ m.lock
82
+ ran = true
83
+ ary << io.read
84
+ m.unlock
85
+ end
86
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
87
+ m.lock
88
+ m.unlock
89
+ end
90
+ assert{ Time.now - t1 < 4.0 }
91
+ end
92
+
93
+ test 'can execute external command at just once, which can handle both of read and write' do
94
+ m = Mutex.new
95
+ ary = []
96
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
97
+ ran = false
98
+ cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; end'"
99
+ @d.child_process_execute(:t2, cmd, mode: [:write, :read]) do |writeio, readio|
100
+ m.lock
101
+ ran = true
102
+
103
+ [[1,2],[3,4],[5,6]].each do |i,j|
104
+ writeio.write "my data#{i}\n"
105
+ writeio.write "my data#{j}\n"
106
+ writeio.flush
107
+ end
108
+ writeio.close
109
+
110
+ while line = readio.readline
111
+ ary << line
112
+ end
113
+ m.unlock
114
+ end
115
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
116
+ m.lock
117
+ m.unlock
118
+ end
119
+
120
+ assert_equal [], @d.log.out.logs
121
+ expected = (1..6).map{|i| "my data#{i}\n" }
122
+ assert_equal expected, ary
123
+ end
124
+
125
+ test 'can execute external command at just once, which can handle all of read, write and stderr' do
126
+ m = Mutex.new
127
+ ary1 = []
128
+ ary2 = []
129
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
130
+ ran = false
131
+ cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; STDERR.puts line.chomp; STDERR.flush rescue nil; end'"
132
+ @d.child_process_execute(:t2a, cmd, mode: [:write, :read, :stderr]) do |writeio, readio, stderrio|
133
+ m.lock
134
+ ran = true
135
+
136
+ [[1,2],[3,4],[5,6]].each do |i,j|
137
+ writeio.write "my data#{i}\n"
138
+ writeio.write "my data#{j}\n"
139
+ writeio.flush
140
+ end
141
+ writeio.close
142
+
143
+ while (line1 = readio.readline) && (line2 = stderrio.readline)
144
+ ary1 << line1
145
+ ary2 << line2
146
+ end
147
+
148
+ m.unlock
149
+ end
150
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
151
+ m.lock
152
+ m.unlock
153
+ end
154
+
155
+ assert_equal [], @d.log.out.logs
156
+ expected = (1..6).map{|i| "my data#{i}\n" }
157
+ assert_equal expected, ary1
158
+ assert_equal expected, ary2
159
+ end
160
+
161
+ test 'can execute external command at just once, which can handle both of write and read (with stderr)' do
162
+ m = Mutex.new
163
+ ary = []
164
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
165
+ ran = false
166
+ cmd = "ruby"
167
+ args = ['-e', 'while !STDIN.eof? && line = STDIN.readline; puts "[s]" + line.chomp; STDOUT.flush rescue nil; STDERR.puts "[e]" + line.chomp; STDERR.flush rescue nil; end']
168
+ @d.child_process_execute(:t2b, cmd, arguments: args, mode: [:write, :read_with_stderr]) do |writeio, readio|
169
+ m.lock
170
+ ran = true
171
+
172
+ [[1,2],[3,4],[5,6]].each do |i,j|
173
+ writeio.write "my data#{i}\n"
174
+ writeio.write "my data#{j}\n"
175
+ writeio.flush
176
+ end
177
+ writeio.close
178
+
179
+ while line = readio.readline
180
+ ary << line
181
+ end
182
+
183
+ m.unlock
184
+ end
185
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
186
+ m.lock
187
+ m.unlock
188
+ end
189
+
190
+ assert_equal [], @d.log.out.logs
191
+ expected = (1..6).map{|i| ["[s]my data#{i}\n", "[e]my data#{i}\n"] }.flatten
192
+ assert_equal expected, ary
193
+ end
194
+
195
+ test 'can execute external command at just once, which runs forever' do
196
+ m = Mutex.new
197
+ ary = []
198
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
199
+ ran = false
200
+ args = ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush; end"]
201
+ @d.child_process_execute(:t3, "ruby", arguments: args, mode: [:read]) do |io|
202
+ m.lock
203
+ ran = true
204
+ begin
205
+ while @d.child_process_running? && line = io.readline
206
+ ary << line
207
+ end
208
+ rescue
209
+ # ignore
210
+ ensure
211
+ m.unlock
212
+ end
213
+ end
214
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
215
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 10
216
+ @d.stop # nothing occures
217
+ @d.shutdown
218
+
219
+ assert{ ary.size > 5 }
220
+
221
+ @d.close
222
+
223
+ @d.terminate
224
+ assert @d._child_process_processes.empty?
225
+ end
226
+ end
227
+
228
+ unless Fluent.windows?
229
+ # In windows environment, child_process try KILL at first (because there's no SIGTERM)
230
+ test 'can execute external command just once, and can terminate it forcedly when shutdown/terminate even if it ignore SIGTERM' do
231
+ m = Mutex.new
232
+ ary = []
233
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
234
+ ran = false
235
+ @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
236
+ m.lock
237
+ ran = true
238
+ begin
239
+ while line = io.readline
240
+ ary << line
241
+ end
242
+ rescue
243
+ # ignore
244
+ ensure
245
+ m.unlock
246
+ end
247
+ end
248
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
249
+
250
+ assert_equal [], @d.log.out.logs
251
+
252
+ @d.stop # nothing occures
253
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
254
+ lines1 = ary.size
255
+ assert{ lines1 > 1 }
256
+
257
+ pid = @d._child_process_processes.keys.first
258
+
259
+ @d.shutdown
260
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
261
+ lines2 = ary.size
262
+ assert{ lines2 > lines1 }
263
+
264
+ @d.close
265
+
266
+ assert_nil((Process.waitpid(pid, Process::WNOHANG) rescue nil))
267
+
268
+ @d.terminate
269
+ assert @d._child_process_processes.empty?
270
+ begin
271
+ Process.waitpid(pid)
272
+ rescue Errno::ECHILD
273
+ end
274
+ # Process successfully KILLed if test reaches here
275
+ assert true
276
+ end
277
+ end
278
+ end
279
+
280
+ test 'can execute external command many times, which finishes immediately' do
281
+ ary = []
282
+ arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 3
283
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
284
+ @d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 5, mode: [:read]) do |io|
285
+ ary << io.read.split("\n").map(&:chomp).join
286
+ end
287
+ sleep 13 # 5sec * 2 + 3sec
288
+ assert_equal [], @d.log.out.logs
289
+ @d.stop
290
+ assert_equal [], @d.log.out.logs
291
+ @d.shutdown; @d.close; @d.terminate
292
+ assert_equal 2, ary.size
293
+ end
294
+ end
295
+
296
+ test 'can execute external command many times, with leading once executed immediately' do
297
+ ary = []
298
+ arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"]
299
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
300
+ @d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 5, immediate: true, mode: [:read]) do |io|
301
+ ary << io.read.split("\n").map(&:chomp).join
302
+ end
303
+ sleep 8 # 5sec * 1 + 3sec
304
+ # but expected lines are same with test above
305
+ @d.stop; @d.shutdown; @d.close; @d.terminate
306
+ assert_equal 2, ary.size
307
+ assert_equal [], @d.log.out.logs
308
+ end
309
+ end
310
+
311
+ test 'does not execute long running external command in parallel in default' do
312
+ ary = []
313
+ arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10
314
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
315
+ assert_equal [], @d.log.out.logs
316
+ @d.log.out.singleton_class.module_eval do
317
+ define_method(:write){|message|
318
+ raise "boo" if message.include?('test: {"test":"test"}') || message.include?('test: {"test"=>"test"}')
319
+ @logs.push message
320
+ }
321
+ end
322
+
323
+ @d.child_process_execute(:t7, "ruby", arguments: arguments, interval: 2, immediate: true, mode: [:read]) do |io|
324
+ ary << io.read.split("\n").map(&:chomp).join
325
+ end
326
+ sleep 4
327
+ assert_equal 1, @d._child_process_processes.size
328
+ @d.stop
329
+ warn_msg = '[warn]: previous child process is still running. skipped. title=:t7 command="ruby" arguments=["-e", "10.times{ puts \'okay\'; STDOUT.flush rescue nil; sleep 0.5 }"] interval=2 parallel=false' + "\n"
330
+ logs = @d.log.out.logs
331
+ assert{ logs.first.end_with?(warn_msg) }
332
+ assert{ logs.all?{|line| line.end_with?(warn_msg) } }
333
+ @d.shutdown; @d.close; @d.terminate
334
+ assert_equal [], @d.log.out.logs
335
+ end
336
+ end
337
+
338
+ test 'can execute long running external command in parallel if specified' do
339
+ ary = []
340
+ arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10 sec
341
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
342
+ @d.child_process_execute(:t8, "ruby", arguments: arguments, interval: 1, immediate: true, parallel: true, mode: [:read]) do |io|
343
+ ary << io.read.split("\n").map(&:chomp).join
344
+ end
345
+ sleep 4
346
+ processes = @d._child_process_processes.size
347
+ assert{ processes >= 3 && processes <= 5 }
348
+ @d.stop; @d.shutdown; @d.close; @d.terminate
349
+ assert_equal [], @d.log.out.logs
350
+ end
351
+ end
352
+
353
+ test 'execute external processes only for writing' do
354
+ m = Mutex.new
355
+ unreadable = false
356
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
357
+ ran = false
358
+ @d.child_process_execute(:t9, "ruby", arguments: ['-e', 'a=""; while b=STDIN.readline; a+=b; end'], mode: [:write]) do |io|
359
+ m.lock
360
+ ran = true
361
+ begin
362
+ io.read
363
+ rescue IOError
364
+ unreadable = true
365
+ end
366
+ 50.times do
367
+ io.write "hahaha\n"
368
+ end
369
+ m.unlock
370
+ end
371
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
372
+ m.lock
373
+ m.unlock
374
+ assert unreadable
375
+ @d.stop; @d.shutdown; @d.close; @d.terminate
376
+ assert_equal [], @d.log.out.logs
377
+ end
378
+ end
379
+
380
+ test 'execute external processes only for reading' do
381
+ m = Mutex.new
382
+ unwritable = false
383
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
384
+ ran = false
385
+ @d.child_process_execute(:t10, "ruby", arguments: ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end"], mode: [:read]) do |io|
386
+ m.lock
387
+ ran = true
388
+ begin
389
+ io.write "foobar"
390
+ rescue IOError
391
+ unwritable = true
392
+ end
393
+ _data = io.readline
394
+ m.unlock
395
+ end
396
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
397
+ m.lock
398
+ m.unlock
399
+ @d.stop; @d.shutdown; @d.close; @d.terminate
400
+ assert unwritable
401
+ assert_equal [], @d.log.out.logs
402
+ end
403
+ end
404
+
405
+ test 'can control external encodings' do
406
+ m = Mutex.new
407
+ encodings = []
408
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
409
+ ran = false
410
+ @d.child_process_execute(:t11, "ruby -e 'sleep 10'", external_encoding: 'ascii-8bit') do |r, w|
411
+ m.lock
412
+ ran = true
413
+ encodings << r.external_encoding
414
+ encodings << w.external_encoding
415
+ m.unlock
416
+ end
417
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
418
+ m.lock
419
+ assert_equal Encoding::ASCII_8BIT, encodings[0]
420
+ assert_equal Encoding::ASCII_8BIT, encodings[1]
421
+ @d.stop; @d.shutdown; @d.close; @d.terminate
422
+ end
423
+ end
424
+
425
+ test 'can control internal encodings' do
426
+ m = Mutex.new
427
+ encodings = []
428
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
429
+ ran = false
430
+ @d.child_process_execute(:t12, "ruby -e 'sleep 10'", external_encoding: 'utf-8', internal_encoding: 'ascii-8bit') do |r, w|
431
+ m.lock
432
+ ran = true
433
+ encodings << r.internal_encoding
434
+ encodings << w.internal_encoding
435
+ m.unlock
436
+ end
437
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
438
+ m.lock
439
+ assert_equal Encoding::ASCII_8BIT, encodings[0]
440
+ assert_equal Encoding::ASCII_8BIT, encodings[1]
441
+ @d.stop; @d.shutdown; @d.close; @d.terminate
442
+ end
443
+ end
444
+
445
+ test 'can convert encodings from ascii-8bit to utf-8' do
446
+ m = Mutex.new
447
+ str = nil
448
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
449
+ ran = false
450
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xA4\xB5\xA4\xC8\xA4\xB7"']
451
+ @d.child_process_execute(:t13, "ruby", arguments: args, external_encoding: 'euc-jp', internal_encoding: 'windows-31j', mode: [:read]) do |io|
452
+ m.lock
453
+ ran = true
454
+ str = io.read
455
+ m.unlock
456
+ end
457
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
458
+ m.lock
459
+ assert_equal Encoding.find('windows-31j'), str.encoding
460
+ expected = "さとし".encode('windows-31j')
461
+ assert_equal expected, str
462
+ @d.stop; @d.shutdown; @d.close; @d.terminate
463
+ end
464
+ end
465
+
466
+ test 'can scrub characters without exceptions' do
467
+ m = Mutex.new
468
+ str = nil
469
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
470
+ ran = false
471
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
472
+ @d.child_process_execute(:t13a, "ruby", arguments: args, mode: [:read]) do |io|
473
+ m.lock
474
+ ran = true
475
+ str = io.read
476
+ m.unlock
477
+ end
478
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
479
+ m.lock
480
+ assert_equal Encoding.find('utf-8'), str.encoding
481
+ expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
482
+ assert_equal expected, str
483
+ @d.stop; @d.shutdown; @d.close; @d.terminate
484
+ end
485
+ end
486
+
487
+ test 'can scrub characters without exceptions and replace specified chars' do
488
+ m = Mutex.new
489
+ str = nil
490
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
491
+ ran = false
492
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
493
+ @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
494
+ m.lock
495
+ ran = true
496
+ str = io.read
497
+ m.unlock
498
+ end
499
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
500
+ m.lock
501
+ assert_equal Encoding.find('utf-8'), str.encoding
502
+ expected = "??\x00??".force_encoding("utf-8")
503
+ assert_equal expected, str
504
+ @d.stop; @d.shutdown; @d.close; @d.terminate
505
+ end
506
+ end
507
+
508
+ unless Fluent.windows?
509
+ test 'can specify subprocess name' do
510
+ io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
511
+ process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
512
+ Process.kill(:TERM, io.pid) rescue nil
513
+ io.close rescue nil
514
+
515
+ # Does TravisCI prohibit process renaming?
516
+ # This test will be passed in such environment
517
+ pend unless process_naming_enabled
518
+
519
+ m = Mutex.new
520
+ pids = []
521
+ proc_lines = []
522
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
523
+ ran = false
524
+ @d.child_process_execute(:t14, "ruby", arguments:['-e', 'sleep 10; puts "hello"'], subprocess_name: "sleeeeeeeeeper", mode: [:read]) do |readio|
525
+ m.lock
526
+ ran = true
527
+ pids << @d.child_process_id
528
+ proc_lines += open("|ps"){|_io| _io.readlines }
529
+ m.unlock
530
+ readio.read
531
+ end
532
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
533
+ m.lock
534
+ pid = pids.first
535
+ # 51358 ttys001 0:00.00 sleeper -e sleep 10
536
+ assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[3] == "sleeeeeeeeeper" }
537
+ @d.stop; @d.shutdown; @d.close; @d.terminate
538
+ end
539
+ end
540
+ end
541
+
542
+ test 'can set ENV variables' do
543
+ m = Mutex.new
544
+ str = nil
545
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
546
+ ran = false
547
+ args = ['-e', 'puts ENV["testing_child_process"]']
548
+ @d.child_process_execute(:t15a, "ruby", arguments: args, mode: [:read], env: {'testing_child_process' => 'Yes! True!'}) do |io|
549
+ m.lock
550
+ ran = true
551
+ str = io.read
552
+ m.unlock
553
+ end
554
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
555
+ m.lock
556
+ expected = "Yes! True!\n"
557
+ assert_equal expected, str
558
+ @d.stop; @d.shutdown; @d.close; @d.terminate
559
+ end
560
+ end
561
+
562
+ test 'can unset ENV variables of Fluentd process' do
563
+ m = Mutex.new
564
+ str = nil
565
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
566
+ current_env_path = ENV['PATH']
567
+ ran = false
568
+ args = ['-e', 'puts ENV["testing_child_process1"].to_s + ENV["testing_child_process2"].to_s']
569
+ ENV['testing_child_process1'] = "No! False!"
570
+ @d.child_process_execute(:t15b, "ruby", arguments: args, mode: [:read], unsetenv: true, env: {'testing_child_process2' => 'Yes! True!', 'PATH' => current_env_path}) do |io|
571
+ m.lock
572
+ ran = true
573
+ str = io.read
574
+ m.unlock
575
+ end
576
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
577
+ m.lock
578
+ expected = "Yes! True!\n"
579
+ assert_equal expected, str
580
+ @d.stop; @d.shutdown; @d.close; @d.terminate
581
+ end
582
+ end
583
+
584
+ unless Fluent.windows?
585
+ test 'can change working directory' do
586
+ # check my real /tmp directory (for mac)
587
+ cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
588
+ mytmpdir = open(cmd){|io| io.read.chomp }
589
+
590
+ m = Mutex.new
591
+ str = nil
592
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
593
+ ran = false
594
+ args = ['-e', 'puts Dir.pwd']
595
+ @d.child_process_execute(:t16, "ruby", arguments: args, mode: [:read], chdir: "/tmp") do |io|
596
+ m.lock
597
+ ran = true
598
+ str = io.read.chomp
599
+ m.unlock
600
+ end
601
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
602
+ m.lock
603
+ assert_equal mytmpdir, str
604
+ @d.stop; @d.shutdown; @d.close; @d.terminate
605
+ end
606
+ end
607
+ end
608
+ end