fluentd 0.12.40 → 1.6.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (428) hide show
  1. checksums.yaml +5 -5
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  4. data/.github/ISSUE_TEMPLATE.md +17 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
  6. data/.gitignore +5 -0
  7. data/.gitlab/cicd-template.yaml +10 -0
  8. data/.gitlab-ci.yml +147 -0
  9. data/.travis.yml +56 -20
  10. data/ADOPTERS.md +5 -0
  11. data/CHANGELOG.md +1369 -0
  12. data/CONTRIBUTING.md +16 -5
  13. data/GOVERNANCE.md +55 -0
  14. data/Gemfile +5 -0
  15. data/GithubWorkflow.md +78 -0
  16. data/LICENSE +202 -0
  17. data/MAINTAINERS.md +7 -0
  18. data/README.md +23 -11
  19. data/Rakefile +48 -2
  20. data/Vagrantfile +17 -0
  21. data/appveyor.yml +37 -0
  22. data/bin/fluent-binlog-reader +7 -0
  23. data/bin/fluent-ca-generate +6 -0
  24. data/bin/fluent-plugin-config-format +5 -0
  25. data/bin/fluent-plugin-generate +5 -0
  26. data/bin/fluentd +3 -0
  27. data/code-of-conduct.md +3 -0
  28. data/example/copy_roundrobin.conf +39 -0
  29. data/example/counter.conf +18 -0
  30. data/example/in_dummy_blocks.conf +17 -0
  31. data/example/in_dummy_with_compression.conf +23 -0
  32. data/example/in_forward.conf +7 -0
  33. data/example/in_forward_client.conf +37 -0
  34. data/example/in_forward_shared_key.conf +15 -0
  35. data/example/in_forward_tls.conf +14 -0
  36. data/example/in_forward_users.conf +24 -0
  37. data/example/in_forward_workers.conf +21 -0
  38. data/example/in_http.conf +3 -1
  39. data/example/in_out_forward.conf +17 -0
  40. data/example/logevents.conf +25 -0
  41. data/example/multi_filters.conf +61 -0
  42. data/example/out_exec_filter.conf +42 -0
  43. data/example/out_forward.conf +13 -13
  44. data/example/out_forward_buf_file.conf +23 -0
  45. data/example/out_forward_client.conf +109 -0
  46. data/example/out_forward_heartbeat_none.conf +16 -0
  47. data/example/out_forward_shared_key.conf +36 -0
  48. data/example/out_forward_tls.conf +18 -0
  49. data/example/out_forward_users.conf +65 -0
  50. data/example/out_null.conf +36 -0
  51. data/example/secondary_file.conf +42 -0
  52. data/example/suppress_config_dump.conf +7 -0
  53. data/example/worker_section.conf +36 -0
  54. data/fluent.conf +29 -0
  55. data/fluentd.gemspec +21 -11
  56. data/lib/fluent/agent.rb +67 -90
  57. data/lib/fluent/clock.rb +62 -0
  58. data/lib/fluent/command/binlog_reader.rb +244 -0
  59. data/lib/fluent/command/ca_generate.rb +181 -0
  60. data/lib/fluent/command/cat.rb +42 -18
  61. data/lib/fluent/command/debug.rb +12 -10
  62. data/lib/fluent/command/fluentd.rb +153 -5
  63. data/lib/fluent/command/plugin_config_formatter.rb +292 -0
  64. data/lib/fluent/command/plugin_generator.rb +324 -0
  65. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  66. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  67. data/lib/fluent/compat/exec_util.rb +129 -0
  68. data/lib/fluent/compat/file_util.rb +54 -0
  69. data/lib/fluent/compat/filter.rb +68 -0
  70. data/lib/fluent/compat/formatter.rb +111 -0
  71. data/lib/fluent/compat/formatter_utils.rb +85 -0
  72. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  73. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  74. data/lib/fluent/compat/input.rb +49 -0
  75. data/lib/fluent/compat/output.rb +718 -0
  76. data/lib/fluent/compat/output_chain.rb +60 -0
  77. data/lib/fluent/compat/parser.rb +310 -0
  78. data/lib/fluent/compat/parser_utils.rb +40 -0
  79. data/lib/fluent/compat/propagate_default.rb +62 -0
  80. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  81. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  82. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  83. data/lib/fluent/compat/socket_util.rb +165 -0
  84. data/lib/fluent/compat/string_util.rb +34 -0
  85. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  86. data/lib/fluent/compat/type_converter.rb +90 -0
  87. data/lib/fluent/config/configure_proxy.rb +210 -62
  88. data/lib/fluent/config/dsl.rb +12 -5
  89. data/lib/fluent/config/element.rb +107 -9
  90. data/lib/fluent/config/literal_parser.rb +9 -3
  91. data/lib/fluent/config/parser.rb +4 -4
  92. data/lib/fluent/config/section.rb +51 -14
  93. data/lib/fluent/config/types.rb +28 -13
  94. data/lib/fluent/config/v1_parser.rb +3 -5
  95. data/lib/fluent/config.rb +23 -20
  96. data/lib/fluent/configurable.rb +79 -21
  97. data/lib/fluent/counter/base_socket.rb +46 -0
  98. data/lib/fluent/counter/client.rb +297 -0
  99. data/lib/fluent/counter/error.rb +86 -0
  100. data/lib/fluent/counter/mutex_hash.rb +163 -0
  101. data/lib/fluent/counter/server.rb +273 -0
  102. data/lib/fluent/counter/store.rb +205 -0
  103. data/lib/fluent/counter/validator.rb +145 -0
  104. data/lib/fluent/counter.rb +23 -0
  105. data/lib/fluent/daemon.rb +15 -0
  106. data/lib/fluent/engine.rb +102 -65
  107. data/lib/fluent/env.rb +7 -3
  108. data/lib/fluent/error.rb +30 -0
  109. data/lib/fluent/event.rb +197 -21
  110. data/lib/fluent/event_router.rb +93 -10
  111. data/lib/fluent/filter.rb +2 -50
  112. data/lib/fluent/formatter.rb +4 -293
  113. data/lib/fluent/input.rb +2 -32
  114. data/lib/fluent/label.rb +10 -2
  115. data/lib/fluent/load.rb +3 -3
  116. data/lib/fluent/log.rb +348 -81
  117. data/lib/fluent/match.rb +37 -36
  118. data/lib/fluent/mixin.rb +12 -176
  119. data/lib/fluent/msgpack_factory.rb +62 -0
  120. data/lib/fluent/output.rb +10 -612
  121. data/lib/fluent/output_chain.rb +23 -0
  122. data/lib/fluent/parser.rb +4 -800
  123. data/lib/fluent/plugin/bare_output.rb +63 -0
  124. data/lib/fluent/plugin/base.rb +192 -0
  125. data/lib/fluent/plugin/buf_file.rb +128 -174
  126. data/lib/fluent/plugin/buf_memory.rb +9 -92
  127. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  128. data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
  129. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  130. data/lib/fluent/plugin/buffer.rb +779 -0
  131. data/lib/fluent/plugin/compressable.rb +92 -0
  132. data/lib/fluent/plugin/exec_util.rb +3 -108
  133. data/lib/fluent/plugin/file_util.rb +4 -34
  134. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  135. data/lib/fluent/plugin/filter.rb +93 -0
  136. data/lib/fluent/plugin/filter_grep.rb +117 -34
  137. data/lib/fluent/plugin/filter_parser.rb +85 -62
  138. data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
  139. data/lib/fluent/plugin/filter_stdout.rb +15 -12
  140. data/lib/fluent/plugin/formatter.rb +50 -0
  141. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  142. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  143. data/lib/fluent/plugin/formatter_json.rb +55 -0
  144. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  145. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  146. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  147. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  148. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  149. data/lib/fluent/plugin/formatter_tsv.rb +38 -0
  150. data/lib/fluent/plugin/in_debug_agent.rb +17 -6
  151. data/lib/fluent/plugin/in_dummy.rb +47 -20
  152. data/lib/fluent/plugin/in_exec.rb +55 -123
  153. data/lib/fluent/plugin/in_forward.rb +299 -216
  154. data/lib/fluent/plugin/in_gc_stat.rb +14 -36
  155. data/lib/fluent/plugin/in_http.rb +204 -91
  156. data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
  157. data/lib/fluent/plugin/in_object_space.rb +13 -41
  158. data/lib/fluent/plugin/in_syslog.rb +112 -134
  159. data/lib/fluent/plugin/in_tail.rb +408 -745
  160. data/lib/fluent/plugin/in_tcp.rb +66 -9
  161. data/lib/fluent/plugin/in_udp.rb +60 -11
  162. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
  163. data/lib/fluent/plugin/input.rb +37 -0
  164. data/lib/fluent/plugin/multi_output.rb +158 -0
  165. data/lib/fluent/plugin/out_copy.rb +23 -35
  166. data/lib/fluent/plugin/out_exec.rb +67 -70
  167. data/lib/fluent/plugin/out_exec_filter.rb +204 -271
  168. data/lib/fluent/plugin/out_file.rb +267 -73
  169. data/lib/fluent/plugin/out_forward.rb +854 -325
  170. data/lib/fluent/plugin/out_null.rb +42 -9
  171. data/lib/fluent/plugin/out_relabel.rb +9 -5
  172. data/lib/fluent/plugin/out_roundrobin.rb +18 -37
  173. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  174. data/lib/fluent/plugin/out_stdout.rb +43 -10
  175. data/lib/fluent/plugin/out_stream.rb +7 -2
  176. data/lib/fluent/plugin/output.rb +1498 -0
  177. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  178. data/lib/fluent/plugin/parser.rb +191 -0
  179. data/lib/fluent/plugin/parser_apache.rb +28 -0
  180. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  181. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  182. data/lib/fluent/plugin/parser_csv.rb +39 -0
  183. data/lib/fluent/plugin/parser_json.rb +94 -0
  184. data/lib/fluent/plugin/parser_ltsv.rb +49 -0
  185. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  186. data/lib/fluent/plugin/parser_multiline.rb +106 -0
  187. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  188. data/lib/fluent/plugin/parser_none.rb +36 -0
  189. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  190. data/lib/fluent/plugin/parser_syslog.rb +142 -0
  191. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  192. data/lib/fluent/plugin/socket_util.rb +3 -143
  193. data/lib/fluent/plugin/storage.rb +84 -0
  194. data/lib/fluent/plugin/storage_local.rb +164 -0
  195. data/lib/fluent/plugin/string_util.rb +3 -15
  196. data/lib/fluent/plugin.rb +122 -121
  197. data/lib/fluent/plugin_helper/cert_option.rb +178 -0
  198. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  199. data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
  200. data/lib/fluent/plugin_helper/counter.rb +51 -0
  201. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  202. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  203. data/lib/fluent/plugin_helper/extract.rb +104 -0
  204. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  205. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  206. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  207. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  208. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  209. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  210. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  211. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  212. data/lib/fluent/plugin_helper/http_server.rb +76 -0
  213. data/lib/fluent/plugin_helper/inject.rb +151 -0
  214. data/lib/fluent/plugin_helper/parser.rb +147 -0
  215. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  216. data/lib/fluent/plugin_helper/retry_state.rb +205 -0
  217. data/lib/fluent/plugin_helper/server.rb +807 -0
  218. data/lib/fluent/plugin_helper/socket.rb +250 -0
  219. data/lib/fluent/plugin_helper/socket_option.rb +80 -0
  220. data/lib/fluent/plugin_helper/storage.rb +349 -0
  221. data/lib/fluent/plugin_helper/thread.rb +179 -0
  222. data/lib/fluent/plugin_helper/timer.rb +92 -0
  223. data/lib/fluent/plugin_helper.rb +73 -0
  224. data/lib/fluent/plugin_id.rb +80 -0
  225. data/lib/fluent/process.rb +3 -489
  226. data/lib/fluent/registry.rb +52 -10
  227. data/lib/fluent/root_agent.rb +204 -42
  228. data/lib/fluent/supervisor.rb +597 -359
  229. data/lib/fluent/system_config.rb +131 -42
  230. data/lib/fluent/test/base.rb +6 -54
  231. data/lib/fluent/test/driver/base.rb +224 -0
  232. data/lib/fluent/test/driver/base_owned.rb +70 -0
  233. data/lib/fluent/test/driver/base_owner.rb +135 -0
  234. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  235. data/lib/fluent/test/driver/filter.rb +57 -0
  236. data/lib/fluent/test/driver/formatter.rb +30 -0
  237. data/lib/fluent/test/driver/input.rb +31 -0
  238. data/lib/fluent/test/driver/multi_output.rb +53 -0
  239. data/lib/fluent/test/driver/output.rb +102 -0
  240. data/lib/fluent/test/driver/parser.rb +30 -0
  241. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  242. data/lib/fluent/test/filter_test.rb +0 -1
  243. data/lib/fluent/test/formatter_test.rb +4 -1
  244. data/lib/fluent/test/helpers.rb +58 -10
  245. data/lib/fluent/test/input_test.rb +27 -19
  246. data/lib/fluent/test/log.rb +79 -0
  247. data/lib/fluent/test/output_test.rb +28 -39
  248. data/lib/fluent/test/parser_test.rb +3 -1
  249. data/lib/fluent/test/startup_shutdown.rb +46 -0
  250. data/lib/fluent/test.rb +33 -1
  251. data/lib/fluent/time.rb +450 -1
  252. data/lib/fluent/timezone.rb +27 -3
  253. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  254. data/lib/fluent/version.rb +1 -1
  255. data/lib/fluent/winsvc.rb +85 -0
  256. data/templates/new_gem/Gemfile +3 -0
  257. data/templates/new_gem/README.md.erb +43 -0
  258. data/templates/new_gem/Rakefile +13 -0
  259. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  260. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  261. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  262. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  263. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  264. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  265. data/templates/new_gem/test/helper.rb.erb +8 -0
  266. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  267. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  268. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  269. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  270. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  271. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  272. data/templates/plugin_config_formatter/param.md.erb +34 -0
  273. data/templates/plugin_config_formatter/section.md.erb +12 -0
  274. data/test/command/test_binlog_reader.rb +346 -0
  275. data/test/command/test_ca_generate.rb +70 -0
  276. data/test/command/test_fluentd.rb +901 -0
  277. data/test/command/test_plugin_config_formatter.rb +276 -0
  278. data/test/command/test_plugin_generator.rb +92 -0
  279. data/test/compat/test_calls_super.rb +166 -0
  280. data/test/compat/test_parser.rb +92 -0
  281. data/test/config/test_config_parser.rb +126 -2
  282. data/test/config/test_configurable.rb +946 -187
  283. data/test/config/test_configure_proxy.rb +424 -74
  284. data/test/config/test_dsl.rb +11 -11
  285. data/test/config/test_element.rb +500 -0
  286. data/test/config/test_literal_parser.rb +8 -0
  287. data/test/config/test_plugin_configuration.rb +56 -0
  288. data/test/config/test_section.rb +79 -7
  289. data/test/config/test_system_config.rb +122 -35
  290. data/test/config/test_types.rb +38 -0
  291. data/test/counter/test_client.rb +559 -0
  292. data/test/counter/test_error.rb +44 -0
  293. data/test/counter/test_mutex_hash.rb +179 -0
  294. data/test/counter/test_server.rb +589 -0
  295. data/test/counter/test_store.rb +258 -0
  296. data/test/counter/test_validator.rb +137 -0
  297. data/test/helper.rb +89 -6
  298. data/test/helpers/fuzzy_assert.rb +89 -0
  299. data/test/plugin/test_bare_output.rb +118 -0
  300. data/test/plugin/test_base.rb +115 -0
  301. data/test/plugin/test_buf_file.rb +823 -460
  302. data/test/plugin/test_buf_memory.rb +32 -194
  303. data/test/plugin/test_buffer.rb +1233 -0
  304. data/test/plugin/test_buffer_chunk.rb +198 -0
  305. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  306. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  307. data/test/plugin/test_compressable.rb +84 -0
  308. data/test/plugin/test_filter.rb +357 -0
  309. data/test/plugin/test_filter_grep.rb +540 -29
  310. data/test/plugin/test_filter_parser.rb +439 -452
  311. data/test/plugin/test_filter_record_transformer.rb +123 -166
  312. data/test/plugin/test_filter_stdout.rb +160 -72
  313. data/test/plugin/test_formatter_csv.rb +111 -0
  314. data/test/plugin/test_formatter_hash.rb +35 -0
  315. data/test/plugin/test_formatter_json.rb +51 -0
  316. data/test/plugin/test_formatter_ltsv.rb +62 -0
  317. data/test/plugin/test_formatter_msgpack.rb +28 -0
  318. data/test/plugin/test_formatter_out_file.rb +95 -0
  319. data/test/plugin/test_formatter_single_value.rb +38 -0
  320. data/test/plugin/test_formatter_tsv.rb +68 -0
  321. data/test/plugin/test_in_debug_agent.rb +24 -1
  322. data/test/plugin/test_in_dummy.rb +111 -18
  323. data/test/plugin/test_in_exec.rb +200 -113
  324. data/test/plugin/test_in_forward.rb +990 -387
  325. data/test/plugin/test_in_gc_stat.rb +10 -8
  326. data/test/plugin/test_in_http.rb +600 -224
  327. data/test/plugin/test_in_monitor_agent.rb +690 -0
  328. data/test/plugin/test_in_object_space.rb +24 -8
  329. data/test/plugin/test_in_syslog.rb +154 -215
  330. data/test/plugin/test_in_tail.rb +1006 -707
  331. data/test/plugin/test_in_tcp.rb +125 -48
  332. data/test/plugin/test_in_udp.rb +204 -63
  333. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
  334. data/test/plugin/test_input.rb +126 -0
  335. data/test/plugin/test_metadata.rb +89 -0
  336. data/test/plugin/test_multi_output.rb +180 -0
  337. data/test/plugin/test_out_copy.rb +117 -112
  338. data/test/plugin/test_out_exec.rb +258 -53
  339. data/test/plugin/test_out_exec_filter.rb +538 -115
  340. data/test/plugin/test_out_file.rb +865 -178
  341. data/test/plugin/test_out_forward.rb +998 -210
  342. data/test/plugin/test_out_null.rb +105 -0
  343. data/test/plugin/test_out_relabel.rb +28 -0
  344. data/test/plugin/test_out_roundrobin.rb +36 -29
  345. data/test/plugin/test_out_secondary_file.rb +458 -0
  346. data/test/plugin/test_out_stdout.rb +135 -37
  347. data/test/plugin/test_out_stream.rb +18 -0
  348. data/test/plugin/test_output.rb +984 -0
  349. data/test/plugin/test_output_as_buffered.rb +2021 -0
  350. data/test/plugin/test_output_as_buffered_backup.rb +312 -0
  351. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  352. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  353. data/test/plugin/test_output_as_buffered_retries.rb +911 -0
  354. data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
  355. data/test/plugin/test_output_as_standard.rb +374 -0
  356. data/test/plugin/test_owned_by.rb +35 -0
  357. data/test/plugin/test_parser.rb +359 -0
  358. data/test/plugin/test_parser_apache.rb +42 -0
  359. data/test/plugin/test_parser_apache2.rb +47 -0
  360. data/test/plugin/test_parser_apache_error.rb +45 -0
  361. data/test/plugin/test_parser_csv.rb +103 -0
  362. data/test/plugin/test_parser_json.rb +138 -0
  363. data/test/plugin/test_parser_labeled_tsv.rb +145 -0
  364. data/test/plugin/test_parser_multiline.rb +100 -0
  365. data/test/plugin/test_parser_nginx.rb +88 -0
  366. data/test/plugin/test_parser_none.rb +52 -0
  367. data/test/plugin/test_parser_regexp.rb +289 -0
  368. data/test/plugin/test_parser_syslog.rb +441 -0
  369. data/test/plugin/test_parser_tsv.rb +122 -0
  370. data/test/plugin/test_storage.rb +167 -0
  371. data/test/plugin/test_storage_local.rb +335 -0
  372. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  373. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  374. data/test/plugin_helper/data/cert/cert.pem +19 -0
  375. data/test/plugin_helper/http_server/test_app.rb +65 -0
  376. data/test/plugin_helper/http_server/test_route.rb +32 -0
  377. data/test/plugin_helper/test_cert_option.rb +16 -0
  378. data/test/plugin_helper/test_child_process.rb +794 -0
  379. data/test/plugin_helper/test_compat_parameters.rb +353 -0
  380. data/test/plugin_helper/test_event_emitter.rb +51 -0
  381. data/test/plugin_helper/test_event_loop.rb +52 -0
  382. data/test/plugin_helper/test_extract.rb +194 -0
  383. data/test/plugin_helper/test_formatter.rb +255 -0
  384. data/test/plugin_helper/test_http_server_helper.rb +205 -0
  385. data/test/plugin_helper/test_inject.rb +519 -0
  386. data/test/plugin_helper/test_parser.rb +264 -0
  387. data/test/plugin_helper/test_record_accessor.rb +197 -0
  388. data/test/plugin_helper/test_retry_state.rb +442 -0
  389. data/test/plugin_helper/test_server.rb +1714 -0
  390. data/test/plugin_helper/test_storage.rb +542 -0
  391. data/test/plugin_helper/test_thread.rb +164 -0
  392. data/test/plugin_helper/test_timer.rb +132 -0
  393. data/test/scripts/exec_script.rb +0 -6
  394. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  395. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  396. data/test/scripts/fluent/plugin/out_test.rb +23 -15
  397. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  398. data/test/test_clock.rb +164 -0
  399. data/test/test_config.rb +16 -7
  400. data/test/test_configdsl.rb +2 -2
  401. data/test/test_event.rb +360 -13
  402. data/test/test_event_router.rb +108 -11
  403. data/test/test_event_time.rb +199 -0
  404. data/test/test_filter.rb +48 -6
  405. data/test/test_formatter.rb +11 -391
  406. data/test/test_input.rb +1 -1
  407. data/test/test_log.rb +591 -31
  408. data/test/test_mixin.rb +1 -1
  409. data/test/test_output.rb +121 -185
  410. data/test/test_plugin.rb +251 -0
  411. data/test/test_plugin_classes.rb +177 -10
  412. data/test/test_plugin_helper.rb +81 -0
  413. data/test/test_plugin_id.rb +101 -0
  414. data/test/test_process.rb +8 -42
  415. data/test/test_root_agent.rb +766 -21
  416. data/test/test_supervisor.rb +481 -0
  417. data/test/test_test_drivers.rb +135 -0
  418. data/test/test_time_formatter.rb +282 -0
  419. data/test/test_time_parser.rb +231 -0
  420. data/test/test_unique_id.rb +47 -0
  421. metadata +454 -60
  422. data/COPYING +0 -14
  423. data/ChangeLog +0 -666
  424. data/lib/fluent/buffer.rb +0 -365
  425. data/lib/fluent/plugin/in_status.rb +0 -76
  426. data/test/plugin/test_in_status.rb +0 -38
  427. data/test/test_buffer.rb +0 -624
  428. data/test/test_parser.rb +0 -1305
@@ -0,0 +1,794 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/child_process'
3
+ require 'fluent/plugin/base'
4
+ require 'timeout'
5
+ require 'tempfile'
6
+
7
+ class ChildProcessTest < Test::Unit::TestCase
8
+ TEST_DEADLOCK_TIMEOUT = 30
9
+ TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING = 0.1 # This may be shorter than ruby's threading timer, but work well
10
+ # @nalsh says that ruby's cpu assignments for threads are almost 200ms or so.
11
+ # Loop interval (expected that it work as specified) should be longer than it.
12
+ TEST_WAIT_INTERVAL_FOR_LOOP = 0.5
13
+
14
+ setup do
15
+ @d = Dummy.new
16
+ @d.configure(config_element())
17
+ @d.start
18
+ end
19
+
20
+ teardown do
21
+ if @d
22
+ @d.stop unless @d.stopped?
23
+ @d.shutdown unless @d.shutdown?
24
+ @d.close unless @d.closed?
25
+ @d.terminate unless @d.terminated?
26
+ @d.log.reset
27
+ end
28
+ end
29
+
30
+ class Dummy < Fluent::Plugin::TestBase
31
+ helpers :child_process
32
+ def configure(conf)
33
+ super
34
+ @_child_process_kill_timeout = 1
35
+ end
36
+ end
37
+
38
+ test 'can be instantiated' do
39
+ d1 = Dummy.new
40
+ assert d1.respond_to?(:_child_process_processes)
41
+ end
42
+
43
+ test 'can be configured and started' do
44
+ d1 = Dummy.new
45
+ assert_nothing_raised do
46
+ d1.configure(config_element())
47
+ end
48
+ assert d1.plugin_id
49
+ assert d1.log
50
+
51
+ d1.start
52
+ end
53
+
54
+ test 'can execute external command asyncronously' do
55
+ m = Mutex.new
56
+ m.lock
57
+ ary = []
58
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
59
+ ran = false
60
+ @d.child_process_execute(:t0, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
61
+ m.lock
62
+ ran = true
63
+ io.read # discard
64
+ ary << 2
65
+ m.unlock
66
+ end
67
+ ary << 1
68
+ m.unlock
69
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
70
+ m.lock
71
+ m.unlock
72
+ end
73
+ assert_equal [1,2], ary
74
+ end
75
+
76
+ test 'can execute external command at just once, which finishes immediately' do
77
+ m = Mutex.new
78
+ t1 = Time.now
79
+ ary = []
80
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
81
+ ran = false
82
+ @d.child_process_execute(:t1, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
83
+ m.lock
84
+ ran = true
85
+ ary << io.read
86
+ m.unlock
87
+ end
88
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
89
+ m.lock
90
+ m.unlock
91
+ end
92
+ assert{ Time.now - t1 < 4.0 }
93
+ end
94
+
95
+ test 'can execute external command at just once, which can handle both of read and write' do
96
+ m = Mutex.new
97
+ ary = []
98
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
99
+ ran = false
100
+ cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; end'"
101
+ @d.child_process_execute(:t2, cmd, mode: [:write, :read]) do |writeio, readio|
102
+ m.lock
103
+ ran = true
104
+
105
+ [[1,2],[3,4],[5,6]].each do |i,j|
106
+ writeio.write "my data#{i}\n"
107
+ writeio.write "my data#{j}\n"
108
+ writeio.flush
109
+ end
110
+ writeio.close
111
+
112
+ while line = readio.readline
113
+ ary << line
114
+ end
115
+ m.unlock
116
+ end
117
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
118
+ m.lock
119
+ m.unlock
120
+ end
121
+
122
+ assert_equal [], @d.log.out.logs
123
+ expected = (1..6).map{|i| "my data#{i}\n" }
124
+ assert_equal expected, ary
125
+ end
126
+
127
+ test 'can execute external command at just once, which can handle all of read, write and stderr' do
128
+ m = Mutex.new
129
+ ary1 = []
130
+ ary2 = []
131
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
132
+ ran = false
133
+ cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; STDERR.puts line.chomp; STDERR.flush rescue nil; end'"
134
+ @d.child_process_execute(:t2a, cmd, mode: [:write, :read, :stderr]) do |writeio, readio, stderrio|
135
+ m.lock
136
+ ran = true
137
+
138
+ [[1,2],[3,4],[5,6]].each do |i,j|
139
+ writeio.write "my data#{i}\n"
140
+ writeio.write "my data#{j}\n"
141
+ writeio.flush
142
+ end
143
+ writeio.close
144
+
145
+ while (line1 = readio.readline) && (line2 = stderrio.readline)
146
+ ary1 << line1
147
+ ary2 << line2
148
+ end
149
+
150
+ m.unlock
151
+ end
152
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
153
+ m.lock
154
+ m.unlock
155
+ end
156
+
157
+ assert_equal [], @d.log.out.logs
158
+ expected = (1..6).map{|i| "my data#{i}\n" }
159
+ assert_equal expected, ary1
160
+ assert_equal expected, ary2
161
+ end
162
+
163
+ test 'can execute external command at just once, which can handle both of write and read (with stderr)' do
164
+ m = Mutex.new
165
+ ary = []
166
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
167
+ ran = false
168
+ cmd = "ruby"
169
+ args = ['-e', 'while !STDIN.eof? && line = STDIN.readline; puts "[s]" + line.chomp; STDOUT.flush rescue nil; STDERR.puts "[e]" + line.chomp; STDERR.flush rescue nil; end']
170
+ @d.child_process_execute(:t2b, cmd, arguments: args, mode: [:write, :read_with_stderr]) do |writeio, readio|
171
+ m.lock
172
+ ran = true
173
+
174
+ [[1,2],[3,4],[5,6]].each do |i,j|
175
+ writeio.write "my data#{i}\n"
176
+ writeio.write "my data#{j}\n"
177
+ writeio.flush
178
+ end
179
+ writeio.close
180
+
181
+ while line = readio.readline
182
+ ary << line
183
+ end
184
+
185
+ m.unlock
186
+ end
187
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
188
+ m.lock
189
+ m.unlock
190
+ end
191
+
192
+ assert_equal [], @d.log.out.logs
193
+ expected = (1..6).map{|i| ["[s]my data#{i}\n", "[e]my data#{i}\n"] }.flatten
194
+ assert_equal expected, ary
195
+ end
196
+
197
+ test 'can execute external command at just once, which runs forever' do
198
+ m = Mutex.new
199
+ ary = []
200
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
201
+ ran = false
202
+ args = ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush; end"]
203
+ @d.child_process_execute(:t3, "ruby", arguments: args, mode: [:read]) do |io|
204
+ m.lock
205
+ ran = true
206
+ begin
207
+ while @d.child_process_running? && line = io.readline
208
+ ary << line
209
+ end
210
+ rescue
211
+ # ignore
212
+ ensure
213
+ m.unlock
214
+ end
215
+ end
216
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
217
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 10
218
+ @d.stop # nothing occurs
219
+ @d.shutdown
220
+
221
+ assert{ ary.size > 5 }
222
+
223
+ @d.close
224
+
225
+ @d.terminate
226
+ assert @d._child_process_processes.empty?
227
+ end
228
+ end
229
+
230
+ # In windows environment, child_process try KILL at first (because there's no SIGTERM)
231
+ test 'can execute external command just once, and can terminate it forcedly when shutdown/terminate even if it ignore SIGTERM' do
232
+ omit "SIGTERM is unavailable on Windows" if Fluent.windows?
233
+
234
+ m = Mutex.new
235
+ ary = []
236
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
237
+ ran = false
238
+ @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
239
+ m.lock
240
+ ran = true
241
+ begin
242
+ while line = io.readline
243
+ ary << line
244
+ end
245
+ rescue
246
+ # ignore
247
+ ensure
248
+ m.unlock
249
+ end
250
+ end
251
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
252
+
253
+ assert_equal [], @d.log.out.logs
254
+
255
+ @d.stop # nothing occurs
256
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
257
+ lines1 = ary.size
258
+ assert{ lines1 > 1 }
259
+
260
+ pid = @d._child_process_processes.keys.first
261
+
262
+ @d.shutdown
263
+ sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
264
+ lines2 = ary.size
265
+ assert{ lines2 > lines1 }
266
+
267
+ @d.close
268
+
269
+ assert_nil((Process.waitpid(pid, Process::WNOHANG) rescue nil))
270
+
271
+ @d.terminate
272
+ assert @d._child_process_processes.empty?
273
+ begin
274
+ Process.waitpid(pid)
275
+ rescue Errno::ECHILD
276
+ end
277
+ # Process successfully KILLed if test reaches here
278
+ assert true
279
+ end
280
+ end
281
+
282
+ test 'can execute external command many times, which finishes immediately' do
283
+ ary = []
284
+ arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 3
285
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
286
+ @d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 5, mode: [:read]) do |io|
287
+ ary << io.read.split("\n").map(&:chomp).join
288
+ end
289
+ sleep 13 # 5sec * 2 + 3sec
290
+ assert_equal [], @d.log.out.logs
291
+ @d.stop
292
+ assert_equal [], @d.log.out.logs
293
+ @d.shutdown; @d.close; @d.terminate
294
+ assert_equal 2, ary.size
295
+ end
296
+ end
297
+
298
+ test 'can execute external command many times, with leading once executed immediately' do
299
+ ary = []
300
+ arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"]
301
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
302
+ @d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 5, immediate: true, mode: [:read]) do |io|
303
+ ary << io.read.split("\n").map(&:chomp).join
304
+ end
305
+ sleep 8 # 5sec * 1 + 3sec
306
+ # but expected lines are same with test above
307
+ @d.stop; @d.shutdown; @d.close; @d.terminate
308
+ assert_equal 2, ary.size
309
+ assert_equal [], @d.log.out.logs
310
+ end
311
+ end
312
+
313
+ test 'does not execute long running external command in parallel in default' do
314
+ ary = []
315
+ arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10
316
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
317
+ assert_equal [], @d.log.out.logs
318
+ @d.log.out.singleton_class.module_eval do
319
+ define_method(:write){|message|
320
+ raise "boo" if message.include?('test: {"test":"test"}') || message.include?('test: {"test"=>"test"}')
321
+ @logs.push message
322
+ }
323
+ end
324
+
325
+ @d.child_process_execute(:t7, "ruby", arguments: arguments, interval: 2, immediate: true, mode: [:read]) do |io|
326
+ ary << io.read.split("\n").map(&:chomp).join
327
+ end
328
+ sleep 4
329
+ assert_equal 1, @d._child_process_processes.size
330
+ @d.stop
331
+ warn_msg = '[warn]: previous child process is still running. skipped. title=:t7 command="ruby" arguments=["-e", "10.times{ puts \'okay\'; STDOUT.flush rescue nil; sleep 0.5 }"] interval=2 parallel=false' + "\n"
332
+ logs = @d.log.out.logs
333
+ assert{ logs.first.end_with?(warn_msg) }
334
+ assert{ logs.all?{|line| line.end_with?(warn_msg) } }
335
+ @d.shutdown; @d.close; @d.terminate
336
+ assert_equal [], @d.log.out.logs
337
+ end
338
+ end
339
+
340
+ test 'can execute long running external command in parallel if specified' do
341
+ ary = []
342
+ arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10 sec
343
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
344
+ @d.child_process_execute(:t8, "ruby", arguments: arguments, interval: 1, immediate: true, parallel: true, mode: [:read]) do |io|
345
+ ary << io.read.split("\n").map(&:chomp).join
346
+ end
347
+ sleep 4
348
+ processes = @d._child_process_processes.size
349
+ assert{ processes >= 3 && processes <= 5 }
350
+ @d.stop; @d.shutdown; @d.close; @d.terminate
351
+ assert_equal [], @d.log.out.logs
352
+ end
353
+ end
354
+
355
+ test 'execute external processes only for writing' do
356
+ m = Mutex.new
357
+ unreadable = false
358
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
359
+ ran = false
360
+ @d.child_process_execute(:t9, "ruby", arguments: ['-e', 'a=""; while b=STDIN.readline; a+=b; end'], mode: [:write]) do |io|
361
+ m.lock
362
+ ran = true
363
+ begin
364
+ io.read
365
+ rescue IOError
366
+ unreadable = true
367
+ end
368
+ 50.times do
369
+ io.write "hahaha\n"
370
+ end
371
+ m.unlock
372
+ end
373
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
374
+ m.lock
375
+ m.unlock
376
+ assert unreadable
377
+ @d.stop; @d.shutdown; @d.close; @d.terminate
378
+ assert_equal [], @d.log.out.logs
379
+ end
380
+ end
381
+
382
+ test 'execute external processes only for reading' do
383
+ m = Mutex.new
384
+ unwritable = false
385
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
386
+ ran = false
387
+ @d.child_process_execute(:t10, "ruby", arguments: ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end"], mode: [:read]) do |io|
388
+ m.lock
389
+ ran = true
390
+ begin
391
+ io.write "foobar"
392
+ rescue IOError
393
+ unwritable = true
394
+ end
395
+ _data = io.readline
396
+ m.unlock
397
+ end
398
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
399
+ m.lock
400
+ m.unlock
401
+ @d.stop; @d.shutdown; @d.close; @d.terminate
402
+ assert unwritable
403
+ assert_equal [], @d.log.out.logs
404
+ end
405
+ end
406
+
407
+ test 'can control external encodings' do
408
+ m = Mutex.new
409
+ encodings = []
410
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
411
+ ran = false
412
+ @d.child_process_execute(:t11, "ruby -e 'sleep 10'", external_encoding: 'ascii-8bit') do |r, w|
413
+ m.lock
414
+ ran = true
415
+ encodings << r.external_encoding
416
+ encodings << w.external_encoding
417
+ m.unlock
418
+ end
419
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
420
+ m.lock
421
+ assert_equal Encoding::ASCII_8BIT, encodings[0]
422
+ assert_equal Encoding::ASCII_8BIT, encodings[1]
423
+ @d.stop; @d.shutdown; @d.close; @d.terminate
424
+ end
425
+ end
426
+
427
+ test 'can control internal encodings' do
428
+ m = Mutex.new
429
+ encodings = []
430
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
431
+ ran = false
432
+ @d.child_process_execute(:t12, "ruby -e 'sleep 10'", external_encoding: 'utf-8', internal_encoding: 'ascii-8bit') do |r, w|
433
+ m.lock
434
+ ran = true
435
+ encodings << r.internal_encoding
436
+ encodings << w.internal_encoding
437
+ m.unlock
438
+ end
439
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
440
+ m.lock
441
+ assert_equal Encoding::ASCII_8BIT, encodings[0]
442
+ assert_equal Encoding::ASCII_8BIT, encodings[1]
443
+ @d.stop; @d.shutdown; @d.close; @d.terminate
444
+ end
445
+ end
446
+
447
+ test 'can convert encodings from ascii-8bit to utf-8' do
448
+ m = Mutex.new
449
+ str = nil
450
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
451
+ ran = false
452
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xA4\xB5\xA4\xC8\xA4\xB7"']
453
+ @d.child_process_execute(:t13, "ruby", arguments: args, external_encoding: 'euc-jp', internal_encoding: 'windows-31j', mode: [:read]) do |io|
454
+ m.lock
455
+ ran = true
456
+ str = io.read
457
+ m.unlock
458
+ end
459
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
460
+ m.lock
461
+ assert_equal Encoding.find('windows-31j'), str.encoding
462
+ expected = "さとし".encode('windows-31j')
463
+ assert_equal expected, str
464
+ @d.stop; @d.shutdown; @d.close; @d.terminate
465
+ end
466
+ end
467
+
468
+ test 'can scrub characters without exceptions' do
469
+ m = Mutex.new
470
+ str = nil
471
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
472
+ ran = false
473
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
474
+ @d.child_process_execute(:t13a, "ruby", arguments: args, mode: [:read]) do |io|
475
+ m.lock
476
+ ran = true
477
+ str = io.read
478
+ m.unlock
479
+ end
480
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
481
+ m.lock
482
+ assert_equal Encoding.find('utf-8'), str.encoding
483
+ expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
484
+ assert_equal expected, str
485
+ @d.stop; @d.shutdown; @d.close; @d.terminate
486
+ end
487
+ end
488
+
489
+ test 'can scrub characters without exceptions and replace specified chars' do
490
+ m = Mutex.new
491
+ str = nil
492
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
493
+ ran = false
494
+ args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
495
+ @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
496
+ m.lock
497
+ ran = true
498
+ str = io.read
499
+ m.unlock
500
+ end
501
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
502
+ m.lock
503
+ assert_equal Encoding.find('utf-8'), str.encoding
504
+ expected = "??\x00??".force_encoding("utf-8")
505
+ assert_equal expected, str
506
+ @d.stop; @d.shutdown; @d.close; @d.terminate
507
+ end
508
+ end
509
+
510
+ unless Fluent.windows?
511
+ test 'can specify subprocess name' do
512
+ io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
513
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
514
+ Process.kill(:TERM, io.pid) rescue nil
515
+ io.close rescue nil
516
+
517
+ # Does TravisCI prohibit process renaming?
518
+ # This test will be passed in such environment
519
+ pend unless process_naming_enabled
520
+
521
+ m = Mutex.new
522
+ pids = []
523
+ proc_lines = []
524
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
525
+ ran = false
526
+ @d.child_process_execute(:t14, "ruby", arguments:['-e', 'sleep 10; puts "hello"'], subprocess_name: "sleeeeeeeeeper", mode: [:read]) do |readio|
527
+ m.lock
528
+ ran = true
529
+ pids << @d.child_process_id
530
+ proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
531
+ m.unlock
532
+ readio.read
533
+ end
534
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
535
+ m.lock
536
+ pid = pids.first
537
+ # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
538
+ assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
539
+ @d.stop; @d.shutdown; @d.close; @d.terminate
540
+ end
541
+ end
542
+ end
543
+
544
+ test 'can set ENV variables' do
545
+ m = Mutex.new
546
+ str = nil
547
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
548
+ ran = false
549
+ args = ['-e', 'puts ENV["testing_child_process"]']
550
+ @d.child_process_execute(:t15a, "ruby", arguments: args, mode: [:read], env: {'testing_child_process' => 'Yes! True!'}) do |io|
551
+ m.lock
552
+ ran = true
553
+ str = io.read
554
+ m.unlock
555
+ end
556
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
557
+ m.lock
558
+ expected = "Yes! True!\n"
559
+ assert_equal expected, str
560
+ @d.stop; @d.shutdown; @d.close; @d.terminate
561
+ end
562
+ end
563
+
564
+ test 'can unset ENV variables of Fluentd process' do
565
+ m = Mutex.new
566
+ str = nil
567
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
568
+ current_env_path = ENV['PATH']
569
+ ran = false
570
+ args = ['-e', 'puts ENV["testing_child_process1"].to_s + ENV["testing_child_process2"].to_s']
571
+ ENV['testing_child_process1'] = "No! False!"
572
+ @d.child_process_execute(:t15b, "ruby", arguments: args, mode: [:read], unsetenv: true, env: {'testing_child_process2' => 'Yes! True!', 'PATH' => current_env_path}) do |io|
573
+ m.lock
574
+ ran = true
575
+ str = io.read
576
+ m.unlock
577
+ end
578
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
579
+ m.lock
580
+ expected = "Yes! True!\n"
581
+ assert_equal expected, str
582
+ @d.stop; @d.shutdown; @d.close; @d.terminate
583
+ end
584
+ end
585
+
586
+ unless Fluent.windows?
587
+ test 'can change working directory' do
588
+ # check my real /tmp directory (for mac)
589
+ cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
590
+ mytmpdir = open(cmd){|io| io.read.chomp }
591
+
592
+ m = Mutex.new
593
+ str = nil
594
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
595
+ ran = false
596
+ args = ['-e', 'puts Dir.pwd']
597
+ @d.child_process_execute(:t16, "ruby", arguments: args, mode: [:read], chdir: "/tmp") do |io|
598
+ m.lock
599
+ ran = true
600
+ str = io.read.chomp
601
+ m.unlock
602
+ end
603
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
604
+ m.lock
605
+ assert_equal mytmpdir, str
606
+ @d.stop; @d.shutdown; @d.close; @d.terminate
607
+ end
608
+ end
609
+ end
610
+
611
+ sub_test_case 'on_exit_callback is specified' do
612
+ setup do
613
+ @temp = Tempfile.create("child_process_wait_with_on_exit_callback")
614
+ @temp_path = @temp.path
615
+ @temp.close
616
+ end
617
+
618
+ teardown do
619
+ File.unlink @temp_path if File.exist?(@temp_path)
620
+ end
621
+
622
+ test 'can return exit status for child process successfully exits using on_exit_callback' do
623
+ assert File.exist?(@temp_path)
624
+
625
+ block_exits = false
626
+ callback_called = false
627
+ exit_status = nil
628
+ args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '1', @temp_path]
629
+ cb = ->(status){ exit_status = status; callback_called = true }
630
+
631
+ str = nil
632
+
633
+ pid = nil
634
+ @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
635
+ pid = @d.instance_eval{ child_process_id }
636
+ str = readio.read.chomp
637
+ block_exits = true
638
+ end
639
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
640
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
641
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
642
+
643
+ assert callback_called
644
+ assert exit_status
645
+ assert_equal 0, exit_status.exitstatus
646
+ assert !File.exist?(@temp_path)
647
+
648
+ assert_equal "yay", str
649
+ end
650
+
651
+ test 'can return exit status with signal code for child process killed by signal using on_exit_callback' do
652
+ omit "SIGQUIT is unsupported on Windows" if Fluent.windows?
653
+
654
+ assert File.exist?(@temp_path)
655
+
656
+ block_exits = false
657
+ callback_called = false
658
+ exit_status = nil
659
+ args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '100', @temp_path]
660
+ cb = ->(status){ exit_status = status; callback_called = true }
661
+
662
+ str = nil
663
+
664
+ pid = nil
665
+ @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
666
+ pid = @d.instance_eval{ child_process_id }
667
+ sleep 10 # to run child process correctly
668
+ Process.kill(:QUIT, pid)
669
+ sleep 1
670
+ Process.kill(:QUIT, pid) rescue nil # once more to send kill
671
+ sleep 1
672
+ Process.kill(:QUIT, pid) rescue nil # just like sync
673
+ str = readio.read.chomp rescue nil # empty string before EOF
674
+ block_exits = true
675
+ end
676
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
677
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
678
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
679
+
680
+ assert callback_called
681
+ assert exit_status
682
+
683
+ # This test sometimes fails on TravisCI
684
+ # with [nil, 11] # SIGSEGV
685
+ # or with [1, nil] # ???
686
+ assert_equal [nil, 3, true, ""], [exit_status.exitstatus, exit_status.termsig, File.exist?(@temp_path), str] # SIGQUIT
687
+ # SIGSEGV looks a kind of BUG of ruby...
688
+ end
689
+
690
+ test 'calls on_exit_callback for each process exits for interval call using on_exit_callback' do
691
+ read_data_list = []
692
+ exit_status_list = []
693
+
694
+ args = ['-e', 'puts "yay"', '1']
695
+ cb = ->(status){ exit_status_list << status }
696
+
697
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
698
+ @d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 2, mode: [:read], on_exit_callback: cb) do |readio|
699
+ read_data_list << readio.read.chomp
700
+ end
701
+ sleep 10
702
+ end
703
+
704
+ assert{ read_data_list.size >= 3 }
705
+ assert{ exit_status_list.size >= 3 }
706
+ end
707
+
708
+ test 'waits lasting child process until wait_timeout if block is not specified' do
709
+ assert File.exist?(@temp_path)
710
+
711
+ callback_called = false
712
+ exit_status = nil
713
+ args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '1', @temp_path]
714
+ cb = ->(status){ exit_status = status; callback_called = true }
715
+
716
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
717
+ @d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 5)
718
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
719
+ end
720
+
721
+ assert callback_called
722
+ assert exit_status
723
+ assert_equal 0, exit_status.exitstatus
724
+ assert !File.exist?(@temp_path)
725
+ end
726
+
727
+ test 'waits lasting child process until wait_timeout after block rans if block is specified' do
728
+ assert File.exist?(@temp_path)
729
+
730
+ callback_called = false
731
+ exit_status = nil
732
+ args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '3', @temp_path]
733
+ cb = ->(status){ exit_status = status; callback_called = true }
734
+
735
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
736
+ @d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 10) do
737
+ sleep 1
738
+ end
739
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
740
+ end
741
+
742
+ assert callback_called
743
+ assert exit_status
744
+ assert_equal 0, exit_status.exitstatus
745
+ assert !File.exist?(@temp_path)
746
+ end
747
+
748
+ test 'kills lasting child process after wait_timeout if block is not specified' do
749
+ assert File.exist?(@temp_path)
750
+
751
+ callback_called = false
752
+ exit_status = nil
753
+ args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
754
+ cb = ->(status){ exit_status = status; callback_called = true }
755
+
756
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
757
+ @d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 3)
758
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
759
+ end
760
+
761
+ assert callback_called
762
+ assert exit_status
763
+ unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
764
+ assert_nil exit_status.exitstatus
765
+ assert_equal 9, exit_status.termsig # SIGKILL
766
+ end
767
+ assert File.exist?(@temp_path)
768
+ end
769
+
770
+ test 'kills lasting child process after block ran and wait_timeout expires if block is specified' do
771
+ assert File.exist?(@temp_path)
772
+
773
+ callback_called = false
774
+ exit_status = nil
775
+ args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
776
+ cb = ->(status){ exit_status = status; callback_called = true }
777
+
778
+ Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
779
+ @d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 3) do
780
+ sleep 3
781
+ end
782
+ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
783
+ end
784
+
785
+ assert callback_called
786
+ assert exit_status
787
+ unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
788
+ assert_nil exit_status.exitstatus
789
+ assert_equal 9, exit_status.termsig # SIGKILL
790
+ end
791
+ assert File.exist?(@temp_path)
792
+ end
793
+ end
794
+ end