fluentd 0.12.40 → 1.6.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (428) hide show
  1. checksums.yaml +5 -5
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  4. data/.github/ISSUE_TEMPLATE.md +17 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
  6. data/.gitignore +5 -0
  7. data/.gitlab/cicd-template.yaml +10 -0
  8. data/.gitlab-ci.yml +147 -0
  9. data/.travis.yml +56 -20
  10. data/ADOPTERS.md +5 -0
  11. data/CHANGELOG.md +1369 -0
  12. data/CONTRIBUTING.md +16 -5
  13. data/GOVERNANCE.md +55 -0
  14. data/Gemfile +5 -0
  15. data/GithubWorkflow.md +78 -0
  16. data/LICENSE +202 -0
  17. data/MAINTAINERS.md +7 -0
  18. data/README.md +23 -11
  19. data/Rakefile +48 -2
  20. data/Vagrantfile +17 -0
  21. data/appveyor.yml +37 -0
  22. data/bin/fluent-binlog-reader +7 -0
  23. data/bin/fluent-ca-generate +6 -0
  24. data/bin/fluent-plugin-config-format +5 -0
  25. data/bin/fluent-plugin-generate +5 -0
  26. data/bin/fluentd +3 -0
  27. data/code-of-conduct.md +3 -0
  28. data/example/copy_roundrobin.conf +39 -0
  29. data/example/counter.conf +18 -0
  30. data/example/in_dummy_blocks.conf +17 -0
  31. data/example/in_dummy_with_compression.conf +23 -0
  32. data/example/in_forward.conf +7 -0
  33. data/example/in_forward_client.conf +37 -0
  34. data/example/in_forward_shared_key.conf +15 -0
  35. data/example/in_forward_tls.conf +14 -0
  36. data/example/in_forward_users.conf +24 -0
  37. data/example/in_forward_workers.conf +21 -0
  38. data/example/in_http.conf +3 -1
  39. data/example/in_out_forward.conf +17 -0
  40. data/example/logevents.conf +25 -0
  41. data/example/multi_filters.conf +61 -0
  42. data/example/out_exec_filter.conf +42 -0
  43. data/example/out_forward.conf +13 -13
  44. data/example/out_forward_buf_file.conf +23 -0
  45. data/example/out_forward_client.conf +109 -0
  46. data/example/out_forward_heartbeat_none.conf +16 -0
  47. data/example/out_forward_shared_key.conf +36 -0
  48. data/example/out_forward_tls.conf +18 -0
  49. data/example/out_forward_users.conf +65 -0
  50. data/example/out_null.conf +36 -0
  51. data/example/secondary_file.conf +42 -0
  52. data/example/suppress_config_dump.conf +7 -0
  53. data/example/worker_section.conf +36 -0
  54. data/fluent.conf +29 -0
  55. data/fluentd.gemspec +21 -11
  56. data/lib/fluent/agent.rb +67 -90
  57. data/lib/fluent/clock.rb +62 -0
  58. data/lib/fluent/command/binlog_reader.rb +244 -0
  59. data/lib/fluent/command/ca_generate.rb +181 -0
  60. data/lib/fluent/command/cat.rb +42 -18
  61. data/lib/fluent/command/debug.rb +12 -10
  62. data/lib/fluent/command/fluentd.rb +153 -5
  63. data/lib/fluent/command/plugin_config_formatter.rb +292 -0
  64. data/lib/fluent/command/plugin_generator.rb +324 -0
  65. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  66. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  67. data/lib/fluent/compat/exec_util.rb +129 -0
  68. data/lib/fluent/compat/file_util.rb +54 -0
  69. data/lib/fluent/compat/filter.rb +68 -0
  70. data/lib/fluent/compat/formatter.rb +111 -0
  71. data/lib/fluent/compat/formatter_utils.rb +85 -0
  72. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  73. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  74. data/lib/fluent/compat/input.rb +49 -0
  75. data/lib/fluent/compat/output.rb +718 -0
  76. data/lib/fluent/compat/output_chain.rb +60 -0
  77. data/lib/fluent/compat/parser.rb +310 -0
  78. data/lib/fluent/compat/parser_utils.rb +40 -0
  79. data/lib/fluent/compat/propagate_default.rb +62 -0
  80. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  81. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  82. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  83. data/lib/fluent/compat/socket_util.rb +165 -0
  84. data/lib/fluent/compat/string_util.rb +34 -0
  85. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  86. data/lib/fluent/compat/type_converter.rb +90 -0
  87. data/lib/fluent/config/configure_proxy.rb +210 -62
  88. data/lib/fluent/config/dsl.rb +12 -5
  89. data/lib/fluent/config/element.rb +107 -9
  90. data/lib/fluent/config/literal_parser.rb +9 -3
  91. data/lib/fluent/config/parser.rb +4 -4
  92. data/lib/fluent/config/section.rb +51 -14
  93. data/lib/fluent/config/types.rb +28 -13
  94. data/lib/fluent/config/v1_parser.rb +3 -5
  95. data/lib/fluent/config.rb +23 -20
  96. data/lib/fluent/configurable.rb +79 -21
  97. data/lib/fluent/counter/base_socket.rb +46 -0
  98. data/lib/fluent/counter/client.rb +297 -0
  99. data/lib/fluent/counter/error.rb +86 -0
  100. data/lib/fluent/counter/mutex_hash.rb +163 -0
  101. data/lib/fluent/counter/server.rb +273 -0
  102. data/lib/fluent/counter/store.rb +205 -0
  103. data/lib/fluent/counter/validator.rb +145 -0
  104. data/lib/fluent/counter.rb +23 -0
  105. data/lib/fluent/daemon.rb +15 -0
  106. data/lib/fluent/engine.rb +102 -65
  107. data/lib/fluent/env.rb +7 -3
  108. data/lib/fluent/error.rb +30 -0
  109. data/lib/fluent/event.rb +197 -21
  110. data/lib/fluent/event_router.rb +93 -10
  111. data/lib/fluent/filter.rb +2 -50
  112. data/lib/fluent/formatter.rb +4 -293
  113. data/lib/fluent/input.rb +2 -32
  114. data/lib/fluent/label.rb +10 -2
  115. data/lib/fluent/load.rb +3 -3
  116. data/lib/fluent/log.rb +348 -81
  117. data/lib/fluent/match.rb +37 -36
  118. data/lib/fluent/mixin.rb +12 -176
  119. data/lib/fluent/msgpack_factory.rb +62 -0
  120. data/lib/fluent/output.rb +10 -612
  121. data/lib/fluent/output_chain.rb +23 -0
  122. data/lib/fluent/parser.rb +4 -800
  123. data/lib/fluent/plugin/bare_output.rb +63 -0
  124. data/lib/fluent/plugin/base.rb +192 -0
  125. data/lib/fluent/plugin/buf_file.rb +128 -174
  126. data/lib/fluent/plugin/buf_memory.rb +9 -92
  127. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  128. data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
  129. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  130. data/lib/fluent/plugin/buffer.rb +779 -0
  131. data/lib/fluent/plugin/compressable.rb +92 -0
  132. data/lib/fluent/plugin/exec_util.rb +3 -108
  133. data/lib/fluent/plugin/file_util.rb +4 -34
  134. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  135. data/lib/fluent/plugin/filter.rb +93 -0
  136. data/lib/fluent/plugin/filter_grep.rb +117 -34
  137. data/lib/fluent/plugin/filter_parser.rb +85 -62
  138. data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
  139. data/lib/fluent/plugin/filter_stdout.rb +15 -12
  140. data/lib/fluent/plugin/formatter.rb +50 -0
  141. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  142. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  143. data/lib/fluent/plugin/formatter_json.rb +55 -0
  144. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  145. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  146. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  147. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  148. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  149. data/lib/fluent/plugin/formatter_tsv.rb +38 -0
  150. data/lib/fluent/plugin/in_debug_agent.rb +17 -6
  151. data/lib/fluent/plugin/in_dummy.rb +47 -20
  152. data/lib/fluent/plugin/in_exec.rb +55 -123
  153. data/lib/fluent/plugin/in_forward.rb +299 -216
  154. data/lib/fluent/plugin/in_gc_stat.rb +14 -36
  155. data/lib/fluent/plugin/in_http.rb +204 -91
  156. data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
  157. data/lib/fluent/plugin/in_object_space.rb +13 -41
  158. data/lib/fluent/plugin/in_syslog.rb +112 -134
  159. data/lib/fluent/plugin/in_tail.rb +408 -745
  160. data/lib/fluent/plugin/in_tcp.rb +66 -9
  161. data/lib/fluent/plugin/in_udp.rb +60 -11
  162. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
  163. data/lib/fluent/plugin/input.rb +37 -0
  164. data/lib/fluent/plugin/multi_output.rb +158 -0
  165. data/lib/fluent/plugin/out_copy.rb +23 -35
  166. data/lib/fluent/plugin/out_exec.rb +67 -70
  167. data/lib/fluent/plugin/out_exec_filter.rb +204 -271
  168. data/lib/fluent/plugin/out_file.rb +267 -73
  169. data/lib/fluent/plugin/out_forward.rb +854 -325
  170. data/lib/fluent/plugin/out_null.rb +42 -9
  171. data/lib/fluent/plugin/out_relabel.rb +9 -5
  172. data/lib/fluent/plugin/out_roundrobin.rb +18 -37
  173. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  174. data/lib/fluent/plugin/out_stdout.rb +43 -10
  175. data/lib/fluent/plugin/out_stream.rb +7 -2
  176. data/lib/fluent/plugin/output.rb +1498 -0
  177. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  178. data/lib/fluent/plugin/parser.rb +191 -0
  179. data/lib/fluent/plugin/parser_apache.rb +28 -0
  180. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  181. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  182. data/lib/fluent/plugin/parser_csv.rb +39 -0
  183. data/lib/fluent/plugin/parser_json.rb +94 -0
  184. data/lib/fluent/plugin/parser_ltsv.rb +49 -0
  185. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  186. data/lib/fluent/plugin/parser_multiline.rb +106 -0
  187. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  188. data/lib/fluent/plugin/parser_none.rb +36 -0
  189. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  190. data/lib/fluent/plugin/parser_syslog.rb +142 -0
  191. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  192. data/lib/fluent/plugin/socket_util.rb +3 -143
  193. data/lib/fluent/plugin/storage.rb +84 -0
  194. data/lib/fluent/plugin/storage_local.rb +164 -0
  195. data/lib/fluent/plugin/string_util.rb +3 -15
  196. data/lib/fluent/plugin.rb +122 -121
  197. data/lib/fluent/plugin_helper/cert_option.rb +178 -0
  198. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  199. data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
  200. data/lib/fluent/plugin_helper/counter.rb +51 -0
  201. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  202. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  203. data/lib/fluent/plugin_helper/extract.rb +104 -0
  204. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  205. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  206. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  207. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  208. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  209. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  210. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  211. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  212. data/lib/fluent/plugin_helper/http_server.rb +76 -0
  213. data/lib/fluent/plugin_helper/inject.rb +151 -0
  214. data/lib/fluent/plugin_helper/parser.rb +147 -0
  215. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  216. data/lib/fluent/plugin_helper/retry_state.rb +205 -0
  217. data/lib/fluent/plugin_helper/server.rb +807 -0
  218. data/lib/fluent/plugin_helper/socket.rb +250 -0
  219. data/lib/fluent/plugin_helper/socket_option.rb +80 -0
  220. data/lib/fluent/plugin_helper/storage.rb +349 -0
  221. data/lib/fluent/plugin_helper/thread.rb +179 -0
  222. data/lib/fluent/plugin_helper/timer.rb +92 -0
  223. data/lib/fluent/plugin_helper.rb +73 -0
  224. data/lib/fluent/plugin_id.rb +80 -0
  225. data/lib/fluent/process.rb +3 -489
  226. data/lib/fluent/registry.rb +52 -10
  227. data/lib/fluent/root_agent.rb +204 -42
  228. data/lib/fluent/supervisor.rb +597 -359
  229. data/lib/fluent/system_config.rb +131 -42
  230. data/lib/fluent/test/base.rb +6 -54
  231. data/lib/fluent/test/driver/base.rb +224 -0
  232. data/lib/fluent/test/driver/base_owned.rb +70 -0
  233. data/lib/fluent/test/driver/base_owner.rb +135 -0
  234. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  235. data/lib/fluent/test/driver/filter.rb +57 -0
  236. data/lib/fluent/test/driver/formatter.rb +30 -0
  237. data/lib/fluent/test/driver/input.rb +31 -0
  238. data/lib/fluent/test/driver/multi_output.rb +53 -0
  239. data/lib/fluent/test/driver/output.rb +102 -0
  240. data/lib/fluent/test/driver/parser.rb +30 -0
  241. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  242. data/lib/fluent/test/filter_test.rb +0 -1
  243. data/lib/fluent/test/formatter_test.rb +4 -1
  244. data/lib/fluent/test/helpers.rb +58 -10
  245. data/lib/fluent/test/input_test.rb +27 -19
  246. data/lib/fluent/test/log.rb +79 -0
  247. data/lib/fluent/test/output_test.rb +28 -39
  248. data/lib/fluent/test/parser_test.rb +3 -1
  249. data/lib/fluent/test/startup_shutdown.rb +46 -0
  250. data/lib/fluent/test.rb +33 -1
  251. data/lib/fluent/time.rb +450 -1
  252. data/lib/fluent/timezone.rb +27 -3
  253. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  254. data/lib/fluent/version.rb +1 -1
  255. data/lib/fluent/winsvc.rb +85 -0
  256. data/templates/new_gem/Gemfile +3 -0
  257. data/templates/new_gem/README.md.erb +43 -0
  258. data/templates/new_gem/Rakefile +13 -0
  259. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  260. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  261. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  262. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  263. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  264. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  265. data/templates/new_gem/test/helper.rb.erb +8 -0
  266. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  267. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  268. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  269. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  270. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  271. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  272. data/templates/plugin_config_formatter/param.md.erb +34 -0
  273. data/templates/plugin_config_formatter/section.md.erb +12 -0
  274. data/test/command/test_binlog_reader.rb +346 -0
  275. data/test/command/test_ca_generate.rb +70 -0
  276. data/test/command/test_fluentd.rb +901 -0
  277. data/test/command/test_plugin_config_formatter.rb +276 -0
  278. data/test/command/test_plugin_generator.rb +92 -0
  279. data/test/compat/test_calls_super.rb +166 -0
  280. data/test/compat/test_parser.rb +92 -0
  281. data/test/config/test_config_parser.rb +126 -2
  282. data/test/config/test_configurable.rb +946 -187
  283. data/test/config/test_configure_proxy.rb +424 -74
  284. data/test/config/test_dsl.rb +11 -11
  285. data/test/config/test_element.rb +500 -0
  286. data/test/config/test_literal_parser.rb +8 -0
  287. data/test/config/test_plugin_configuration.rb +56 -0
  288. data/test/config/test_section.rb +79 -7
  289. data/test/config/test_system_config.rb +122 -35
  290. data/test/config/test_types.rb +38 -0
  291. data/test/counter/test_client.rb +559 -0
  292. data/test/counter/test_error.rb +44 -0
  293. data/test/counter/test_mutex_hash.rb +179 -0
  294. data/test/counter/test_server.rb +589 -0
  295. data/test/counter/test_store.rb +258 -0
  296. data/test/counter/test_validator.rb +137 -0
  297. data/test/helper.rb +89 -6
  298. data/test/helpers/fuzzy_assert.rb +89 -0
  299. data/test/plugin/test_bare_output.rb +118 -0
  300. data/test/plugin/test_base.rb +115 -0
  301. data/test/plugin/test_buf_file.rb +823 -460
  302. data/test/plugin/test_buf_memory.rb +32 -194
  303. data/test/plugin/test_buffer.rb +1233 -0
  304. data/test/plugin/test_buffer_chunk.rb +198 -0
  305. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  306. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  307. data/test/plugin/test_compressable.rb +84 -0
  308. data/test/plugin/test_filter.rb +357 -0
  309. data/test/plugin/test_filter_grep.rb +540 -29
  310. data/test/plugin/test_filter_parser.rb +439 -452
  311. data/test/plugin/test_filter_record_transformer.rb +123 -166
  312. data/test/plugin/test_filter_stdout.rb +160 -72
  313. data/test/plugin/test_formatter_csv.rb +111 -0
  314. data/test/plugin/test_formatter_hash.rb +35 -0
  315. data/test/plugin/test_formatter_json.rb +51 -0
  316. data/test/plugin/test_formatter_ltsv.rb +62 -0
  317. data/test/plugin/test_formatter_msgpack.rb +28 -0
  318. data/test/plugin/test_formatter_out_file.rb +95 -0
  319. data/test/plugin/test_formatter_single_value.rb +38 -0
  320. data/test/plugin/test_formatter_tsv.rb +68 -0
  321. data/test/plugin/test_in_debug_agent.rb +24 -1
  322. data/test/plugin/test_in_dummy.rb +111 -18
  323. data/test/plugin/test_in_exec.rb +200 -113
  324. data/test/plugin/test_in_forward.rb +990 -387
  325. data/test/plugin/test_in_gc_stat.rb +10 -8
  326. data/test/plugin/test_in_http.rb +600 -224
  327. data/test/plugin/test_in_monitor_agent.rb +690 -0
  328. data/test/plugin/test_in_object_space.rb +24 -8
  329. data/test/plugin/test_in_syslog.rb +154 -215
  330. data/test/plugin/test_in_tail.rb +1006 -707
  331. data/test/plugin/test_in_tcp.rb +125 -48
  332. data/test/plugin/test_in_udp.rb +204 -63
  333. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
  334. data/test/plugin/test_input.rb +126 -0
  335. data/test/plugin/test_metadata.rb +89 -0
  336. data/test/plugin/test_multi_output.rb +180 -0
  337. data/test/plugin/test_out_copy.rb +117 -112
  338. data/test/plugin/test_out_exec.rb +258 -53
  339. data/test/plugin/test_out_exec_filter.rb +538 -115
  340. data/test/plugin/test_out_file.rb +865 -178
  341. data/test/plugin/test_out_forward.rb +998 -210
  342. data/test/plugin/test_out_null.rb +105 -0
  343. data/test/plugin/test_out_relabel.rb +28 -0
  344. data/test/plugin/test_out_roundrobin.rb +36 -29
  345. data/test/plugin/test_out_secondary_file.rb +458 -0
  346. data/test/plugin/test_out_stdout.rb +135 -37
  347. data/test/plugin/test_out_stream.rb +18 -0
  348. data/test/plugin/test_output.rb +984 -0
  349. data/test/plugin/test_output_as_buffered.rb +2021 -0
  350. data/test/plugin/test_output_as_buffered_backup.rb +312 -0
  351. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  352. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  353. data/test/plugin/test_output_as_buffered_retries.rb +911 -0
  354. data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
  355. data/test/plugin/test_output_as_standard.rb +374 -0
  356. data/test/plugin/test_owned_by.rb +35 -0
  357. data/test/plugin/test_parser.rb +359 -0
  358. data/test/plugin/test_parser_apache.rb +42 -0
  359. data/test/plugin/test_parser_apache2.rb +47 -0
  360. data/test/plugin/test_parser_apache_error.rb +45 -0
  361. data/test/plugin/test_parser_csv.rb +103 -0
  362. data/test/plugin/test_parser_json.rb +138 -0
  363. data/test/plugin/test_parser_labeled_tsv.rb +145 -0
  364. data/test/plugin/test_parser_multiline.rb +100 -0
  365. data/test/plugin/test_parser_nginx.rb +88 -0
  366. data/test/plugin/test_parser_none.rb +52 -0
  367. data/test/plugin/test_parser_regexp.rb +289 -0
  368. data/test/plugin/test_parser_syslog.rb +441 -0
  369. data/test/plugin/test_parser_tsv.rb +122 -0
  370. data/test/plugin/test_storage.rb +167 -0
  371. data/test/plugin/test_storage_local.rb +335 -0
  372. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  373. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  374. data/test/plugin_helper/data/cert/cert.pem +19 -0
  375. data/test/plugin_helper/http_server/test_app.rb +65 -0
  376. data/test/plugin_helper/http_server/test_route.rb +32 -0
  377. data/test/plugin_helper/test_cert_option.rb +16 -0
  378. data/test/plugin_helper/test_child_process.rb +794 -0
  379. data/test/plugin_helper/test_compat_parameters.rb +353 -0
  380. data/test/plugin_helper/test_event_emitter.rb +51 -0
  381. data/test/plugin_helper/test_event_loop.rb +52 -0
  382. data/test/plugin_helper/test_extract.rb +194 -0
  383. data/test/plugin_helper/test_formatter.rb +255 -0
  384. data/test/plugin_helper/test_http_server_helper.rb +205 -0
  385. data/test/plugin_helper/test_inject.rb +519 -0
  386. data/test/plugin_helper/test_parser.rb +264 -0
  387. data/test/plugin_helper/test_record_accessor.rb +197 -0
  388. data/test/plugin_helper/test_retry_state.rb +442 -0
  389. data/test/plugin_helper/test_server.rb +1714 -0
  390. data/test/plugin_helper/test_storage.rb +542 -0
  391. data/test/plugin_helper/test_thread.rb +164 -0
  392. data/test/plugin_helper/test_timer.rb +132 -0
  393. data/test/scripts/exec_script.rb +0 -6
  394. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  395. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  396. data/test/scripts/fluent/plugin/out_test.rb +23 -15
  397. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  398. data/test/test_clock.rb +164 -0
  399. data/test/test_config.rb +16 -7
  400. data/test/test_configdsl.rb +2 -2
  401. data/test/test_event.rb +360 -13
  402. data/test/test_event_router.rb +108 -11
  403. data/test/test_event_time.rb +199 -0
  404. data/test/test_filter.rb +48 -6
  405. data/test/test_formatter.rb +11 -391
  406. data/test/test_input.rb +1 -1
  407. data/test/test_log.rb +591 -31
  408. data/test/test_mixin.rb +1 -1
  409. data/test/test_output.rb +121 -185
  410. data/test/test_plugin.rb +251 -0
  411. data/test/test_plugin_classes.rb +177 -10
  412. data/test/test_plugin_helper.rb +81 -0
  413. data/test/test_plugin_id.rb +101 -0
  414. data/test/test_process.rb +8 -42
  415. data/test/test_root_agent.rb +766 -21
  416. data/test/test_supervisor.rb +481 -0
  417. data/test/test_test_drivers.rb +135 -0
  418. data/test/test_time_formatter.rb +282 -0
  419. data/test/test_time_parser.rb +231 -0
  420. data/test/test_unique_id.rb +47 -0
  421. metadata +454 -60
  422. data/COPYING +0 -14
  423. data/ChangeLog +0 -666
  424. data/lib/fluent/buffer.rb +0 -365
  425. data/lib/fluent/plugin/in_status.rb +0 -76
  426. data/test/plugin/test_in_status.rb +0 -38
  427. data/test/test_buffer.rb +0 -624
  428. data/test/test_parser.rb +0 -1305
@@ -0,0 +1,807 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/event_loop'
18
+
19
+ require 'serverengine'
20
+ require 'cool.io'
21
+ require 'socket'
22
+ require 'ipaddr'
23
+ require 'fcntl'
24
+ require 'openssl'
25
+
26
+ require_relative 'socket_option'
27
+ require_relative 'cert_option'
28
+
29
+ module Fluent
30
+ module PluginHelper
31
+ module Server
32
+ include Fluent::PluginHelper::EventLoop
33
+ include Fluent::PluginHelper::SocketOption
34
+ include Fluent::PluginHelper::CertOption
35
+
36
+ # This plugin helper doesn't support these things for now:
37
+ # * TCP/TLS keepalive
38
+ # * TLS session cache/tickets
39
+ # * unix domain sockets
40
+
41
+ # stop : [-]
42
+ # shutdown : detach server event handler from event loop (event_loop)
43
+ # close : close listening sockets
44
+ # terminate: remote all server instances
45
+
46
+ attr_reader :_servers # for tests
47
+
48
+ def server_wait_until_start
49
+ # event_loop_wait_until_start works well for this
50
+ end
51
+
52
+ def server_wait_until_stop
53
+ sleep 0.1 while @_servers.any?{|si| si.server.attached? }
54
+ @_servers.each{|si| si.server.close rescue nil }
55
+ end
56
+
57
+ PROTOCOLS = [:tcp, :udp, :tls, :unix]
58
+ CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
59
+
60
+ # server_create_connection(:title, @port) do |conn|
61
+ # # on connection
62
+ # source_addr = conn.remote_host
63
+ # source_port = conn.remote_port
64
+ # conn.data do |data|
65
+ # # on data
66
+ # conn.write resp # ...
67
+ # conn.close
68
+ # end
69
+ # end
70
+ def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)
71
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
72
+
73
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
74
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
75
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
76
+ raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
77
+
78
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
79
+
80
+ raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
81
+ raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
82
+
83
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
84
+ socket_options[:linger_timeout] ||= 0
85
+ end
86
+
87
+ socket_option_validate!(proto, **socket_options)
88
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
89
+
90
+ case proto
91
+ when :tcp
92
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
93
+ when :tls
94
+ transport_config = if tls_options
95
+ server_create_transport_section_object(tls_options)
96
+ elsif @transport_config && @transport_config.protocol == :tls
97
+ @transport_config
98
+ else
99
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
100
+ end
101
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)
102
+ when :unix
103
+ raise "not implemented yet"
104
+ else
105
+ raise "unknown protocol #{proto}"
106
+ end
107
+
108
+ server_attach(title, proto, port, bind, shared, server)
109
+ end
110
+
111
+ # server_create(:title, @port) do |data|
112
+ # # ...
113
+ # end
114
+ # server_create(:title, @port) do |data, conn|
115
+ # # ...
116
+ # end
117
+ # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
118
+ # sock.remote_host
119
+ # sock.remote_port
120
+ # # ...
121
+ # end
122
+ def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
123
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
124
+
125
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
126
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
127
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
128
+
129
+ raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
130
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
131
+
132
+ raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
133
+ raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
134
+
135
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
136
+ socket_options[:linger_timeout] ||= 0
137
+ end
138
+
139
+ unless socket
140
+ socket_option_validate!(proto, **socket_options)
141
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
142
+ end
143
+
144
+ if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
145
+ raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
146
+ end
147
+ if proto != :udp # UDP options
148
+ raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
149
+ raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
150
+ end
151
+
152
+ case proto
153
+ when :tcp
154
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
155
+ conn.data(&callback)
156
+ end
157
+ when :tls
158
+ transport_config = if tls_options
159
+ server_create_transport_section_object(tls_options)
160
+ elsif @transport_config && @transport_config.protocol == :tls
161
+ @transport_config
162
+ else
163
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
164
+ end
165
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|
166
+ conn.data(&callback)
167
+ end
168
+ when :udp
169
+ raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
170
+ if socket
171
+ sock = socket
172
+ close_socket = false
173
+ else
174
+ sock = server_create_udp_socket(shared, bind, port)
175
+ socket_option_setter.call(sock)
176
+ close_socket = true
177
+ end
178
+ server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
179
+ when :unix
180
+ raise "not implemented yet"
181
+ else
182
+ raise "BUG: unknown protocol #{proto}"
183
+ end
184
+
185
+ server_attach(title, proto, port, bind, shared, server)
186
+ end
187
+
188
+ def server_create_tcp(title, port, **kwargs, &callback)
189
+ server_create(title, port, proto: :tcp, **kwargs, &callback)
190
+ end
191
+
192
+ def server_create_udp(title, port, **kwargs, &callback)
193
+ server_create(title, port, proto: :udp, **kwargs, &callback)
194
+ end
195
+
196
+ def server_create_tls(title, port, **kwargs, &callback)
197
+ server_create(title, port, proto: :tls, **kwargs, &callback)
198
+ end
199
+
200
+ def server_create_unix(title, port, **kwargs, &callback)
201
+ server_create(title, port, proto: :unix, **kwargs, &callback)
202
+ end
203
+
204
+ ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
205
+
206
+ def server_attach(title, proto, port, bind, shared, server)
207
+ @_servers << ServerInfo.new(title, proto, port, bind, shared, server)
208
+ event_loop_attach(server)
209
+ end
210
+
211
+ def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
212
+ sock = server_create_tcp_socket(shared, bind, port)
213
+ socket_option_setter.call(sock)
214
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
215
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
216
+ unless conn.closing
217
+ @_server_mutex.synchronize do
218
+ @_server_connections << conn
219
+ end
220
+ end
221
+ end
222
+ server.listen(backlog) if backlog
223
+ server
224
+ end
225
+
226
+ def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)
227
+ context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
228
+ sock = server_create_tcp_socket(shared, bind, port)
229
+ socket_option_setter.call(sock)
230
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
231
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
232
+ unless conn.closing
233
+ @_server_mutex.synchronize do
234
+ @_server_connections << conn
235
+ end
236
+ end
237
+ end
238
+ server.listen(backlog) if backlog
239
+ server
240
+ end
241
+
242
+ SERVER_TRANSPORT_PARAMS = [
243
+ :protocol, :version, :ciphers, :insecure,
244
+ :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
245
+ :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
246
+ :generate_private_key_length,
247
+ :generate_cert_country, :generate_cert_state, :generate_cert_state,
248
+ :generate_cert_locality, :generate_cert_common_name,
249
+ :generate_cert_expiration, :generate_cert_digest,
250
+ ]
251
+
252
+ def server_create_transport_section_object(opts)
253
+ transport_section = configured_section_create(:transport)
254
+ SERVER_TRANSPORT_PARAMS.each do |param|
255
+ if opts.has_key?(param)
256
+ transport_section[param] = opts[param]
257
+ end
258
+ end
259
+ transport_section
260
+ end
261
+
262
+ module ServerTransportParams
263
+ TLS_DEFAULT_VERSION = :'TLSv1_2'
264
+ TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
265
+ ### follow httpclient configuration by nahi
266
+ # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
267
+ CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
268
+
269
+ include Fluent::Configurable
270
+ config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
271
+ config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
272
+ config_param :version, :enum, list: TLS_SUPPORTED_VERSIONS, default: TLS_DEFAULT_VERSION
273
+
274
+ config_param :ciphers, :string, default: CIPHERS_DEFAULT
275
+ config_param :insecure, :bool, default: false
276
+
277
+ # Cert signed by public CA
278
+ config_param :ca_path, :string, default: nil
279
+ config_param :cert_path, :string, default: nil
280
+ config_param :private_key_path, :string, default: nil
281
+ config_param :private_key_passphrase, :string, default: nil, secret: true
282
+ config_param :client_cert_auth, :bool, default: false
283
+
284
+ # Cert generated and signed by private CA Certificate
285
+ config_param :ca_cert_path, :string, default: nil
286
+ config_param :ca_private_key_path, :string, default: nil
287
+ config_param :ca_private_key_passphrase, :string, default: nil, secret: true
288
+
289
+ # Options for generating certs by private CA certs or self-signed
290
+ config_param :generate_private_key_length, :integer, default: 2048
291
+ config_param :generate_cert_country, :string, default: 'US'
292
+ config_param :generate_cert_state, :string, default: 'CA'
293
+ config_param :generate_cert_locality, :string, default: 'Mountain View'
294
+ config_param :generate_cert_common_name, :string, default: nil
295
+ config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later
296
+ config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256
297
+ end
298
+ end
299
+
300
+ def self.included(mod)
301
+ mod.include ServerTransportParams
302
+ end
303
+
304
+ def initialize
305
+ super
306
+ @_servers = []
307
+ @_server_connections = []
308
+ @_server_mutex = Mutex.new
309
+ end
310
+
311
+ def configure(conf)
312
+ super
313
+
314
+ if @transport_config
315
+ if @transport_config.protocol == :tls
316
+ cert_option_server_validate!(@transport_config)
317
+ end
318
+ end
319
+ end
320
+
321
+ def stop
322
+ @_server_mutex.synchronize do
323
+ @_servers.each do |si|
324
+ si.server.detach if si.server.attached?
325
+ # to refuse more connections: (connected sockets are still alive here)
326
+ si.server.close rescue nil
327
+ end
328
+ end
329
+
330
+ super
331
+ end
332
+
333
+ def shutdown
334
+ @_server_connections.each do |conn|
335
+ conn.close rescue nil
336
+ end
337
+
338
+ super
339
+ end
340
+
341
+ def terminate
342
+ @_servers = []
343
+ super
344
+ end
345
+
346
+ def server_socket_manager_client
347
+ socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
348
+ if Fluent.windows?
349
+ socket_manager_path = socket_manager_path.to_i
350
+ end
351
+ ServerEngine::SocketManager::Client.new(socket_manager_path)
352
+ end
353
+
354
+ def server_create_tcp_socket(shared, bind, port)
355
+ sock = if shared
356
+ server_socket_manager_client.listen_tcp(bind, port)
357
+ else
358
+ TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
359
+ end
360
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
361
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
362
+ sock
363
+ end
364
+
365
+ def server_create_udp_socket(shared, bind, port)
366
+ sock = if shared
367
+ server_socket_manager_client.listen_udp(bind, port)
368
+ else
369
+ family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
370
+ usock = UDPSocket.new(family)
371
+ usock.bind(bind, port)
372
+ usock
373
+ end
374
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
375
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
376
+ sock
377
+ end
378
+
379
+ # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
380
+ PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
381
+
382
+ class CallbackSocket
383
+ def initialize(server_type, sock, enabled_events = [], close_socket: true)
384
+ @server_type = server_type
385
+ @sock = sock
386
+ @peeraddr = nil
387
+ @enabled_events = enabled_events
388
+ @close_socket = close_socket
389
+ end
390
+
391
+ def remote_addr
392
+ @peeraddr[3]
393
+ end
394
+
395
+ def remote_host
396
+ @peeraddr[2]
397
+ end
398
+
399
+ def remote_port
400
+ @peeraddr[1]
401
+ end
402
+
403
+ def send(data, flags = 0)
404
+ @sock.send(data, flags)
405
+ end
406
+
407
+ def write(data)
408
+ raise "not implemented here"
409
+ end
410
+
411
+ def close_after_write_complete
412
+ @sock.close_after_write_complete = true
413
+ end
414
+
415
+ def close
416
+ @sock.close if @close_socket
417
+ end
418
+
419
+ def data(&callback)
420
+ on(:data, &callback)
421
+ end
422
+
423
+ def on(event, &callback)
424
+ raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
425
+ case event
426
+ when :data
427
+ @sock.data(&callback)
428
+ when :write_complete
429
+ cb = ->(){ callback.call(self) }
430
+ @sock.on_write_complete(&cb)
431
+ when :close
432
+ cb = ->(){ callback.call(self) }
433
+ @sock.on_close(&cb)
434
+ else
435
+ raise "BUG: unknown event: #{event}"
436
+ end
437
+ end
438
+ end
439
+
440
+ class TCPCallbackSocket < CallbackSocket
441
+ ENABLED_EVENTS = [:data, :write_complete, :close]
442
+
443
+ attr_accessor :buffer
444
+
445
+ def initialize(sock)
446
+ super("tcp", sock, ENABLED_EVENTS)
447
+ @peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)
448
+ @buffer = ''
449
+ end
450
+
451
+ def write(data)
452
+ @sock.write(data)
453
+ end
454
+ end
455
+
456
+ class TLSCallbackSocket < CallbackSocket
457
+ ENABLED_EVENTS = [:data, :write_complete, :close]
458
+
459
+ attr_accessor :buffer
460
+
461
+ def initialize(sock)
462
+ super("tls", sock, ENABLED_EVENTS)
463
+ @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
464
+ @buffer = ''
465
+ end
466
+
467
+ def write(data)
468
+ @sock.write(data)
469
+ end
470
+ end
471
+
472
+ class UDPCallbackSocket < CallbackSocket
473
+ ENABLED_EVENTS = []
474
+
475
+ def initialize(sock, peeraddr, **kwargs)
476
+ super("udp", sock, ENABLED_EVENTS, **kwargs)
477
+ @peeraddr = peeraddr
478
+ end
479
+
480
+ def remote_addr
481
+ @peeraddr[3]
482
+ end
483
+
484
+ def remote_host
485
+ @peeraddr[2]
486
+ end
487
+
488
+ def remote_port
489
+ @peeraddr[1]
490
+ end
491
+
492
+ def write(data)
493
+ @sock.send(data, 0, @peeraddr[3], @peeraddr[1])
494
+ end
495
+ end
496
+
497
+ module EventHandler
498
+ class UDPServer < Coolio::IO
499
+ attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
500
+
501
+ def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
502
+ raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
503
+
504
+ super(sock)
505
+
506
+ @sock = sock
507
+ @max_bytes = max_bytes
508
+ @flags = flags
509
+ @close_socket = close_socket
510
+ @log = log
511
+ @under_plugin_development = under_plugin_development
512
+ @callback = callback
513
+
514
+ on_readable_impl = case @callback.arity
515
+ when 1 then :on_readable_without_sock
516
+ when 2 then :on_readable_with_sock
517
+ else
518
+ raise "BUG: callback block must have 1 or 2 arguments"
519
+ end
520
+ self.define_singleton_method(:on_readable, method(on_readable_impl))
521
+ end
522
+
523
+ def on_readable_without_sock
524
+ begin
525
+ data = @sock.recv(@max_bytes, @flags)
526
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
527
+ return
528
+ end
529
+ @callback.call(data)
530
+ rescue => e
531
+ @log.error "unexpected error in processing UDP data", error: e
532
+ @log.error_backtrace
533
+ raise if @under_plugin_development
534
+ end
535
+
536
+ def on_readable_with_sock
537
+ begin
538
+ data, addr = @sock.recvfrom(@max_bytes)
539
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
540
+ return
541
+ end
542
+ @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
543
+ rescue => e
544
+ @log.error "unexpected error in processing UDP data", error: e
545
+ @log.error_backtrace
546
+ raise if @under_plugin_development
547
+ end
548
+ end
549
+
550
+ class TCPServer < Coolio::TCPSocket
551
+ attr_reader :closing
552
+ attr_writer :close_after_write_complete
553
+
554
+ def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
555
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
556
+
557
+ socket_option_setter.call(sock)
558
+
559
+ @_handler_socket = sock
560
+ super(sock)
561
+
562
+ @log = log
563
+ @under_plugin_development = under_plugin_development
564
+
565
+ @connect_callback = connect_callback
566
+ @data_callback = nil
567
+ @close_callback = close_callback
568
+
569
+ @callback_connection = nil
570
+ @close_after_write_complete = false
571
+ @closing = false
572
+
573
+ @mutex = Mutex.new # to serialize #write and #close
574
+ end
575
+
576
+ def to_io
577
+ @_handler_socket
578
+ end
579
+
580
+ def data(&callback)
581
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
582
+ @data_callback = callback
583
+ on_read_impl = case callback.arity
584
+ when 1 then :on_read_without_connection
585
+ when 2 then :on_read_with_connection
586
+ else
587
+ raise "BUG: callback block must have 1 or 2 arguments"
588
+ end
589
+ self.define_singleton_method(:on_read, method(on_read_impl))
590
+ end
591
+
592
+ def write(data)
593
+ @mutex.synchronize do
594
+ super
595
+ end
596
+ end
597
+
598
+ def on_writable
599
+ super
600
+ close if @close_after_write_complete
601
+ end
602
+
603
+ def on_connect
604
+ @callback_connection = TCPCallbackSocket.new(self)
605
+ @connect_callback.call(@callback_connection)
606
+ unless @data_callback
607
+ raise "connection callback must call #data to set data callback"
608
+ end
609
+ end
610
+
611
+ def on_read_without_connection(data)
612
+ @data_callback.call(data)
613
+ rescue => e
614
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
615
+ @log.error_backtrace
616
+ close rescue nil
617
+ raise if @under_plugin_development
618
+ end
619
+
620
+ def on_read_with_connection(data)
621
+ @data_callback.call(data, @callback_connection)
622
+ rescue => e
623
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
624
+ @log.error_backtrace
625
+ close rescue nil
626
+ raise if @under_plugin_development
627
+ end
628
+
629
+ def close
630
+ @mutex.synchronize do
631
+ return if @closing
632
+ @closing = true
633
+ @close_callback.call(self)
634
+ super
635
+ end
636
+ end
637
+ end
638
+
639
+ class TLSServer < Coolio::Socket
640
+ attr_reader :closing
641
+ attr_writer :close_after_write_complete
642
+
643
+ # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
644
+ def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
645
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
646
+
647
+ socket_option_setter.call(sock)
648
+ @_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)
649
+ @_handler_socket.sync_close = true
650
+ @_handler_write_buffer = ''.force_encoding('ascii-8bit')
651
+ @_handler_accepted = false
652
+ super(@_handler_socket)
653
+
654
+ @log = log
655
+ @under_plugin_development = under_plugin_development
656
+
657
+ @connect_callback = connect_callback
658
+ @data_callback = nil
659
+ @close_callback = close_callback
660
+
661
+ @callback_connection = nil
662
+ @close_after_write_complete = false
663
+ @closing = false
664
+
665
+ @mutex = Mutex.new # to serialize #write and #close
666
+ end
667
+
668
+ def to_io
669
+ @_handler_socket.to_io
670
+ end
671
+
672
+ def data(&callback)
673
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
674
+ @data_callback = callback
675
+ on_read_impl = case callback.arity
676
+ when 1 then :on_read_without_connection
677
+ when 2 then :on_read_with_connection
678
+ else
679
+ raise "BUG: callback block must have 1 or 2 arguments"
680
+ end
681
+ self.define_singleton_method(:on_read, method(on_read_impl))
682
+ end
683
+
684
+ def write(data)
685
+ @mutex.synchronize do
686
+ @_handler_write_buffer << data
687
+ schedule_write
688
+ data.bytesize
689
+ end
690
+ end
691
+
692
+ if RUBY_VERSION.to_f >= 2.3
693
+ NONBLOCK_ARG = {exception: false}
694
+ def try_handshake
695
+ @_handler_socket.accept_nonblock(NONBLOCK_ARG)
696
+ end
697
+ else
698
+ def try_handshake
699
+ @_handler_socket.accept_nonblock
700
+ rescue IO::WaitReadable
701
+ :wait_readable
702
+ rescue IO::WaitWritable
703
+ :wait_writable
704
+ end
705
+ end
706
+
707
+ def try_tls_accept
708
+ return true if @_handler_accepted
709
+
710
+ begin
711
+ result = try_handshake # this method call actually try to do handshake via TLS
712
+ if result == :wait_readable || result == :wait_writable
713
+ # retry accept_nonblock: there aren't enough data in underlying socket buffer
714
+ else
715
+ @_handler_accepted = true
716
+
717
+ @callback_connection = TLSCallbackSocket.new(self)
718
+ @connect_callback.call(@callback_connection)
719
+ unless @data_callback
720
+ raise "connection callback must call #data to set data callback"
721
+ end
722
+
723
+ return true
724
+ end
725
+ rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
726
+ @log.trace "unexpected error before accepting TLS connection", error: e
727
+ close rescue nil
728
+ end
729
+
730
+ false
731
+ end
732
+
733
+ def on_connect
734
+ try_tls_accept
735
+ end
736
+
737
+ def on_readable
738
+ if try_tls_accept
739
+ super
740
+ end
741
+ rescue IO::WaitReadable, IO::WaitWritable
742
+ # ignore and return with doing nothing
743
+ rescue OpenSSL::SSL::SSLError => e
744
+ @log.warn "close socket due to unexpected ssl error: #{e}"
745
+ close rescue nil
746
+ end
747
+
748
+ def on_writable
749
+ begin
750
+ @mutex.synchronize do
751
+ # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
752
+ written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
753
+ @_handler_write_buffer.slice!(0, written_bytes)
754
+ end
755
+
756
+ # No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.
757
+ # Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.
758
+ # Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.
759
+ # If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete
760
+ # eventually calls TLSServer#close which try to get a lock.
761
+ super
762
+
763
+ close if @close_after_write_complete
764
+ rescue IO::WaitWritable, IO::WaitReadable
765
+ return
766
+ rescue Errno::EINTR
767
+ return
768
+ rescue SystemCallError, IOError, SocketError
769
+ # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
770
+ close rescue nil
771
+ return
772
+ rescue OpenSSL::SSL::SSLError => e
773
+ @log.debug "unexpected SSLError while writing data into socket connected via TLS", error: e
774
+ end
775
+ end
776
+
777
+ def on_read_without_connection(data)
778
+ @data_callback.call(data)
779
+ rescue => e
780
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
781
+ @log.error_backtrace
782
+ close rescue nil
783
+ raise if @under_plugin_development
784
+ end
785
+
786
+ def on_read_with_connection(data)
787
+ @data_callback.call(data, @callback_connection)
788
+ rescue => e
789
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
790
+ @log.error_backtrace
791
+ close rescue nil
792
+ raise if @under_plugin_development
793
+ end
794
+
795
+ def close
796
+ @mutex.synchronize do
797
+ return if @closing
798
+ @closing = true
799
+ @close_callback.call(self)
800
+ super
801
+ end
802
+ end
803
+ end
804
+ end
805
+ end
806
+ end
807
+ end