fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,801 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/event_loop'
18
+
19
+ require 'serverengine'
20
+ require 'cool.io'
21
+ require 'socket'
22
+ require 'ipaddr'
23
+ require 'fcntl'
24
+ require 'openssl'
25
+
26
+ require_relative 'socket_option'
27
+ require_relative 'cert_option'
28
+
29
+ module Fluent
30
+ module PluginHelper
31
+ module Server
32
+ include Fluent::PluginHelper::EventLoop
33
+ include Fluent::PluginHelper::SocketOption
34
+ include Fluent::PluginHelper::CertOption
35
+
36
+ # This plugin helper doesn't support these things for now:
37
+ # * TCP/TLS keepalive
38
+ # * TLS session cache/tickets
39
+ # * unix domain sockets
40
+
41
+ # stop : [-]
42
+ # shutdown : detach server event handler from event loop (event_loop)
43
+ # close : close listening sockets
44
+ # terminate: remote all server instances
45
+
46
+ attr_reader :_servers # for tests
47
+
48
+ def server_wait_until_start
49
+ # event_loop_wait_until_start works well for this
50
+ end
51
+
52
+ def server_wait_until_stop
53
+ sleep 0.1 while @_servers.any?{|si| si.server.attached? }
54
+ @_servers.each{|si| si.server.close rescue nil }
55
+ end
56
+
57
+ PROTOCOLS = [:tcp, :udp, :tls, :unix]
58
+ CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
59
+
60
+ # server_create_connection(:title, @port) do |conn|
61
+ # # on connection
62
+ # source_addr = conn.remote_host
63
+ # source_port = conn.remote_port
64
+ # conn.data do |data|
65
+ # # on data
66
+ # conn.write resp # ...
67
+ # conn.close
68
+ # end
69
+ # end
70
+ def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)
71
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
72
+
73
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
74
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
75
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
76
+ raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
77
+
78
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
79
+
80
+ raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
81
+ raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
82
+
83
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
84
+ socket_options[:linger_timeout] ||= 0
85
+ end
86
+
87
+ socket_option_validate!(proto, **socket_options)
88
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
89
+
90
+ case proto
91
+ when :tcp
92
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
93
+ when :tls
94
+ transport_config = if tls_options
95
+ server_create_transport_section_object(tls_options)
96
+ elsif @transport_config && @transport_config.protocol == :tls
97
+ @transport_config
98
+ else
99
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
100
+ end
101
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)
102
+ when :unix
103
+ raise "not implemented yet"
104
+ else
105
+ raise "unknown protocol #{proto}"
106
+ end
107
+
108
+ server_attach(title, proto, port, bind, shared, server)
109
+ end
110
+
111
+ # server_create(:title, @port) do |data|
112
+ # # ...
113
+ # end
114
+ # server_create(:title, @port) do |data, conn|
115
+ # # ...
116
+ # end
117
+ # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
118
+ # sock.remote_host
119
+ # sock.remote_port
120
+ # # ...
121
+ # end
122
+ def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
123
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
124
+
125
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
126
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
127
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
128
+
129
+ raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
130
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
131
+
132
+ raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
133
+ raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
134
+
135
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
136
+ socket_options[:linger_timeout] ||= 0
137
+ end
138
+
139
+ unless socket
140
+ socket_option_validate!(proto, **socket_options)
141
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
142
+ end
143
+
144
+ if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
145
+ raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
146
+ end
147
+ if proto != :udp # UDP options
148
+ raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
149
+ raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
150
+ end
151
+
152
+ case proto
153
+ when :tcp
154
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
155
+ conn.data(&callback)
156
+ end
157
+ when :tls
158
+ transport_config = if tls_options
159
+ server_create_transport_section_object(tls_options)
160
+ elsif @transport_config && @transport_config.protocol == :tls
161
+ @transport_config
162
+ else
163
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
164
+ end
165
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|
166
+ conn.data(&callback)
167
+ end
168
+ when :udp
169
+ raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
170
+ if socket
171
+ sock = socket
172
+ close_socket = false
173
+ else
174
+ sock = server_create_udp_socket(shared, bind, port)
175
+ socket_option_setter.call(sock)
176
+ close_socket = true
177
+ end
178
+ server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
179
+ when :unix
180
+ raise "not implemented yet"
181
+ else
182
+ raise "BUG: unknown protocol #{proto}"
183
+ end
184
+
185
+ server_attach(title, proto, port, bind, shared, server)
186
+ end
187
+
188
+ def server_create_tcp(title, port, **kwargs, &callback)
189
+ server_create(title, port, proto: :tcp, **kwargs, &callback)
190
+ end
191
+
192
+ def server_create_udp(title, port, **kwargs, &callback)
193
+ server_create(title, port, proto: :udp, **kwargs, &callback)
194
+ end
195
+
196
+ def server_create_tls(title, port, **kwargs, &callback)
197
+ server_create(title, port, proto: :tls, **kwargs, &callback)
198
+ end
199
+
200
+ def server_create_unix(title, port, **kwargs, &callback)
201
+ server_create(title, port, proto: :unix, **kwargs, &callback)
202
+ end
203
+
204
+ ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
205
+
206
+ def server_attach(title, proto, port, bind, shared, server)
207
+ @_servers << ServerInfo.new(title, proto, port, bind, shared, server)
208
+ event_loop_attach(server)
209
+ end
210
+
211
+ def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
212
+ sock = server_create_tcp_socket(shared, bind, port)
213
+ socket_option_setter.call(sock)
214
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
215
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
216
+ unless conn.closing
217
+ @_server_mutex.synchronize do
218
+ @_server_connections << conn
219
+ end
220
+ end
221
+ end
222
+ server.listen(backlog) if backlog
223
+ server
224
+ end
225
+
226
+ def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)
227
+ context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
228
+ sock = server_create_tcp_socket(shared, bind, port)
229
+ socket_option_setter.call(sock)
230
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
231
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
232
+ unless conn.closing
233
+ @_server_mutex.synchronize do
234
+ @_server_connections << conn
235
+ end
236
+ end
237
+ end
238
+ server.listen(backlog) if backlog
239
+ server
240
+ end
241
+
242
+ SERVER_TRANSPORT_PARAMS = [
243
+ :protocol, :version, :min_version, :max_version, :ciphers, :insecure,
244
+ :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
245
+ :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
246
+ :cert_verifier, :generate_private_key_length,
247
+ :generate_cert_country, :generate_cert_state, :generate_cert_state,
248
+ :generate_cert_locality, :generate_cert_common_name,
249
+ :generate_cert_expiration, :generate_cert_digest,
250
+ ]
251
+
252
+ def server_create_transport_section_object(opts)
253
+ transport_section = configured_section_create(:transport)
254
+ SERVER_TRANSPORT_PARAMS.each do |param|
255
+ if opts.has_key?(param)
256
+ transport_section[param] = opts[param]
257
+ end
258
+ end
259
+ transport_section
260
+ end
261
+
262
+ module ServerTransportParams
263
+ include Fluent::Configurable
264
+ config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
265
+ config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
266
+ config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
267
+ config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
268
+ config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
269
+ config_param :ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
270
+ config_param :insecure, :bool, default: false
271
+
272
+ # Cert signed by public CA
273
+ config_param :ca_path, :string, default: nil
274
+ config_param :cert_path, :string, default: nil
275
+ config_param :private_key_path, :string, default: nil
276
+ config_param :private_key_passphrase, :string, default: nil, secret: true
277
+ config_param :client_cert_auth, :bool, default: false
278
+
279
+ # Cert generated and signed by private CA Certificate
280
+ config_param :ca_cert_path, :string, default: nil
281
+ config_param :ca_private_key_path, :string, default: nil
282
+ config_param :ca_private_key_passphrase, :string, default: nil, secret: true
283
+
284
+ config_param :cert_verifier, :string, default: nil
285
+
286
+ # Options for generating certs by private CA certs or self-signed
287
+ config_param :generate_private_key_length, :integer, default: 2048
288
+ config_param :generate_cert_country, :string, default: 'US'
289
+ config_param :generate_cert_state, :string, default: 'CA'
290
+ config_param :generate_cert_locality, :string, default: 'Mountain View'
291
+ config_param :generate_cert_common_name, :string, default: nil
292
+ config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later
293
+ config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256
294
+ end
295
+ end
296
+
297
+ def self.included(mod)
298
+ mod.include ServerTransportParams
299
+ end
300
+
301
+ def initialize
302
+ super
303
+ @_servers = []
304
+ @_server_connections = []
305
+ @_server_mutex = Mutex.new
306
+ end
307
+
308
+ def configure(conf)
309
+ super
310
+
311
+ if @transport_config
312
+ if @transport_config.protocol == :tls
313
+ cert_option_server_validate!(@transport_config)
314
+ end
315
+ end
316
+ end
317
+
318
+ def stop
319
+ @_server_mutex.synchronize do
320
+ @_servers.each do |si|
321
+ si.server.detach if si.server.attached?
322
+ # to refuse more connections: (connected sockets are still alive here)
323
+ si.server.close rescue nil
324
+ end
325
+ end
326
+
327
+ super
328
+ end
329
+
330
+ def shutdown
331
+ @_server_connections.each do |conn|
332
+ conn.close rescue nil
333
+ end
334
+
335
+ super
336
+ end
337
+
338
+ def terminate
339
+ @_servers = []
340
+ super
341
+ end
342
+
343
+ def server_socket_manager_client
344
+ socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
345
+ if Fluent.windows?
346
+ socket_manager_path = socket_manager_path.to_i
347
+ end
348
+ ServerEngine::SocketManager::Client.new(socket_manager_path)
349
+ end
350
+
351
+ def server_create_tcp_socket(shared, bind, port)
352
+ sock = if shared
353
+ server_socket_manager_client.listen_tcp(bind, port)
354
+ else
355
+ # TCPServer.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.
356
+ # backlog will be set by the caller, we don't need to set backlog here
357
+ tsock = Addrinfo.tcp(bind, port).listen
358
+ tsock.autoclose = false
359
+ TCPServer.for_fd(tsock.fileno)
360
+ end
361
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
362
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
363
+ sock
364
+ end
365
+
366
+ def server_create_udp_socket(shared, bind, port)
367
+ sock = if shared
368
+ server_socket_manager_client.listen_udp(bind, port)
369
+ else
370
+ # UDPSocket.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.
371
+ usock = Addrinfo.udp(bind, port).bind
372
+ usock.autoclose = false
373
+ UDPSocket.for_fd(usock.fileno)
374
+ end
375
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
376
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
377
+ sock
378
+ end
379
+
380
+ # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
381
+ PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
382
+
383
+ class CallbackSocket
384
+ def initialize(server_type, sock, enabled_events = [], close_socket: true)
385
+ @server_type = server_type
386
+ @sock = sock
387
+ @peeraddr = nil
388
+ @enabled_events = enabled_events
389
+ @close_socket = close_socket
390
+ end
391
+
392
+ def remote_addr
393
+ @peeraddr[3]
394
+ end
395
+
396
+ def remote_host
397
+ @peeraddr[2]
398
+ end
399
+
400
+ def remote_port
401
+ @peeraddr[1]
402
+ end
403
+
404
+ def send(data, flags = 0)
405
+ @sock.send(data, flags)
406
+ end
407
+
408
+ def write(data)
409
+ raise "not implemented here"
410
+ end
411
+
412
+ def close_after_write_complete
413
+ @sock.close_after_write_complete = true
414
+ end
415
+
416
+ def close
417
+ @sock.close if @close_socket
418
+ end
419
+
420
+ def data(&callback)
421
+ on(:data, &callback)
422
+ end
423
+
424
+ def on(event, &callback)
425
+ raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
426
+ case event
427
+ when :data
428
+ @sock.data(&callback)
429
+ when :write_complete
430
+ cb = ->(){ callback.call(self) }
431
+ @sock.on_write_complete(&cb)
432
+ when :close
433
+ cb = ->(){ callback.call(self) }
434
+ @sock.on_close(&cb)
435
+ else
436
+ raise "BUG: unknown event: #{event}"
437
+ end
438
+ end
439
+ end
440
+
441
+ class TCPCallbackSocket < CallbackSocket
442
+ ENABLED_EVENTS = [:data, :write_complete, :close]
443
+
444
+ attr_accessor :buffer
445
+
446
+ def initialize(sock)
447
+ super("tcp", sock, ENABLED_EVENTS)
448
+ @peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)
449
+ @buffer = ''
450
+ end
451
+
452
+ def write(data)
453
+ @sock.write(data)
454
+ end
455
+ end
456
+
457
+ class TLSCallbackSocket < CallbackSocket
458
+ ENABLED_EVENTS = [:data, :write_complete, :close]
459
+
460
+ attr_accessor :buffer
461
+
462
+ def initialize(sock)
463
+ super("tls", sock, ENABLED_EVENTS)
464
+ @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
465
+ @buffer = ''
466
+ end
467
+
468
+ def write(data)
469
+ @sock.write(data)
470
+ end
471
+ end
472
+
473
+ class UDPCallbackSocket < CallbackSocket
474
+ ENABLED_EVENTS = []
475
+
476
+ def initialize(sock, peeraddr, **kwargs)
477
+ super("udp", sock, ENABLED_EVENTS, **kwargs)
478
+ @peeraddr = peeraddr
479
+ end
480
+
481
+ def remote_addr
482
+ @peeraddr[3]
483
+ end
484
+
485
+ def remote_host
486
+ @peeraddr[2]
487
+ end
488
+
489
+ def remote_port
490
+ @peeraddr[1]
491
+ end
492
+
493
+ def write(data)
494
+ @sock.send(data, 0, @peeraddr[3], @peeraddr[1])
495
+ end
496
+ end
497
+
498
+ module EventHandler
499
+ class UDPServer < Coolio::IO
500
+ attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
501
+
502
+ def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
503
+ raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
504
+
505
+ super(sock)
506
+
507
+ @sock = sock
508
+ @max_bytes = max_bytes
509
+ @flags = flags
510
+ @close_socket = close_socket
511
+ @log = log
512
+ @under_plugin_development = under_plugin_development
513
+ @callback = callback
514
+
515
+ on_readable_impl = case @callback.arity
516
+ when 1 then :on_readable_without_sock
517
+ when 2 then :on_readable_with_sock
518
+ else
519
+ raise "BUG: callback block must have 1 or 2 arguments"
520
+ end
521
+ self.define_singleton_method(:on_readable, method(on_readable_impl))
522
+ end
523
+
524
+ def on_readable_without_sock
525
+ begin
526
+ data = @sock.recv(@max_bytes, @flags)
527
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
528
+ return
529
+ end
530
+ @callback.call(data)
531
+ rescue => e
532
+ @log.error "unexpected error in processing UDP data", error: e
533
+ @log.error_backtrace
534
+ raise if @under_plugin_development
535
+ end
536
+
537
+ def on_readable_with_sock
538
+ begin
539
+ data, addr = @sock.recvfrom(@max_bytes)
540
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
541
+ return
542
+ end
543
+ @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
544
+ rescue => e
545
+ @log.error "unexpected error in processing UDP data", error: e
546
+ @log.error_backtrace
547
+ raise if @under_plugin_development
548
+ end
549
+ end
550
+
551
+ class TCPServer < Coolio::TCPSocket
552
+ attr_reader :closing
553
+ attr_writer :close_after_write_complete
554
+
555
+ def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
556
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
557
+
558
+ socket_option_setter.call(sock)
559
+
560
+ @_handler_socket = sock
561
+ super(sock)
562
+
563
+ @log = log
564
+ @under_plugin_development = under_plugin_development
565
+
566
+ @connect_callback = connect_callback
567
+ @data_callback = nil
568
+ @close_callback = close_callback
569
+
570
+ @callback_connection = nil
571
+ @close_after_write_complete = false
572
+ @closing = false
573
+
574
+ @mutex = Mutex.new # to serialize #write and #close
575
+ end
576
+
577
+ def to_io
578
+ @_handler_socket
579
+ end
580
+
581
+ def data(&callback)
582
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
583
+ @data_callback = callback
584
+ on_read_impl = case callback.arity
585
+ when 1 then :on_read_without_connection
586
+ when 2 then :on_read_with_connection
587
+ else
588
+ raise "BUG: callback block must have 1 or 2 arguments"
589
+ end
590
+ self.define_singleton_method(:on_read, method(on_read_impl))
591
+ end
592
+
593
+ def write(data)
594
+ @mutex.synchronize do
595
+ super
596
+ end
597
+ end
598
+
599
+ def on_writable
600
+ super
601
+ close if @close_after_write_complete
602
+ end
603
+
604
+ def on_connect
605
+ @callback_connection = TCPCallbackSocket.new(self)
606
+ @connect_callback.call(@callback_connection)
607
+ unless @data_callback
608
+ raise "connection callback must call #data to set data callback"
609
+ end
610
+ end
611
+
612
+ def on_read_without_connection(data)
613
+ @data_callback.call(data)
614
+ rescue => e
615
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
616
+ @log.error_backtrace
617
+ close rescue nil
618
+ raise if @under_plugin_development
619
+ end
620
+
621
+ def on_read_with_connection(data)
622
+ @data_callback.call(data, @callback_connection)
623
+ rescue => e
624
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
625
+ @log.error_backtrace
626
+ close rescue nil
627
+ raise if @under_plugin_development
628
+ end
629
+
630
+ def close
631
+ @mutex.synchronize do
632
+ return if @closing
633
+ @closing = true
634
+ @close_callback.call(self)
635
+ super
636
+ end
637
+ end
638
+ end
639
+
640
+ class TLSServer < Coolio::Socket
641
+ attr_reader :closing
642
+ attr_writer :close_after_write_complete
643
+
644
+ # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
645
+ def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
646
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
647
+
648
+ socket_option_setter.call(sock)
649
+ @_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)
650
+ @_handler_socket.sync_close = true
651
+ @_handler_write_buffer = ''.force_encoding('ascii-8bit')
652
+ @_handler_accepted = false
653
+ super(@_handler_socket)
654
+
655
+ @log = log
656
+ @under_plugin_development = under_plugin_development
657
+
658
+ @connect_callback = connect_callback
659
+ @data_callback = nil
660
+ @close_callback = close_callback
661
+
662
+ @callback_connection = nil
663
+ @close_after_write_complete = false
664
+ @closing = false
665
+
666
+ @mutex = Mutex.new # to serialize #write and #close
667
+ end
668
+
669
+ def to_io
670
+ @_handler_socket.to_io
671
+ end
672
+
673
+ def data(&callback)
674
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
675
+ @data_callback = callback
676
+ on_read_impl = case callback.arity
677
+ when 1 then :on_read_without_connection
678
+ when 2 then :on_read_with_connection
679
+ else
680
+ raise "BUG: callback block must have 1 or 2 arguments"
681
+ end
682
+ self.define_singleton_method(:on_read, method(on_read_impl))
683
+ end
684
+
685
+ def write(data)
686
+ @mutex.synchronize do
687
+ @_handler_write_buffer << data
688
+ schedule_write
689
+ data.bytesize
690
+ end
691
+ end
692
+
693
+ def try_tls_accept
694
+ return true if @_handler_accepted
695
+
696
+ begin
697
+ result = @_handler_socket.accept_nonblock(exception: false) # this method call actually try to do handshake via TLS
698
+ if result == :wait_readable || result == :wait_writable
699
+ # retry accept_nonblock: there aren't enough data in underlying socket buffer
700
+ else
701
+ @_handler_accepted = true
702
+
703
+ @callback_connection = TLSCallbackSocket.new(self)
704
+ @connect_callback.call(@callback_connection)
705
+ unless @data_callback
706
+ raise "connection callback must call #data to set data callback"
707
+ end
708
+
709
+ return true
710
+ end
711
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
712
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
713
+ @log.trace "unexpected error before accepting TLS connection",
714
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
715
+ close rescue nil
716
+ rescue OpenSSL::SSL::SSLError => e
717
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
718
+ # Use same log level as on_readable
719
+ @log.warn "unexpected error before accepting TLS connection by OpenSSL",
720
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
721
+ close rescue nil
722
+ end
723
+
724
+ false
725
+ end
726
+
727
+ def on_connect
728
+ try_tls_accept
729
+ end
730
+
731
+ def on_readable
732
+ if try_tls_accept
733
+ super
734
+ end
735
+ rescue IO::WaitReadable, IO::WaitWritable
736
+ # ignore and return with doing nothing
737
+ rescue OpenSSL::SSL::SSLError => e
738
+ @log.warn "close socket due to unexpected ssl error: #{e}"
739
+ close rescue nil
740
+ end
741
+
742
+ def on_writable
743
+ begin
744
+ @mutex.synchronize do
745
+ # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
746
+ written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
747
+ @_handler_write_buffer.slice!(0, written_bytes)
748
+ end
749
+
750
+ # No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.
751
+ # Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.
752
+ # Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.
753
+ # If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete
754
+ # eventually calls TLSServer#close which try to get a lock.
755
+ super
756
+
757
+ close if @close_after_write_complete
758
+ rescue IO::WaitWritable, IO::WaitReadable
759
+ return
760
+ rescue Errno::EINTR
761
+ return
762
+ rescue SystemCallError, IOError, SocketError
763
+ # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
764
+ close rescue nil
765
+ return
766
+ rescue OpenSSL::SSL::SSLError => e
767
+ @log.debug "unexpected SSLError while writing data into socket connected via TLS", error: e
768
+ end
769
+ end
770
+
771
+ def on_read_without_connection(data)
772
+ @data_callback.call(data)
773
+ rescue => e
774
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
775
+ @log.error_backtrace
776
+ close rescue nil
777
+ raise if @under_plugin_development
778
+ end
779
+
780
+ def on_read_with_connection(data)
781
+ @data_callback.call(data, @callback_connection)
782
+ rescue => e
783
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
784
+ @log.error_backtrace
785
+ close rescue nil
786
+ raise if @under_plugin_development
787
+ end
788
+
789
+ def close
790
+ @mutex.synchronize do
791
+ return if @closing
792
+ @closing = true
793
+ @close_callback.call(self)
794
+ super
795
+ end
796
+ end
797
+ end
798
+ end
799
+ end
800
+ end
801
+ end