fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,473 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+
18
+ require 'fluent/plugin/input'
19
+ require 'fluent/msgpack_factory'
20
+ require 'yajl'
21
+ require 'digest'
22
+ require 'securerandom'
23
+
24
+ module Fluent::Plugin
25
+ class ForwardInput < Input
26
+ Fluent::Plugin.register_input('forward', self)
27
+
28
+ # See the wiki page below for protocol specification
29
+ # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
30
+
31
+ helpers :server
32
+
33
+ LISTEN_PORT = 24224
34
+
35
+ desc 'The port to listen to.'
36
+ config_param :port, :integer, default: LISTEN_PORT
37
+ desc 'The bind address to listen to.'
38
+ config_param :bind, :string, default: '0.0.0.0'
39
+
40
+ config_param :backlog, :integer, default: nil
41
+ # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
42
+ desc 'The timeout time used to set linger option.'
43
+ config_param :linger_timeout, :integer, default: 0
44
+ # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
45
+ config_param :blocking_timeout, :time, default: 0.5
46
+ desc 'Try to resolve hostname from IP addresses or not.'
47
+ config_param :resolve_hostname, :bool, default: nil
48
+ desc 'Connections will be disconnected right after receiving first message if this value is true.'
49
+ config_param :deny_keepalive, :bool, default: false
50
+ desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
51
+ config_param :send_keepalive_packet, :bool, default: false
52
+
53
+ desc 'Log warning if received chunk size is larger than this value.'
54
+ config_param :chunk_size_warn_limit, :size, default: nil
55
+ desc 'Received chunk is dropped if it is larger than this value.'
56
+ config_param :chunk_size_limit, :size, default: nil
57
+ desc 'Skip an event if incoming event is invalid.'
58
+ config_param :skip_invalid_event, :bool, default: false
59
+
60
+ desc "The field name of the client's source address."
61
+ config_param :source_address_key, :string, default: nil
62
+ desc "The field name of the client's hostname."
63
+ config_param :source_hostname_key, :string, default: nil
64
+
65
+ desc "New tag instead of incoming tag"
66
+ config_param :tag, :string, default: nil
67
+ desc "Add prefix to incoming tag"
68
+ config_param :add_tag_prefix, :string, default: nil
69
+
70
+ config_section :security, required: false, multi: false do
71
+ desc 'The hostname'
72
+ config_param :self_hostname, :string
73
+ desc 'Shared key for authentication'
74
+ config_param :shared_key, :string, secret: true
75
+ desc 'If true, use user based authentication'
76
+ config_param :user_auth, :bool, default: false
77
+ desc 'Allow anonymous source. <client> sections required if disabled.'
78
+ config_param :allow_anonymous_source, :bool, default: true
79
+
80
+ ### User based authentication
81
+ config_section :user, param_name: :users, required: false, multi: true do
82
+ desc 'The username for authentication'
83
+ config_param :username, :string
84
+ desc 'The password for authentication'
85
+ config_param :password, :string, secret: true
86
+ end
87
+
88
+ ### Client ip/network authentication & per_host shared key
89
+ config_section :client, param_name: :clients, required: false, multi: true do
90
+ desc 'The IP address or host name of the client'
91
+ config_param :host, :string, default: nil
92
+ desc 'Network address specification'
93
+ config_param :network, :string, default: nil
94
+ desc 'Shared key per client'
95
+ config_param :shared_key, :string, default: nil, secret: true
96
+ desc 'Array of username.'
97
+ config_param :users, :array, default: []
98
+ end
99
+ end
100
+
101
+ def configure(conf)
102
+ super
103
+
104
+ if @source_hostname_key
105
+ # TODO: add test
106
+ if @resolve_hostname.nil?
107
+ @resolve_hostname = true
108
+ elsif !@resolve_hostname # user specifies "false" in config
109
+ raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
110
+ end
111
+ end
112
+ @enable_field_injection = @source_address_key || @source_hostname_key
113
+
114
+ raise Fluent::ConfigError, "'tag' parameter must not be empty" if @tag && @tag.empty?
115
+ raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
116
+
117
+ if @security
118
+ if @security.user_auth && @security.users.empty?
119
+ raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
120
+ end
121
+ if !@security.allow_anonymous_source && @security.clients.empty?
122
+ raise Fluent::ConfigError, "<client> sections required if allow_anonymous_source disabled"
123
+ end
124
+
125
+ @nodes = []
126
+
127
+ @security.clients.each do |client|
128
+ if client.host && client.network
129
+ raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
130
+ end
131
+ if !client.host && !client.network
132
+ raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
133
+ end
134
+ source = nil
135
+ if client.host
136
+ begin
137
+ source = IPSocket.getaddress(client.host)
138
+ rescue SocketError
139
+ raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
140
+ end
141
+ end
142
+ source_addr = begin
143
+ IPAddr.new(source || client.network)
144
+ rescue ArgumentError
145
+ raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
146
+ end
147
+ @nodes.push({
148
+ address: source_addr,
149
+ shared_key: (client.shared_key || @security.shared_key),
150
+ users: client.users
151
+ })
152
+ end
153
+ end
154
+
155
+ if @send_keepalive_packet && @deny_keepalive
156
+ raise Fluent::ConfigError, "both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true"
157
+ end
158
+ end
159
+
160
+ def multi_workers_ready?
161
+ true
162
+ end
163
+
164
+ HEARTBEAT_UDP_PAYLOAD = "\0"
165
+
166
+ def start
167
+ super
168
+
169
+ shared_socket = system_config.workers > 1
170
+
171
+ log.info "listening port", port: @port, bind: @bind
172
+ server_create_connection(
173
+ :in_forward_server, @port,
174
+ bind: @bind,
175
+ shared: shared_socket,
176
+ resolve_name: @resolve_hostname,
177
+ linger_timeout: @linger_timeout,
178
+ send_keepalive_packet: @send_keepalive_packet,
179
+ backlog: @backlog,
180
+ &method(:handle_connection)
181
+ )
182
+
183
+ server_create(:in_forward_server_udp_heartbeat, @port, shared: shared_socket, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
184
+ log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
185
+ begin
186
+ sock.write HEARTBEAT_UDP_PAYLOAD
187
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
188
+ log.trace "error while heartbeat response", host: sock.remote_host, error: e
189
+ end
190
+ end
191
+ end
192
+
193
+ def handle_connection(conn)
194
+ send_data = ->(serializer, data){ conn.write serializer.call(data) }
195
+
196
+ log.trace "connected fluent socket", addr: conn.remote_addr, port: conn.remote_port
197
+ state = :established
198
+ nonce = nil
199
+ user_auth_salt = nil
200
+
201
+ if @security
202
+ # security enabled session MUST use MessagePack as serialization format
203
+ state = :helo
204
+ nonce = generate_salt
205
+ user_auth_salt = generate_salt
206
+ send_data.call(:to_msgpack.to_proc, generate_helo(nonce, user_auth_salt))
207
+ state = :pingpong
208
+ end
209
+
210
+ log.trace "accepted fluent socket", addr: conn.remote_addr, port: conn.remote_port
211
+
212
+ read_messages(conn) do |msg, chunk_size, serializer|
213
+ case state
214
+ when :pingpong
215
+ success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
216
+ unless success
217
+ conn.on(:write_complete) { |c| c.close_after_write_complete }
218
+ send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
219
+ next
220
+ end
221
+ send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))
222
+
223
+ log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
224
+ state = :established
225
+ when :established
226
+ options = on_message(msg, chunk_size, conn)
227
+ if options && r = response(options)
228
+ log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
229
+ conn.on(:write_complete) { |c| c.close } if @deny_keepalive
230
+ send_data.call(serializer, r)
231
+ else
232
+ if @deny_keepalive
233
+ conn.close
234
+ end
235
+ end
236
+ else
237
+ raise "BUG: unknown session state: #{state}"
238
+ end
239
+ end
240
+ end
241
+
242
+ def read_messages(conn, &block)
243
+ feeder = nil
244
+ serializer = nil
245
+ bytes = 0
246
+ conn.data do |data|
247
+ # only for first call of callback
248
+ unless feeder
249
+ first = data[0]
250
+ if first == '{' || first == '[' # json
251
+ parser = Yajl::Parser.new
252
+ parser.on_parse_complete = ->(obj){
253
+ block.call(obj, bytes, serializer)
254
+ bytes = 0
255
+ }
256
+ serializer = :to_json.to_proc
257
+ feeder = ->(d){ parser << d }
258
+ else # msgpack
259
+ parser = Fluent::MessagePackFactory.msgpack_unpacker
260
+ serializer = :to_msgpack.to_proc
261
+ feeder = ->(d){
262
+ parser.feed_each(d){|obj|
263
+ block.call(obj, bytes, serializer)
264
+ bytes = 0
265
+ }
266
+ }
267
+ end
268
+ end
269
+
270
+ bytes += data.bytesize
271
+ feeder.call(data)
272
+ end
273
+ end
274
+
275
+ def response(option)
276
+ if option && option['chunk']
277
+ return { 'ack' => option['chunk'] }
278
+ end
279
+ nil
280
+ end
281
+
282
+ def on_message(msg, chunk_size, conn)
283
+ if msg.nil?
284
+ # for future TCP heartbeat_request
285
+ return
286
+ end
287
+
288
+ # TODO: raise an exception if broken chunk is generated by recoverable situation
289
+ unless msg.is_a?(Array)
290
+ log.warn "incoming chunk is broken:", host: conn.remote_host, msg: msg
291
+ return
292
+ end
293
+
294
+ tag = msg[0]
295
+ entries = msg[1]
296
+
297
+ if @chunk_size_limit && (chunk_size > @chunk_size_limit)
298
+ log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size
299
+ return
300
+ elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
301
+ log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
302
+ end
303
+
304
+ tag = @tag.dup if @tag
305
+ tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
306
+
307
+ case entries
308
+ when String
309
+ # PackedForward
310
+ option = msg[2]
311
+ size = (option && option['size']) || 0
312
+ es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
313
+ es = es_class.new(entries, nil, size.to_i)
314
+ es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
315
+ if @enable_field_injection
316
+ es = add_source_info(es, conn)
317
+ end
318
+ router.emit_stream(tag, es)
319
+
320
+ when Array
321
+ # Forward
322
+ es = if @skip_invalid_event
323
+ check_and_skip_invalid_event(tag, entries, conn.remote_host)
324
+ else
325
+ es = Fluent::MultiEventStream.new
326
+ entries.each { |e|
327
+ record = e[1]
328
+ next if record.nil?
329
+ time = e[0]
330
+ time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
331
+ es.add(time, record)
332
+ }
333
+ es
334
+ end
335
+ if @enable_field_injection
336
+ es = add_source_info(es, conn)
337
+ end
338
+ router.emit_stream(tag, es)
339
+ option = msg[2]
340
+
341
+ else
342
+ # Message
343
+ time = msg[1]
344
+ record = msg[2]
345
+ if @skip_invalid_event && invalid_event?(tag, time, record)
346
+ log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
347
+ return msg[3] # retry never succeeded so return ack and drop incoming event.
348
+ end
349
+ return if record.nil?
350
+ time = Fluent::EventTime.now if time.to_i == 0
351
+ if @enable_field_injection
352
+ record[@source_address_key] = conn.remote_addr if @source_address_key
353
+ record[@source_hostname_key] = conn.remote_host if @source_hostname_key
354
+ end
355
+ router.emit(tag, time, record)
356
+ option = msg[3]
357
+ end
358
+
359
+ # return option for response
360
+ option
361
+ end
362
+
363
+ def invalid_event?(tag, time, record)
364
+ !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
365
+ end
366
+
367
+ def check_and_skip_invalid_event(tag, es, remote_host)
368
+ new_es = Fluent::MultiEventStream.new
369
+ es.each { |time, record|
370
+ if invalid_event?(tag, time, record)
371
+ log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
372
+ next
373
+ end
374
+ new_es.add(time, record)
375
+ }
376
+ new_es
377
+ end
378
+
379
+ def add_source_info(es, conn)
380
+ new_es = Fluent::MultiEventStream.new
381
+ if @source_address_key && @source_hostname_key
382
+ address = conn.remote_addr
383
+ hostname = conn.remote_host
384
+ es.each { |time, record|
385
+ record[@source_address_key] = address
386
+ record[@source_hostname_key] = hostname
387
+ new_es.add(time, record)
388
+ }
389
+ elsif @source_address_key
390
+ address = conn.remote_addr
391
+ es.each { |time, record|
392
+ record[@source_address_key] = address
393
+ new_es.add(time, record)
394
+ }
395
+ elsif @source_hostname_key
396
+ hostname = conn.remote_host
397
+ es.each { |time, record|
398
+ record[@source_hostname_key] = hostname
399
+ new_es.add(time, record)
400
+ }
401
+ else
402
+ raise "BUG: don't call this method in this case"
403
+ end
404
+ new_es
405
+ end
406
+
407
+ def select_authenticate_users(node, username)
408
+ if node.nil? || node[:users].empty?
409
+ @security.users.select{|u| u.username == username}
410
+ else
411
+ @security.users.select{|u| node[:users].include?(u.username) && u.username == username}
412
+ end
413
+ end
414
+
415
+ def generate_salt
416
+ ::SecureRandom.random_bytes(16)
417
+ end
418
+
419
+ def generate_helo(nonce, user_auth_salt)
420
+ log.debug "generating helo"
421
+ # ['HELO', options(hash)]
422
+ ['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
423
+ end
424
+
425
+ def check_ping(message, remote_addr, user_auth_salt, nonce)
426
+ log.debug "checking ping"
427
+ # ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']
428
+ unless message.size == 6 && message[0] == 'PING'
429
+ return false, 'invalid ping message'
430
+ end
431
+ _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
432
+
433
+ node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
434
+ if !node && !@security.allow_anonymous_source
435
+ log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
436
+ return false, "anonymous source host '#{remote_addr}' denied", nil
437
+ end
438
+
439
+ shared_key = node ? node[:shared_key] : @security.shared_key
440
+ serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(nonce).update(shared_key).hexdigest
441
+ if shared_key_hexdigest != serverside
442
+ log.warn "Shared key mismatch", address: remote_addr, hostname: hostname
443
+ return false, 'shared_key mismatch', nil
444
+ end
445
+
446
+ if @security.user_auth
447
+ users = select_authenticate_users(node, username)
448
+ success = false
449
+ users.each do |user|
450
+ passhash = Digest::SHA512.new.update(user_auth_salt).update(username).update(user[:password]).hexdigest
451
+ success ||= (passhash == password_digest)
452
+ end
453
+ unless success
454
+ log.warn "Authentication failed", address: remote_addr, hostname: hostname, username: username
455
+ return false, 'username/password mismatch', nil
456
+ end
457
+ end
458
+
459
+ return true, shared_key_salt, shared_key
460
+ end
461
+
462
+ def generate_pong(auth_result, reason_or_salt, nonce, shared_key)
463
+ log.debug "generating pong"
464
+ # ['PONG', bool(authentication result), 'reason if authentication failed', self_hostname, sha512_hex(salt + self_hostname + nonce + sharedkey)]
465
+ unless auth_result
466
+ return ['PONG', false, reason_or_salt, '', '']
467
+ end
468
+
469
+ shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
470
+ ['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
471
+ end
472
+ end
473
+ end
@@ -0,0 +1,72 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/input'
18
+
19
+ module Fluent::Plugin
20
+ class GCStatInput < Fluent::Plugin::Input
21
+ Fluent::Plugin.register_input('gc_stat', self)
22
+
23
+ helpers :timer
24
+
25
+ def initialize
26
+ super
27
+ @key_map = nil
28
+ end
29
+
30
+ config_param :emit_interval, :time, default: 60
31
+ config_param :use_symbol_keys, :bool, default: true
32
+ config_param :tag, :string
33
+
34
+ def configure(conf)
35
+ super
36
+
37
+ unless @use_symbol_keys
38
+ @key_map = {}
39
+ GC.stat.each_key { |key|
40
+ @key_map[key] = key.to_s
41
+ }
42
+ end
43
+ end
44
+
45
+ def multi_workers_ready?
46
+ true
47
+ end
48
+
49
+ def start
50
+ super
51
+
52
+ timer_execute(:in_gc_stat, @emit_interval, &method(:on_timer))
53
+ end
54
+
55
+ def shutdown
56
+ super
57
+ end
58
+
59
+ def on_timer
60
+ now = Fluent::EventTime.now
61
+ record = GC.stat
62
+ unless @use_symbol_keys
63
+ new_record = {}
64
+ record.each_pair { |k, v|
65
+ new_record[@key_map[k]] = v
66
+ }
67
+ record = new_record
68
+ end
69
+ router.emit(@tag, now, record)
70
+ end
71
+ end
72
+ end