fluentd 0.12.40 → 1.6.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (428) hide show
  1. checksums.yaml +5 -5
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  4. data/.github/ISSUE_TEMPLATE.md +17 -0
  5. data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
  6. data/.gitignore +5 -0
  7. data/.gitlab/cicd-template.yaml +10 -0
  8. data/.gitlab-ci.yml +147 -0
  9. data/.travis.yml +56 -20
  10. data/ADOPTERS.md +5 -0
  11. data/CHANGELOG.md +1369 -0
  12. data/CONTRIBUTING.md +16 -5
  13. data/GOVERNANCE.md +55 -0
  14. data/Gemfile +5 -0
  15. data/GithubWorkflow.md +78 -0
  16. data/LICENSE +202 -0
  17. data/MAINTAINERS.md +7 -0
  18. data/README.md +23 -11
  19. data/Rakefile +48 -2
  20. data/Vagrantfile +17 -0
  21. data/appveyor.yml +37 -0
  22. data/bin/fluent-binlog-reader +7 -0
  23. data/bin/fluent-ca-generate +6 -0
  24. data/bin/fluent-plugin-config-format +5 -0
  25. data/bin/fluent-plugin-generate +5 -0
  26. data/bin/fluentd +3 -0
  27. data/code-of-conduct.md +3 -0
  28. data/example/copy_roundrobin.conf +39 -0
  29. data/example/counter.conf +18 -0
  30. data/example/in_dummy_blocks.conf +17 -0
  31. data/example/in_dummy_with_compression.conf +23 -0
  32. data/example/in_forward.conf +7 -0
  33. data/example/in_forward_client.conf +37 -0
  34. data/example/in_forward_shared_key.conf +15 -0
  35. data/example/in_forward_tls.conf +14 -0
  36. data/example/in_forward_users.conf +24 -0
  37. data/example/in_forward_workers.conf +21 -0
  38. data/example/in_http.conf +3 -1
  39. data/example/in_out_forward.conf +17 -0
  40. data/example/logevents.conf +25 -0
  41. data/example/multi_filters.conf +61 -0
  42. data/example/out_exec_filter.conf +42 -0
  43. data/example/out_forward.conf +13 -13
  44. data/example/out_forward_buf_file.conf +23 -0
  45. data/example/out_forward_client.conf +109 -0
  46. data/example/out_forward_heartbeat_none.conf +16 -0
  47. data/example/out_forward_shared_key.conf +36 -0
  48. data/example/out_forward_tls.conf +18 -0
  49. data/example/out_forward_users.conf +65 -0
  50. data/example/out_null.conf +36 -0
  51. data/example/secondary_file.conf +42 -0
  52. data/example/suppress_config_dump.conf +7 -0
  53. data/example/worker_section.conf +36 -0
  54. data/fluent.conf +29 -0
  55. data/fluentd.gemspec +21 -11
  56. data/lib/fluent/agent.rb +67 -90
  57. data/lib/fluent/clock.rb +62 -0
  58. data/lib/fluent/command/binlog_reader.rb +244 -0
  59. data/lib/fluent/command/ca_generate.rb +181 -0
  60. data/lib/fluent/command/cat.rb +42 -18
  61. data/lib/fluent/command/debug.rb +12 -10
  62. data/lib/fluent/command/fluentd.rb +153 -5
  63. data/lib/fluent/command/plugin_config_formatter.rb +292 -0
  64. data/lib/fluent/command/plugin_generator.rb +324 -0
  65. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  66. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  67. data/lib/fluent/compat/exec_util.rb +129 -0
  68. data/lib/fluent/compat/file_util.rb +54 -0
  69. data/lib/fluent/compat/filter.rb +68 -0
  70. data/lib/fluent/compat/formatter.rb +111 -0
  71. data/lib/fluent/compat/formatter_utils.rb +85 -0
  72. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  73. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  74. data/lib/fluent/compat/input.rb +49 -0
  75. data/lib/fluent/compat/output.rb +718 -0
  76. data/lib/fluent/compat/output_chain.rb +60 -0
  77. data/lib/fluent/compat/parser.rb +310 -0
  78. data/lib/fluent/compat/parser_utils.rb +40 -0
  79. data/lib/fluent/compat/propagate_default.rb +62 -0
  80. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  81. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  82. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  83. data/lib/fluent/compat/socket_util.rb +165 -0
  84. data/lib/fluent/compat/string_util.rb +34 -0
  85. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  86. data/lib/fluent/compat/type_converter.rb +90 -0
  87. data/lib/fluent/config/configure_proxy.rb +210 -62
  88. data/lib/fluent/config/dsl.rb +12 -5
  89. data/lib/fluent/config/element.rb +107 -9
  90. data/lib/fluent/config/literal_parser.rb +9 -3
  91. data/lib/fluent/config/parser.rb +4 -4
  92. data/lib/fluent/config/section.rb +51 -14
  93. data/lib/fluent/config/types.rb +28 -13
  94. data/lib/fluent/config/v1_parser.rb +3 -5
  95. data/lib/fluent/config.rb +23 -20
  96. data/lib/fluent/configurable.rb +79 -21
  97. data/lib/fluent/counter/base_socket.rb +46 -0
  98. data/lib/fluent/counter/client.rb +297 -0
  99. data/lib/fluent/counter/error.rb +86 -0
  100. data/lib/fluent/counter/mutex_hash.rb +163 -0
  101. data/lib/fluent/counter/server.rb +273 -0
  102. data/lib/fluent/counter/store.rb +205 -0
  103. data/lib/fluent/counter/validator.rb +145 -0
  104. data/lib/fluent/counter.rb +23 -0
  105. data/lib/fluent/daemon.rb +15 -0
  106. data/lib/fluent/engine.rb +102 -65
  107. data/lib/fluent/env.rb +7 -3
  108. data/lib/fluent/error.rb +30 -0
  109. data/lib/fluent/event.rb +197 -21
  110. data/lib/fluent/event_router.rb +93 -10
  111. data/lib/fluent/filter.rb +2 -50
  112. data/lib/fluent/formatter.rb +4 -293
  113. data/lib/fluent/input.rb +2 -32
  114. data/lib/fluent/label.rb +10 -2
  115. data/lib/fluent/load.rb +3 -3
  116. data/lib/fluent/log.rb +348 -81
  117. data/lib/fluent/match.rb +37 -36
  118. data/lib/fluent/mixin.rb +12 -176
  119. data/lib/fluent/msgpack_factory.rb +62 -0
  120. data/lib/fluent/output.rb +10 -612
  121. data/lib/fluent/output_chain.rb +23 -0
  122. data/lib/fluent/parser.rb +4 -800
  123. data/lib/fluent/plugin/bare_output.rb +63 -0
  124. data/lib/fluent/plugin/base.rb +192 -0
  125. data/lib/fluent/plugin/buf_file.rb +128 -174
  126. data/lib/fluent/plugin/buf_memory.rb +9 -92
  127. data/lib/fluent/plugin/buffer/chunk.rb +221 -0
  128. data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
  129. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  130. data/lib/fluent/plugin/buffer.rb +779 -0
  131. data/lib/fluent/plugin/compressable.rb +92 -0
  132. data/lib/fluent/plugin/exec_util.rb +3 -108
  133. data/lib/fluent/plugin/file_util.rb +4 -34
  134. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  135. data/lib/fluent/plugin/filter.rb +93 -0
  136. data/lib/fluent/plugin/filter_grep.rb +117 -34
  137. data/lib/fluent/plugin/filter_parser.rb +85 -62
  138. data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
  139. data/lib/fluent/plugin/filter_stdout.rb +15 -12
  140. data/lib/fluent/plugin/formatter.rb +50 -0
  141. data/lib/fluent/plugin/formatter_csv.rb +52 -0
  142. data/lib/fluent/plugin/formatter_hash.rb +33 -0
  143. data/lib/fluent/plugin/formatter_json.rb +55 -0
  144. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  145. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  146. data/lib/fluent/plugin/formatter_out_file.rb +51 -0
  147. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  148. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  149. data/lib/fluent/plugin/formatter_tsv.rb +38 -0
  150. data/lib/fluent/plugin/in_debug_agent.rb +17 -6
  151. data/lib/fluent/plugin/in_dummy.rb +47 -20
  152. data/lib/fluent/plugin/in_exec.rb +55 -123
  153. data/lib/fluent/plugin/in_forward.rb +299 -216
  154. data/lib/fluent/plugin/in_gc_stat.rb +14 -36
  155. data/lib/fluent/plugin/in_http.rb +204 -91
  156. data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
  157. data/lib/fluent/plugin/in_object_space.rb +13 -41
  158. data/lib/fluent/plugin/in_syslog.rb +112 -134
  159. data/lib/fluent/plugin/in_tail.rb +408 -745
  160. data/lib/fluent/plugin/in_tcp.rb +66 -9
  161. data/lib/fluent/plugin/in_udp.rb +60 -11
  162. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
  163. data/lib/fluent/plugin/input.rb +37 -0
  164. data/lib/fluent/plugin/multi_output.rb +158 -0
  165. data/lib/fluent/plugin/out_copy.rb +23 -35
  166. data/lib/fluent/plugin/out_exec.rb +67 -70
  167. data/lib/fluent/plugin/out_exec_filter.rb +204 -271
  168. data/lib/fluent/plugin/out_file.rb +267 -73
  169. data/lib/fluent/plugin/out_forward.rb +854 -325
  170. data/lib/fluent/plugin/out_null.rb +42 -9
  171. data/lib/fluent/plugin/out_relabel.rb +9 -5
  172. data/lib/fluent/plugin/out_roundrobin.rb +18 -37
  173. data/lib/fluent/plugin/out_secondary_file.rb +133 -0
  174. data/lib/fluent/plugin/out_stdout.rb +43 -10
  175. data/lib/fluent/plugin/out_stream.rb +7 -2
  176. data/lib/fluent/plugin/output.rb +1498 -0
  177. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  178. data/lib/fluent/plugin/parser.rb +191 -0
  179. data/lib/fluent/plugin/parser_apache.rb +28 -0
  180. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  181. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  182. data/lib/fluent/plugin/parser_csv.rb +39 -0
  183. data/lib/fluent/plugin/parser_json.rb +94 -0
  184. data/lib/fluent/plugin/parser_ltsv.rb +49 -0
  185. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  186. data/lib/fluent/plugin/parser_multiline.rb +106 -0
  187. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  188. data/lib/fluent/plugin/parser_none.rb +36 -0
  189. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  190. data/lib/fluent/plugin/parser_syslog.rb +142 -0
  191. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  192. data/lib/fluent/plugin/socket_util.rb +3 -143
  193. data/lib/fluent/plugin/storage.rb +84 -0
  194. data/lib/fluent/plugin/storage_local.rb +164 -0
  195. data/lib/fluent/plugin/string_util.rb +3 -15
  196. data/lib/fluent/plugin.rb +122 -121
  197. data/lib/fluent/plugin_helper/cert_option.rb +178 -0
  198. data/lib/fluent/plugin_helper/child_process.rb +364 -0
  199. data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
  200. data/lib/fluent/plugin_helper/counter.rb +51 -0
  201. data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
  202. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  203. data/lib/fluent/plugin_helper/extract.rb +104 -0
  204. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  205. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  206. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  207. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  208. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  209. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  210. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  211. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  212. data/lib/fluent/plugin_helper/http_server.rb +76 -0
  213. data/lib/fluent/plugin_helper/inject.rb +151 -0
  214. data/lib/fluent/plugin_helper/parser.rb +147 -0
  215. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  216. data/lib/fluent/plugin_helper/retry_state.rb +205 -0
  217. data/lib/fluent/plugin_helper/server.rb +807 -0
  218. data/lib/fluent/plugin_helper/socket.rb +250 -0
  219. data/lib/fluent/plugin_helper/socket_option.rb +80 -0
  220. data/lib/fluent/plugin_helper/storage.rb +349 -0
  221. data/lib/fluent/plugin_helper/thread.rb +179 -0
  222. data/lib/fluent/plugin_helper/timer.rb +92 -0
  223. data/lib/fluent/plugin_helper.rb +73 -0
  224. data/lib/fluent/plugin_id.rb +80 -0
  225. data/lib/fluent/process.rb +3 -489
  226. data/lib/fluent/registry.rb +52 -10
  227. data/lib/fluent/root_agent.rb +204 -42
  228. data/lib/fluent/supervisor.rb +597 -359
  229. data/lib/fluent/system_config.rb +131 -42
  230. data/lib/fluent/test/base.rb +6 -54
  231. data/lib/fluent/test/driver/base.rb +224 -0
  232. data/lib/fluent/test/driver/base_owned.rb +70 -0
  233. data/lib/fluent/test/driver/base_owner.rb +135 -0
  234. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  235. data/lib/fluent/test/driver/filter.rb +57 -0
  236. data/lib/fluent/test/driver/formatter.rb +30 -0
  237. data/lib/fluent/test/driver/input.rb +31 -0
  238. data/lib/fluent/test/driver/multi_output.rb +53 -0
  239. data/lib/fluent/test/driver/output.rb +102 -0
  240. data/lib/fluent/test/driver/parser.rb +30 -0
  241. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  242. data/lib/fluent/test/filter_test.rb +0 -1
  243. data/lib/fluent/test/formatter_test.rb +4 -1
  244. data/lib/fluent/test/helpers.rb +58 -10
  245. data/lib/fluent/test/input_test.rb +27 -19
  246. data/lib/fluent/test/log.rb +79 -0
  247. data/lib/fluent/test/output_test.rb +28 -39
  248. data/lib/fluent/test/parser_test.rb +3 -1
  249. data/lib/fluent/test/startup_shutdown.rb +46 -0
  250. data/lib/fluent/test.rb +33 -1
  251. data/lib/fluent/time.rb +450 -1
  252. data/lib/fluent/timezone.rb +27 -3
  253. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  254. data/lib/fluent/version.rb +1 -1
  255. data/lib/fluent/winsvc.rb +85 -0
  256. data/templates/new_gem/Gemfile +3 -0
  257. data/templates/new_gem/README.md.erb +43 -0
  258. data/templates/new_gem/Rakefile +13 -0
  259. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  260. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  261. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  262. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  263. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  264. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  265. data/templates/new_gem/test/helper.rb.erb +8 -0
  266. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  267. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  268. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  269. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  270. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  271. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  272. data/templates/plugin_config_formatter/param.md.erb +34 -0
  273. data/templates/plugin_config_formatter/section.md.erb +12 -0
  274. data/test/command/test_binlog_reader.rb +346 -0
  275. data/test/command/test_ca_generate.rb +70 -0
  276. data/test/command/test_fluentd.rb +901 -0
  277. data/test/command/test_plugin_config_formatter.rb +276 -0
  278. data/test/command/test_plugin_generator.rb +92 -0
  279. data/test/compat/test_calls_super.rb +166 -0
  280. data/test/compat/test_parser.rb +92 -0
  281. data/test/config/test_config_parser.rb +126 -2
  282. data/test/config/test_configurable.rb +946 -187
  283. data/test/config/test_configure_proxy.rb +424 -74
  284. data/test/config/test_dsl.rb +11 -11
  285. data/test/config/test_element.rb +500 -0
  286. data/test/config/test_literal_parser.rb +8 -0
  287. data/test/config/test_plugin_configuration.rb +56 -0
  288. data/test/config/test_section.rb +79 -7
  289. data/test/config/test_system_config.rb +122 -35
  290. data/test/config/test_types.rb +38 -0
  291. data/test/counter/test_client.rb +559 -0
  292. data/test/counter/test_error.rb +44 -0
  293. data/test/counter/test_mutex_hash.rb +179 -0
  294. data/test/counter/test_server.rb +589 -0
  295. data/test/counter/test_store.rb +258 -0
  296. data/test/counter/test_validator.rb +137 -0
  297. data/test/helper.rb +89 -6
  298. data/test/helpers/fuzzy_assert.rb +89 -0
  299. data/test/plugin/test_bare_output.rb +118 -0
  300. data/test/plugin/test_base.rb +115 -0
  301. data/test/plugin/test_buf_file.rb +823 -460
  302. data/test/plugin/test_buf_memory.rb +32 -194
  303. data/test/plugin/test_buffer.rb +1233 -0
  304. data/test/plugin/test_buffer_chunk.rb +198 -0
  305. data/test/plugin/test_buffer_file_chunk.rb +844 -0
  306. data/test/plugin/test_buffer_memory_chunk.rb +338 -0
  307. data/test/plugin/test_compressable.rb +84 -0
  308. data/test/plugin/test_filter.rb +357 -0
  309. data/test/plugin/test_filter_grep.rb +540 -29
  310. data/test/plugin/test_filter_parser.rb +439 -452
  311. data/test/plugin/test_filter_record_transformer.rb +123 -166
  312. data/test/plugin/test_filter_stdout.rb +160 -72
  313. data/test/plugin/test_formatter_csv.rb +111 -0
  314. data/test/plugin/test_formatter_hash.rb +35 -0
  315. data/test/plugin/test_formatter_json.rb +51 -0
  316. data/test/plugin/test_formatter_ltsv.rb +62 -0
  317. data/test/plugin/test_formatter_msgpack.rb +28 -0
  318. data/test/plugin/test_formatter_out_file.rb +95 -0
  319. data/test/plugin/test_formatter_single_value.rb +38 -0
  320. data/test/plugin/test_formatter_tsv.rb +68 -0
  321. data/test/plugin/test_in_debug_agent.rb +24 -1
  322. data/test/plugin/test_in_dummy.rb +111 -18
  323. data/test/plugin/test_in_exec.rb +200 -113
  324. data/test/plugin/test_in_forward.rb +990 -387
  325. data/test/plugin/test_in_gc_stat.rb +10 -8
  326. data/test/plugin/test_in_http.rb +600 -224
  327. data/test/plugin/test_in_monitor_agent.rb +690 -0
  328. data/test/plugin/test_in_object_space.rb +24 -8
  329. data/test/plugin/test_in_syslog.rb +154 -215
  330. data/test/plugin/test_in_tail.rb +1006 -707
  331. data/test/plugin/test_in_tcp.rb +125 -48
  332. data/test/plugin/test_in_udp.rb +204 -63
  333. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
  334. data/test/plugin/test_input.rb +126 -0
  335. data/test/plugin/test_metadata.rb +89 -0
  336. data/test/plugin/test_multi_output.rb +180 -0
  337. data/test/plugin/test_out_copy.rb +117 -112
  338. data/test/plugin/test_out_exec.rb +258 -53
  339. data/test/plugin/test_out_exec_filter.rb +538 -115
  340. data/test/plugin/test_out_file.rb +865 -178
  341. data/test/plugin/test_out_forward.rb +998 -210
  342. data/test/plugin/test_out_null.rb +105 -0
  343. data/test/plugin/test_out_relabel.rb +28 -0
  344. data/test/plugin/test_out_roundrobin.rb +36 -29
  345. data/test/plugin/test_out_secondary_file.rb +458 -0
  346. data/test/plugin/test_out_stdout.rb +135 -37
  347. data/test/plugin/test_out_stream.rb +18 -0
  348. data/test/plugin/test_output.rb +984 -0
  349. data/test/plugin/test_output_as_buffered.rb +2021 -0
  350. data/test/plugin/test_output_as_buffered_backup.rb +312 -0
  351. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  352. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  353. data/test/plugin/test_output_as_buffered_retries.rb +911 -0
  354. data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
  355. data/test/plugin/test_output_as_standard.rb +374 -0
  356. data/test/plugin/test_owned_by.rb +35 -0
  357. data/test/plugin/test_parser.rb +359 -0
  358. data/test/plugin/test_parser_apache.rb +42 -0
  359. data/test/plugin/test_parser_apache2.rb +47 -0
  360. data/test/plugin/test_parser_apache_error.rb +45 -0
  361. data/test/plugin/test_parser_csv.rb +103 -0
  362. data/test/plugin/test_parser_json.rb +138 -0
  363. data/test/plugin/test_parser_labeled_tsv.rb +145 -0
  364. data/test/plugin/test_parser_multiline.rb +100 -0
  365. data/test/plugin/test_parser_nginx.rb +88 -0
  366. data/test/plugin/test_parser_none.rb +52 -0
  367. data/test/plugin/test_parser_regexp.rb +289 -0
  368. data/test/plugin/test_parser_syslog.rb +441 -0
  369. data/test/plugin/test_parser_tsv.rb +122 -0
  370. data/test/plugin/test_storage.rb +167 -0
  371. data/test/plugin/test_storage_local.rb +335 -0
  372. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  373. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  374. data/test/plugin_helper/data/cert/cert.pem +19 -0
  375. data/test/plugin_helper/http_server/test_app.rb +65 -0
  376. data/test/plugin_helper/http_server/test_route.rb +32 -0
  377. data/test/plugin_helper/test_cert_option.rb +16 -0
  378. data/test/plugin_helper/test_child_process.rb +794 -0
  379. data/test/plugin_helper/test_compat_parameters.rb +353 -0
  380. data/test/plugin_helper/test_event_emitter.rb +51 -0
  381. data/test/plugin_helper/test_event_loop.rb +52 -0
  382. data/test/plugin_helper/test_extract.rb +194 -0
  383. data/test/plugin_helper/test_formatter.rb +255 -0
  384. data/test/plugin_helper/test_http_server_helper.rb +205 -0
  385. data/test/plugin_helper/test_inject.rb +519 -0
  386. data/test/plugin_helper/test_parser.rb +264 -0
  387. data/test/plugin_helper/test_record_accessor.rb +197 -0
  388. data/test/plugin_helper/test_retry_state.rb +442 -0
  389. data/test/plugin_helper/test_server.rb +1714 -0
  390. data/test/plugin_helper/test_storage.rb +542 -0
  391. data/test/plugin_helper/test_thread.rb +164 -0
  392. data/test/plugin_helper/test_timer.rb +132 -0
  393. data/test/scripts/exec_script.rb +0 -6
  394. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  395. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  396. data/test/scripts/fluent/plugin/out_test.rb +23 -15
  397. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  398. data/test/test_clock.rb +164 -0
  399. data/test/test_config.rb +16 -7
  400. data/test/test_configdsl.rb +2 -2
  401. data/test/test_event.rb +360 -13
  402. data/test/test_event_router.rb +108 -11
  403. data/test/test_event_time.rb +199 -0
  404. data/test/test_filter.rb +48 -6
  405. data/test/test_formatter.rb +11 -391
  406. data/test/test_input.rb +1 -1
  407. data/test/test_log.rb +591 -31
  408. data/test/test_mixin.rb +1 -1
  409. data/test/test_output.rb +121 -185
  410. data/test/test_plugin.rb +251 -0
  411. data/test/test_plugin_classes.rb +177 -10
  412. data/test/test_plugin_helper.rb +81 -0
  413. data/test/test_plugin_id.rb +101 -0
  414. data/test/test_process.rb +8 -42
  415. data/test/test_root_agent.rb +766 -21
  416. data/test/test_supervisor.rb +481 -0
  417. data/test/test_test_drivers.rb +135 -0
  418. data/test/test_time_formatter.rb +282 -0
  419. data/test/test_time_parser.rb +231 -0
  420. data/test/test_unique_id.rb +47 -0
  421. metadata +454 -60
  422. data/COPYING +0 -14
  423. data/ChangeLog +0 -666
  424. data/lib/fluent/buffer.rb +0 -365
  425. data/lib/fluent/plugin/in_status.rb +0 -76
  426. data/test/plugin/test_in_status.rb +0 -38
  427. data/test/test_buffer.rb +0 -624
  428. data/test/test_parser.rb +0 -1305
@@ -14,32 +14,41 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'fcntl'
18
17
 
19
- require 'cool.io'
18
+ require 'fluent/plugin/input'
19
+ require 'fluent/msgpack_factory'
20
20
  require 'yajl'
21
+ require 'digest'
22
+ require 'securerandom'
21
23
 
22
- require 'fluent/input'
23
-
24
- module Fluent
24
+ module Fluent::Plugin
25
25
  class ForwardInput < Input
26
- Plugin.register_input('forward', self)
26
+ Fluent::Plugin.register_input('forward', self)
27
27
 
28
- def initialize
29
- super
30
- require 'fluent/plugin/socket_util'
31
- end
28
+ # See the wiki page below for protocol specification
29
+ # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
30
+
31
+ helpers :server
32
+
33
+ LISTEN_PORT = 24224
32
34
 
33
35
  desc 'The port to listen to.'
34
- config_param :port, :integer, default: DEFAULT_LISTEN_PORT
36
+ config_param :port, :integer, default: LISTEN_PORT
35
37
  desc 'The bind address to listen to.'
36
38
  config_param :bind, :string, default: '0.0.0.0'
39
+
37
40
  config_param :backlog, :integer, default: nil
38
41
  # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
39
42
  desc 'The timeout time used to set linger option.'
40
43
  config_param :linger_timeout, :integer, default: 0
41
44
  # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
42
45
  config_param :blocking_timeout, :time, default: 0.5
46
+ desc 'Try to resolve hostname from IP addresses or not.'
47
+ config_param :resolve_hostname, :bool, default: nil
48
+ desc 'Connections will be disconnected right after receiving first message if this value is true.'
49
+ config_param :deny_keepalive, :bool, default: false
50
+ desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
51
+ config_param :send_keepalive_packet, :bool, default: false
43
52
 
44
53
  desc 'Log warning if received chunk size is larger than this value.'
45
54
  config_param :chunk_size_warn_limit, :size, default: nil
@@ -47,108 +56,230 @@ module Fluent
47
56
  config_param :chunk_size_limit, :size, default: nil
48
57
  desc 'Skip an event if incoming event is invalid.'
49
58
  config_param :skip_invalid_event, :bool, default: false
50
- desc 'Try to resolve hostname from IP addresses or not.'
51
- config_param :resolve_hostname, :bool, default: nil
59
+
52
60
  desc "The field name of the client's source address."
53
61
  config_param :source_address_key, :string, default: nil
54
62
  desc "The field name of the client's hostname."
55
63
  config_param :source_hostname_key, :string, default: nil
56
64
 
65
+ desc "New tag instead of incoming tag"
66
+ config_param :tag, :string, default: nil
67
+ desc "Add prefix to incoming tag"
68
+ config_param :add_tag_prefix, :string, default: nil
69
+
70
+ config_section :security, required: false, multi: false do
71
+ desc 'The hostname'
72
+ config_param :self_hostname, :string
73
+ desc 'Shared key for authentication'
74
+ config_param :shared_key, :string, secret: true
75
+ desc 'If true, use user based authentication'
76
+ config_param :user_auth, :bool, default: false
77
+ desc 'Allow anonymous source. <client> sections required if disabled.'
78
+ config_param :allow_anonymous_source, :bool, default: true
79
+
80
+ ### User based authentication
81
+ config_section :user, param_name: :users, required: false, multi: true do
82
+ desc 'The username for authentication'
83
+ config_param :username, :string
84
+ desc 'The password for authentication'
85
+ config_param :password, :string, secret: true
86
+ end
87
+
88
+ ### Client ip/network authentication & per_host shared key
89
+ config_section :client, param_name: :clients, required: false, multi: true do
90
+ desc 'The IP address or host name of the client'
91
+ config_param :host, :string, default: nil
92
+ desc 'Network address specification'
93
+ config_param :network, :string, default: nil
94
+ desc 'Shared key per client'
95
+ config_param :shared_key, :string, default: nil, secret: true
96
+ desc 'Array of username.'
97
+ config_param :users, :array, default: []
98
+ end
99
+ end
100
+
57
101
  def configure(conf)
58
102
  super
59
103
 
60
104
  if @source_hostname_key
105
+ # TODO: add test
61
106
  if @resolve_hostname.nil?
62
107
  @resolve_hostname = true
63
- elsif !@resolve_hostname # user specifies "false" in configure
108
+ elsif !@resolve_hostname # user specifies "false" in config
64
109
  raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
65
110
  end
66
111
  end
67
112
  @enable_field_injection = @source_address_key || @source_hostname_key
113
+
114
+ raise Fluent::ConfigError, "'tag' parameter must not be empty" if @tag && @tag.empty?
115
+ raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
116
+
117
+ if @security
118
+ if @security.user_auth && @security.users.empty?
119
+ raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
120
+ end
121
+ if !@security.allow_anonymous_source && @security.clients.empty?
122
+ raise Fluent::ConfigError, "<client> sections required if allow_anonymous_source disabled"
123
+ end
124
+
125
+ @nodes = []
126
+
127
+ @security.clients.each do |client|
128
+ if client.host && client.network
129
+ raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
130
+ end
131
+ if !client.host && !client.network
132
+ raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
133
+ end
134
+ source = nil
135
+ if client.host
136
+ begin
137
+ source = IPSocket.getaddress(client.host)
138
+ rescue SocketError
139
+ raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
140
+ end
141
+ end
142
+ source_addr = begin
143
+ IPAddr.new(source || client.network)
144
+ rescue ArgumentError
145
+ raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
146
+ end
147
+ @nodes.push({
148
+ address: source_addr,
149
+ shared_key: (client.shared_key || @security.shared_key),
150
+ users: client.users
151
+ })
152
+ end
153
+ end
154
+
155
+ if @send_keepalive_packet && @deny_keepalive
156
+ raise Fluent::ConfigError, "both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true"
157
+ end
68
158
  end
69
159
 
70
- def start
71
- @loop = Coolio::Loop.new
160
+ def multi_workers_ready?
161
+ true
162
+ end
72
163
 
73
- @lsock = listen
74
- @loop.attach(@lsock)
164
+ HEARTBEAT_UDP_PAYLOAD = "\0"
75
165
 
76
- @usock = SocketUtil.create_udp_socket(@bind)
77
- @usock.bind(@bind, @port)
78
- @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
79
- @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
80
- @loop.attach(@hbr)
166
+ def start
167
+ super
81
168
 
82
- @thread = Thread.new(&method(:run))
169
+ shared_socket = system_config.workers > 1
170
+
171
+ log.info "listening port", port: @port, bind: @bind
172
+ server_create_connection(
173
+ :in_forward_server, @port,
174
+ bind: @bind,
175
+ shared: shared_socket,
176
+ resolve_name: @resolve_hostname,
177
+ linger_timeout: @linger_timeout,
178
+ send_keepalive_packet: @send_keepalive_packet,
179
+ backlog: @backlog,
180
+ &method(:handle_connection)
181
+ )
182
+
183
+ server_create(:in_forward_server_udp_heartbeat, @port, shared: shared_socket, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
184
+ log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
185
+ begin
186
+ sock.write HEARTBEAT_UDP_PAYLOAD
187
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
188
+ log.trace "error while heartbeat response", host: sock.remote_host, error: e
189
+ end
190
+ end
83
191
  end
84
192
 
85
- def shutdown
86
- # In test cases it occasionally appeared that when detaching a watcher, another watcher is also detached.
87
- # In the case in the iteration of watchers, a watcher that has been already detached is intended to be detached
88
- # and therfore RuntimeError occurs saying that it is not attached to a loop.
89
- # It occures only when testing for sending responses to ForwardOutput.
90
- # Sending responses needs to write the socket that is previously used only to read
91
- # and a handler has 2 watchers that is used to read and to write.
92
- # This problem occurs possibly because those watchers are thought to be related to each other
93
- # and when detaching one of them the other is also detached for some reasons.
94
- # As a workaround, check if watchers are attached before detaching them.
95
- @loop.watchers.each {|w| w.detach if w.attached? }
96
- @loop.stop
97
- @usock.close
98
- @thread.join
99
- @lsock.close
193
+ def handle_connection(conn)
194
+ send_data = ->(serializer, data){ conn.write serializer.call(data) }
195
+
196
+ log.trace "connected fluent socket", addr: conn.remote_addr, port: conn.remote_port
197
+ state = :established
198
+ nonce = nil
199
+ user_auth_salt = nil
200
+
201
+ if @security
202
+ # security enabled session MUST use MessagePack as serialization format
203
+ state = :helo
204
+ nonce = generate_salt
205
+ user_auth_salt = generate_salt
206
+ send_data.call(:to_msgpack.to_proc, generate_helo(nonce, user_auth_salt))
207
+ state = :pingpong
208
+ end
209
+
210
+ log.trace "accepted fluent socket", addr: conn.remote_addr, port: conn.remote_port
211
+
212
+ read_messages(conn) do |msg, chunk_size, serializer|
213
+ case state
214
+ when :pingpong
215
+ success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
216
+ unless success
217
+ conn.on(:write_complete) { |c| c.close_after_write_complete }
218
+ send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
219
+ next
220
+ end
221
+ send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))
222
+
223
+ log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
224
+ state = :established
225
+ when :established
226
+ options = on_message(msg, chunk_size, conn)
227
+ if options && r = response(options)
228
+ log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
229
+ conn.on(:write_complete) { |c| c.close } if @deny_keepalive
230
+ send_data.call(serializer, r)
231
+ else
232
+ if @deny_keepalive
233
+ conn.close
234
+ end
235
+ end
236
+ else
237
+ raise "BUG: unknown session state: #{state}"
238
+ end
239
+ end
100
240
  end
101
241
 
102
- def listen
103
- log.info "listening fluent socket on #{@bind}:#{@port}"
104
- s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, @resolve_hostname, method(:on_message))
105
- s.listen(@backlog) unless @backlog.nil?
106
- s
242
+ def read_messages(conn, &block)
243
+ feeder = nil
244
+ serializer = nil
245
+ bytes = 0
246
+ conn.data do |data|
247
+ # only for first call of callback
248
+ unless feeder
249
+ first = data[0]
250
+ if first == '{' || first == '[' # json
251
+ parser = Yajl::Parser.new
252
+ parser.on_parse_complete = ->(obj){
253
+ block.call(obj, bytes, serializer)
254
+ bytes = 0
255
+ }
256
+ serializer = :to_json.to_proc
257
+ feeder = ->(d){ parser << d }
258
+ else # msgpack
259
+ parser = Fluent::Engine.msgpack_factory.unpacker
260
+ serializer = :to_msgpack.to_proc
261
+ feeder = ->(d){
262
+ parser.feed_each(d){|obj|
263
+ block.call(obj, bytes, serializer)
264
+ bytes = 0
265
+ }
266
+ }
267
+ end
268
+ end
269
+
270
+ bytes += data.bytesize
271
+ feeder.call(data)
272
+ end
107
273
  end
108
274
 
109
- #config_param :path, :string, :default => DEFAULT_SOCKET_PATH
110
- #def listen
111
- # if File.exist?(@path)
112
- # File.unlink(@path)
113
- # end
114
- # FileUtils.mkdir_p File.dirname(@path)
115
- # log.debug "listening fluent socket on #{@path}"
116
- # Coolio::UNIXServer.new(@path, Handler, method(:on_message))
117
- #end
118
-
119
- def run
120
- @loop.run(@blocking_timeout)
121
- rescue => e
122
- log.error "unexpected error", error: e, error_class: e.class
123
- log.error_backtrace
275
+ def response(option)
276
+ if option && option['chunk']
277
+ return { 'ack' => option['chunk'] }
278
+ end
279
+ nil
124
280
  end
125
281
 
126
- private
127
-
128
- # message Entry {
129
- # 1: long time
130
- # 2: object record
131
- # }
132
- #
133
- # message Forward {
134
- # 1: string tag
135
- # 2: list<Entry> entries
136
- # 3: object option (optional)
137
- # }
138
- #
139
- # message PackedForward {
140
- # 1: string tag
141
- # 2: raw entries # msgpack stream of Entry
142
- # 3: object option (optional)
143
- # }
144
- #
145
- # message Message {
146
- # 1: string tag
147
- # 2: long? time
148
- # 3: object record
149
- # 4: object option (optional)
150
- # }
151
- def on_message(msg, chunk_size, peeraddr)
282
+ def on_message(msg, chunk_size, conn)
152
283
  if msg.nil?
153
284
  # for future TCP heartbeat_request
154
285
  return
@@ -156,7 +287,7 @@ module Fluent
156
287
 
157
288
  # TODO: raise an exception if broken chunk is generated by recoverable situation
158
289
  unless msg.is_a?(Array)
159
- log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
290
+ log.warn "incoming chunk is broken:", host: conn.remote_host, msg: msg
160
291
  return
161
292
  end
162
293
 
@@ -164,36 +295,46 @@ module Fluent
164
295
  entries = msg[1]
165
296
 
166
297
  if @chunk_size_limit && (chunk_size > @chunk_size_limit)
167
- log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
298
+ log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size
168
299
  return
169
300
  elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
170
- log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
301
+ log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
171
302
  end
172
303
 
173
- if entries.class == String
304
+ tag = @tag.dup if @tag
305
+ tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
306
+
307
+ case entries
308
+ when String
174
309
  # PackedForward
175
- es = MessagePackEventStream.new(entries)
176
- es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
177
- es = add_source_host(es, peeraddr) if @enable_field_injection
178
- router.emit_stream(tag, es)
179
310
  option = msg[2]
311
+ size = (option && option['size']) || 0
312
+ es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
313
+ es = es_class.new(entries, nil, size.to_i)
314
+ es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
315
+ if @enable_field_injection
316
+ es = add_source_info(es, conn)
317
+ end
318
+ router.emit_stream(tag, es)
180
319
 
181
- elsif entries.class == Array
320
+ when Array
182
321
  # Forward
183
322
  es = if @skip_invalid_event
184
- check_and_skip_invalid_event(tag, entries, peeraddr)
323
+ check_and_skip_invalid_event(tag, entries, conn.remote_host)
185
324
  else
186
- es = MultiEventStream.new
325
+ es = Fluent::MultiEventStream.new
187
326
  entries.each { |e|
188
327
  record = e[1]
189
328
  next if record.nil?
190
329
  time = e[0]
191
- time = (now ||= Engine.now) if time.to_i == 0
330
+ time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
192
331
  es.add(time, record)
193
332
  }
194
333
  es
195
334
  end
196
- es = add_source_host(es, peeraddr) if @enable_field_injection
335
+ if @enable_field_injection
336
+ es = add_source_info(es, conn)
337
+ end
197
338
  router.emit_stream(tag, es)
198
339
  option = msg[2]
199
340
 
@@ -202,14 +343,14 @@ module Fluent
202
343
  time = msg[1]
203
344
  record = msg[2]
204
345
  if @skip_invalid_event && invalid_event?(tag, time, record)
205
- log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
346
+ log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
206
347
  return msg[3] # retry never succeeded so return ack and drop incoming event.
207
348
  end
208
349
  return if record.nil?
209
- time = Engine.now if time == 0
350
+ time = Fluent::Engine.now if time.to_i == 0
210
351
  if @enable_field_injection
211
- record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
212
- record[@source_address_key] = peeraddr[3] if @source_address_key
352
+ record[@source_address_key] = conn.remote_addr if @source_address_key
353
+ record[@source_hostname_key] = conn.remote_host if @source_hostname_key
213
354
  end
214
355
  router.emit(tag, time, record)
215
356
  option = msg[3]
@@ -220,14 +361,14 @@ module Fluent
220
361
  end
221
362
 
222
363
  def invalid_event?(tag, time, record)
223
- !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
364
+ !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
224
365
  end
225
366
 
226
- def check_and_skip_invalid_event(tag, es, peeraddr)
227
- new_es = MultiEventStream.new
367
+ def check_and_skip_invalid_event(tag, es, remote_host)
368
+ new_es = Fluent::MultiEventStream.new
228
369
  es.each { |time, record|
229
370
  if invalid_event?(tag, time, record)
230
- log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
371
+ log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
231
372
  next
232
373
  end
233
374
  new_es.add(time, record)
@@ -235,24 +376,24 @@ module Fluent
235
376
  new_es
236
377
  end
237
378
 
238
- def add_source_host(es, peeraddr)
239
- new_es = MultiEventStream.new
379
+ def add_source_info(es, conn)
380
+ new_es = Fluent::MultiEventStream.new
240
381
  if @source_address_key && @source_hostname_key
241
- address = peeraddr[3]
242
- hostname = peeraddr[2]
382
+ address = conn.remote_addr
383
+ hostname = conn.remote_host
243
384
  es.each { |time, record|
244
385
  record[@source_address_key] = address
245
386
  record[@source_hostname_key] = hostname
246
387
  new_es.add(time, record)
247
388
  }
248
389
  elsif @source_address_key
249
- address = peeraddr[3]
390
+ address = conn.remote_addr
250
391
  es.each { |time, record|
251
392
  record[@source_address_key] = address
252
393
  new_es.add(time, record)
253
394
  }
254
395
  elsif @source_hostname_key
255
- hostname = peeraddr[2]
396
+ hostname = conn.remote_host
256
397
  es.each { |time, record|
257
398
  record[@source_hostname_key] = hostname
258
399
  new_es.add(time, record)
@@ -263,128 +404,70 @@ module Fluent
263
404
  new_es
264
405
  end
265
406
 
266
- def source_message(peeraddr)
267
- _, port, host, addr = peeraddr
268
- "host: #{host}, addr: #{addr}, port: #{port}"
407
+ def select_authenticate_users(node, username)
408
+ if node.nil? || node[:users].empty?
409
+ @security.users.select{|u| u.username == username}
410
+ else
411
+ @security.users.select{|u| node[:users].include?(u.username) && u.username == username}
412
+ end
269
413
  end
270
414
 
271
- class Handler < Coolio::Socket
272
- PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
273
-
274
- def initialize(io, linger_timeout, log, resolve_hostname, on_message)
275
- super(io)
276
-
277
- @peeraddr = nil
278
- if io.is_a?(TCPSocket) # for unix domain socket support in the future
279
- io.do_not_reverse_lookup = !resolve_hostname unless resolve_hostname.nil?
280
-
281
- @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
282
- opt = [1, linger_timeout].pack('I!I!') # { int l_onoff; int l_linger; }
283
- io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
284
- end
415
+ def generate_salt
416
+ ::SecureRandom.random_bytes(16)
417
+ end
285
418
 
286
- @chunk_counter = 0
287
- @on_message = on_message
288
- @log = log
289
- @log.trace {
290
- begin
291
- remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
292
- rescue => e
293
- remote_port = nil
294
- remote_addr = nil
295
- end
296
- "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
297
- }
298
- end
419
+ def generate_helo(nonce, user_auth_salt)
420
+ log.debug "generating helo"
421
+ # ['HELO', options(hash)]
422
+ ['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
423
+ end
299
424
 
300
- def on_connect
425
+ def check_ping(message, remote_addr, user_auth_salt, nonce)
426
+ log.debug "checking ping"
427
+ # ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']
428
+ unless message.size == 6 && message[0] == 'PING'
429
+ return false, 'invalid ping message'
301
430
  end
431
+ _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
302
432
 
303
- def on_read(data)
304
- first = data[0]
305
- if first == '{' || first == '['
306
- m = method(:on_read_json)
307
- @serializer = :to_json.to_proc
308
- @y = Yajl::Parser.new
309
- @y.on_parse_complete = lambda { |obj|
310
- option = @on_message.call(obj, @chunk_counter, @peeraddr)
311
- respond option if option
312
- @chunk_counter = 0
313
- }
314
- else
315
- m = method(:on_read_msgpack)
316
- @serializer = :to_msgpack.to_proc
317
- @u = Fluent::Engine.msgpack_factory.unpacker
318
- end
319
-
320
- (class << self; self; end).module_eval do
321
- define_method(:on_read, m)
322
- end
323
- m.call(data)
433
+ node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
434
+ if !node && !@security.allow_anonymous_source
435
+ log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
436
+ return false, "anonymous source host '#{remote_addr}' denied", nil
324
437
  end
325
438
 
326
- def on_read_json(data)
327
- @chunk_counter += data.bytesize
328
- @y << data
329
- rescue => e
330
- @log.error "forward error", error: e, error_class: e.class
331
- @log.error_backtrace
332
- close
439
+ shared_key = node ? node[:shared_key] : @security.shared_key
440
+ serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(nonce).update(shared_key).hexdigest
441
+ if shared_key_hexdigest != serverside
442
+ log.warn "Shared key mismatch", address: remote_addr, hostname: hostname
443
+ return false, 'shared_key mismatch', nil
333
444
  end
334
445
 
335
- def on_read_msgpack(data)
336
- @chunk_counter += data.bytesize
337
- @u.feed_each(data) do |obj|
338
- option = @on_message.call(obj, @chunk_counter, @peeraddr)
339
- respond option if option
340
- @chunk_counter = 0
446
+ if @security.user_auth
447
+ users = select_authenticate_users(node, username)
448
+ success = false
449
+ users.each do |user|
450
+ passhash = Digest::SHA512.new.update(user_auth_salt).update(username).update(user[:password]).hexdigest
451
+ success ||= (passhash == password_digest)
341
452
  end
342
- rescue => e
343
- @log.error "forward error", error: e, error_class: e.class
344
- @log.error_backtrace
345
- close
346
- end
347
-
348
- def respond(option)
349
- if option && option['chunk']
350
- res = { 'ack' => option['chunk'] }
351
- write @serializer.call(res)
352
- @log.trace { "sent response to fluent socket" }
453
+ unless success
454
+ log.warn "Authentication failed", address: remote_addr, hostname: hostname, username: username
455
+ return false, 'username/password mismatch', nil
353
456
  end
354
457
  end
355
458
 
356
- def on_close
357
- @log.trace { "closed socket" }
358
- end
459
+ return true, shared_key_salt, shared_key
359
460
  end
360
461
 
361
- class HeartbeatRequestHandler < Coolio::IO
362
- def initialize(io, callback)
363
- super(io)
364
- @io = io
365
- @callback = callback
366
- end
367
-
368
- def on_readable
369
- begin
370
- msg, addr = @io.recvfrom(1024)
371
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
372
- return
373
- end
374
- host = addr[3]
375
- port = addr[1]
376
- @callback.call(host, port, msg)
377
- rescue
378
- # TODO log?
462
+ def generate_pong(auth_result, reason_or_salt, nonce, shared_key)
463
+ log.debug "generating pong"
464
+ # ['PONG', bool(authentication result), 'reason if authentication failed', self_hostname, sha512_hex(salt + self_hostname + nonce + sharedkey)]
465
+ unless auth_result
466
+ return ['PONG', false, reason_or_salt, '', '']
379
467
  end
380
- end
381
468
 
382
- def on_heartbeat_request(host, port, msg)
383
- #log.trace "heartbeat request from #{host}:#{port}"
384
- begin
385
- @usock.send "\0", 0, host, port
386
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
387
- end
469
+ shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
470
+ ['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
388
471
  end
389
472
  end
390
473
  end