fluentd222 1.16.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (562) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.yaml +71 -0
  4. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  5. data/.github/ISSUE_TEMPLATE/feature_request.yaml +39 -0
  6. data/.github/ISSUE_TEMPLATE.md +17 -0
  7. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  8. data/.github/workflows/stale-actions.yml +24 -0
  9. data/.github/workflows/test-ruby-head.yaml +31 -0
  10. data/.github/workflows/test.yaml +32 -0
  11. data/.gitignore +30 -0
  12. data/ADOPTERS.md +5 -0
  13. data/AUTHORS +2 -0
  14. data/CHANGELOG.md +2720 -0
  15. data/CONTRIBUTING.md +45 -0
  16. data/GOVERNANCE.md +55 -0
  17. data/Gemfile +9 -0
  18. data/GithubWorkflow.md +78 -0
  19. data/LICENSE +202 -0
  20. data/MAINTAINERS.md +13 -0
  21. data/README.md +75 -0
  22. data/Rakefile +79 -0
  23. data/SECURITY.md +14 -0
  24. data/bin/fluent-binlog-reader +7 -0
  25. data/bin/fluent-ca-generate +6 -0
  26. data/bin/fluent-cap-ctl +7 -0
  27. data/bin/fluent-cat +5 -0
  28. data/bin/fluent-ctl +7 -0
  29. data/bin/fluent-debug +5 -0
  30. data/bin/fluent-gem +9 -0
  31. data/bin/fluent-plugin-config-format +5 -0
  32. data/bin/fluent-plugin-generate +5 -0
  33. data/bin/fluentd +15 -0
  34. data/code-of-conduct.md +3 -0
  35. data/docs/SECURITY_AUDIT.pdf +0 -0
  36. data/example/copy_roundrobin.conf +39 -0
  37. data/example/counter.conf +18 -0
  38. data/example/filter_stdout.conf +22 -0
  39. data/example/in_forward.conf +14 -0
  40. data/example/in_forward_client.conf +37 -0
  41. data/example/in_forward_shared_key.conf +15 -0
  42. data/example/in_forward_tls.conf +14 -0
  43. data/example/in_forward_users.conf +24 -0
  44. data/example/in_forward_workers.conf +21 -0
  45. data/example/in_http.conf +16 -0
  46. data/example/in_out_forward.conf +17 -0
  47. data/example/in_sample_blocks.conf +17 -0
  48. data/example/in_sample_with_compression.conf +23 -0
  49. data/example/in_syslog.conf +15 -0
  50. data/example/in_tail.conf +14 -0
  51. data/example/in_tcp.conf +13 -0
  52. data/example/in_udp.conf +13 -0
  53. data/example/logevents.conf +25 -0
  54. data/example/multi_filters.conf +61 -0
  55. data/example/out_copy.conf +20 -0
  56. data/example/out_exec_filter.conf +42 -0
  57. data/example/out_file.conf +13 -0
  58. data/example/out_forward.conf +35 -0
  59. data/example/out_forward_buf_file.conf +23 -0
  60. data/example/out_forward_client.conf +109 -0
  61. data/example/out_forward_heartbeat_none.conf +16 -0
  62. data/example/out_forward_sd.conf +17 -0
  63. data/example/out_forward_shared_key.conf +36 -0
  64. data/example/out_forward_tls.conf +18 -0
  65. data/example/out_forward_users.conf +65 -0
  66. data/example/out_null.conf +36 -0
  67. data/example/sd.yaml +8 -0
  68. data/example/secondary_file.conf +42 -0
  69. data/example/suppress_config_dump.conf +7 -0
  70. data/example/v0_12_filter.conf +78 -0
  71. data/example/v1_literal_example.conf +36 -0
  72. data/example/worker_section.conf +36 -0
  73. data/fluent.conf +139 -0
  74. data/fluentd.gemspec +54 -0
  75. data/lib/fluent/agent.rb +168 -0
  76. data/lib/fluent/capability.rb +87 -0
  77. data/lib/fluent/clock.rb +66 -0
  78. data/lib/fluent/command/binlog_reader.rb +244 -0
  79. data/lib/fluent/command/bundler_injection.rb +45 -0
  80. data/lib/fluent/command/ca_generate.rb +184 -0
  81. data/lib/fluent/command/cap_ctl.rb +174 -0
  82. data/lib/fluent/command/cat.rb +365 -0
  83. data/lib/fluent/command/ctl.rb +180 -0
  84. data/lib/fluent/command/debug.rb +103 -0
  85. data/lib/fluent/command/fluentd.rb +374 -0
  86. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  87. data/lib/fluent/command/plugin_generator.rb +365 -0
  88. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  89. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  90. data/lib/fluent/compat/exec_util.rb +129 -0
  91. data/lib/fluent/compat/file_util.rb +54 -0
  92. data/lib/fluent/compat/filter.rb +68 -0
  93. data/lib/fluent/compat/formatter.rb +111 -0
  94. data/lib/fluent/compat/formatter_utils.rb +85 -0
  95. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  96. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  97. data/lib/fluent/compat/input.rb +49 -0
  98. data/lib/fluent/compat/output.rb +721 -0
  99. data/lib/fluent/compat/output_chain.rb +60 -0
  100. data/lib/fluent/compat/parser.rb +310 -0
  101. data/lib/fluent/compat/parser_utils.rb +40 -0
  102. data/lib/fluent/compat/propagate_default.rb +62 -0
  103. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  104. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  105. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  106. data/lib/fluent/compat/socket_util.rb +165 -0
  107. data/lib/fluent/compat/string_util.rb +34 -0
  108. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  109. data/lib/fluent/compat/type_converter.rb +90 -0
  110. data/lib/fluent/config/basic_parser.rb +123 -0
  111. data/lib/fluent/config/configure_proxy.rb +424 -0
  112. data/lib/fluent/config/dsl.rb +152 -0
  113. data/lib/fluent/config/element.rb +265 -0
  114. data/lib/fluent/config/error.rb +44 -0
  115. data/lib/fluent/config/literal_parser.rb +286 -0
  116. data/lib/fluent/config/parser.rb +107 -0
  117. data/lib/fluent/config/section.rb +272 -0
  118. data/lib/fluent/config/types.rb +249 -0
  119. data/lib/fluent/config/v1_parser.rb +192 -0
  120. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  121. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  122. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  123. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  124. data/lib/fluent/config/yaml_parser.rb +56 -0
  125. data/lib/fluent/config.rb +89 -0
  126. data/lib/fluent/configurable.rb +201 -0
  127. data/lib/fluent/counter/base_socket.rb +44 -0
  128. data/lib/fluent/counter/client.rb +297 -0
  129. data/lib/fluent/counter/error.rb +86 -0
  130. data/lib/fluent/counter/mutex_hash.rb +163 -0
  131. data/lib/fluent/counter/server.rb +273 -0
  132. data/lib/fluent/counter/store.rb +205 -0
  133. data/lib/fluent/counter/validator.rb +145 -0
  134. data/lib/fluent/counter.rb +23 -0
  135. data/lib/fluent/daemon.rb +13 -0
  136. data/lib/fluent/daemonizer.rb +88 -0
  137. data/lib/fluent/engine.rb +253 -0
  138. data/lib/fluent/env.rb +40 -0
  139. data/lib/fluent/error.rb +37 -0
  140. data/lib/fluent/event.rb +330 -0
  141. data/lib/fluent/event_router.rb +315 -0
  142. data/lib/fluent/ext_monitor_require.rb +28 -0
  143. data/lib/fluent/file_wrapper.rb +137 -0
  144. data/lib/fluent/filter.rb +21 -0
  145. data/lib/fluent/fluent_log_event_router.rb +139 -0
  146. data/lib/fluent/formatter.rb +23 -0
  147. data/lib/fluent/input.rb +21 -0
  148. data/lib/fluent/label.rb +46 -0
  149. data/lib/fluent/load.rb +34 -0
  150. data/lib/fluent/log/console_adapter.rb +66 -0
  151. data/lib/fluent/log.rb +752 -0
  152. data/lib/fluent/match.rb +187 -0
  153. data/lib/fluent/mixin.rb +31 -0
  154. data/lib/fluent/msgpack_factory.rb +111 -0
  155. data/lib/fluent/oj_options.rb +61 -0
  156. data/lib/fluent/output.rb +29 -0
  157. data/lib/fluent/output_chain.rb +23 -0
  158. data/lib/fluent/parser.rb +23 -0
  159. data/lib/fluent/plugin/bare_output.rb +104 -0
  160. data/lib/fluent/plugin/base.rb +214 -0
  161. data/lib/fluent/plugin/buf_file.rb +242 -0
  162. data/lib/fluent/plugin/buf_file_single.rb +254 -0
  163. data/lib/fluent/plugin/buf_memory.rb +34 -0
  164. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  165. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  166. data/lib/fluent/plugin/buffer/file_single_chunk.rb +310 -0
  167. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  168. data/lib/fluent/plugin/buffer.rb +941 -0
  169. data/lib/fluent/plugin/compressable.rb +96 -0
  170. data/lib/fluent/plugin/exec_util.rb +22 -0
  171. data/lib/fluent/plugin/file_util.rb +22 -0
  172. data/lib/fluent/plugin/filter.rb +127 -0
  173. data/lib/fluent/plugin/filter_grep.rb +189 -0
  174. data/lib/fluent/plugin/filter_parser.rb +130 -0
  175. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  176. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  177. data/lib/fluent/plugin/formatter.rb +75 -0
  178. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  179. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  180. data/lib/fluent/plugin/formatter_json.rb +59 -0
  181. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  182. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  183. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  184. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  185. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  186. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  187. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  188. data/lib/fluent/plugin/in_dummy.rb +18 -0
  189. data/lib/fluent/plugin/in_exec.rb +110 -0
  190. data/lib/fluent/plugin/in_forward.rb +473 -0
  191. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  192. data/lib/fluent/plugin/in_http.rb +677 -0
  193. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  194. data/lib/fluent/plugin/in_object_space.rb +93 -0
  195. data/lib/fluent/plugin/in_sample.rb +141 -0
  196. data/lib/fluent/plugin/in_syslog.rb +276 -0
  197. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  198. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  199. data/lib/fluent/plugin/in_tail.rb +1299 -0
  200. data/lib/fluent/plugin/in_tcp.rb +226 -0
  201. data/lib/fluent/plugin/in_udp.rb +92 -0
  202. data/lib/fluent/plugin/in_unix.rb +195 -0
  203. data/lib/fluent/plugin/input.rb +75 -0
  204. data/lib/fluent/plugin/metrics.rb +119 -0
  205. data/lib/fluent/plugin/metrics_local.rb +96 -0
  206. data/lib/fluent/plugin/multi_output.rb +195 -0
  207. data/lib/fluent/plugin/out_copy.rb +120 -0
  208. data/lib/fluent/plugin/out_exec.rb +105 -0
  209. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  210. data/lib/fluent/plugin/out_file.rb +340 -0
  211. data/lib/fluent/plugin/out_forward/ack_handler.rb +176 -0
  212. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  213. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  214. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  215. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  216. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  217. data/lib/fluent/plugin/out_forward/socket_cache.rb +142 -0
  218. data/lib/fluent/plugin/out_forward.rb +826 -0
  219. data/lib/fluent/plugin/out_http.rb +275 -0
  220. data/lib/fluent/plugin/out_null.rb +74 -0
  221. data/lib/fluent/plugin/out_relabel.rb +32 -0
  222. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  223. data/lib/fluent/plugin/out_secondary_file.rb +148 -0
  224. data/lib/fluent/plugin/out_stdout.rb +74 -0
  225. data/lib/fluent/plugin/out_stream.rb +130 -0
  226. data/lib/fluent/plugin/output.rb +1603 -0
  227. data/lib/fluent/plugin/owned_by_mixin.rb +41 -0
  228. data/lib/fluent/plugin/parser.rb +274 -0
  229. data/lib/fluent/plugin/parser_apache.rb +28 -0
  230. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  231. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  232. data/lib/fluent/plugin/parser_csv.rb +114 -0
  233. data/lib/fluent/plugin/parser_json.rb +96 -0
  234. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  235. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  236. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  237. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  238. data/lib/fluent/plugin/parser_none.rb +36 -0
  239. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  240. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  241. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  242. data/lib/fluent/plugin/sd_file.rb +156 -0
  243. data/lib/fluent/plugin/sd_srv.rb +135 -0
  244. data/lib/fluent/plugin/sd_static.rb +58 -0
  245. data/lib/fluent/plugin/service_discovery.rb +65 -0
  246. data/lib/fluent/plugin/socket_util.rb +22 -0
  247. data/lib/fluent/plugin/storage.rb +84 -0
  248. data/lib/fluent/plugin/storage_local.rb +162 -0
  249. data/lib/fluent/plugin/string_util.rb +22 -0
  250. data/lib/fluent/plugin.rb +206 -0
  251. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  252. data/lib/fluent/plugin_helper/child_process.rb +369 -0
  253. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  254. data/lib/fluent/plugin_helper/counter.rb +51 -0
  255. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  256. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  257. data/lib/fluent/plugin_helper/extract.rb +104 -0
  258. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  259. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  260. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  261. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  262. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  263. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  264. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  265. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  266. data/lib/fluent/plugin_helper/http_server/server.rb +94 -0
  267. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  268. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  269. data/lib/fluent/plugin_helper/inject.rb +154 -0
  270. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  271. data/lib/fluent/plugin_helper/parser.rb +147 -0
  272. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  273. data/lib/fluent/plugin_helper/retry_state.rb +219 -0
  274. data/lib/fluent/plugin_helper/server.rb +828 -0
  275. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  276. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  277. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  278. data/lib/fluent/plugin_helper/socket.rb +288 -0
  279. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  280. data/lib/fluent/plugin_helper/storage.rb +349 -0
  281. data/lib/fluent/plugin_helper/thread.rb +180 -0
  282. data/lib/fluent/plugin_helper/timer.rb +92 -0
  283. data/lib/fluent/plugin_helper.rb +75 -0
  284. data/lib/fluent/plugin_id.rb +93 -0
  285. data/lib/fluent/process.rb +22 -0
  286. data/lib/fluent/registry.rb +117 -0
  287. data/lib/fluent/root_agent.rb +372 -0
  288. data/lib/fluent/rpc.rb +95 -0
  289. data/lib/fluent/static_config_analysis.rb +194 -0
  290. data/lib/fluent/supervisor.rb +1076 -0
  291. data/lib/fluent/system_config.rb +189 -0
  292. data/lib/fluent/test/base.rb +78 -0
  293. data/lib/fluent/test/driver/base.rb +231 -0
  294. data/lib/fluent/test/driver/base_owned.rb +83 -0
  295. data/lib/fluent/test/driver/base_owner.rb +135 -0
  296. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  297. data/lib/fluent/test/driver/filter.rb +61 -0
  298. data/lib/fluent/test/driver/formatter.rb +30 -0
  299. data/lib/fluent/test/driver/input.rb +31 -0
  300. data/lib/fluent/test/driver/multi_output.rb +53 -0
  301. data/lib/fluent/test/driver/output.rb +102 -0
  302. data/lib/fluent/test/driver/parser.rb +30 -0
  303. data/lib/fluent/test/driver/storage.rb +30 -0
  304. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  305. data/lib/fluent/test/filter_test.rb +77 -0
  306. data/lib/fluent/test/formatter_test.rb +65 -0
  307. data/lib/fluent/test/helpers.rb +134 -0
  308. data/lib/fluent/test/input_test.rb +174 -0
  309. data/lib/fluent/test/log.rb +79 -0
  310. data/lib/fluent/test/output_test.rb +156 -0
  311. data/lib/fluent/test/parser_test.rb +70 -0
  312. data/lib/fluent/test/startup_shutdown.rb +44 -0
  313. data/lib/fluent/test.rb +58 -0
  314. data/lib/fluent/time.rb +512 -0
  315. data/lib/fluent/timezone.rb +171 -0
  316. data/lib/fluent/tls.rb +81 -0
  317. data/lib/fluent/unique_id.rb +39 -0
  318. data/lib/fluent/variable_store.rb +40 -0
  319. data/lib/fluent/version.rb +21 -0
  320. data/lib/fluent/win32api.rb +38 -0
  321. data/lib/fluent/winsvc.rb +100 -0
  322. data/templates/new_gem/Gemfile +3 -0
  323. data/templates/new_gem/README.md.erb +43 -0
  324. data/templates/new_gem/Rakefile +13 -0
  325. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  326. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  327. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  328. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  329. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  330. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  331. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  332. data/templates/new_gem/test/helper.rb.erb +7 -0
  333. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  335. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  336. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  337. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  338. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  339. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  340. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  341. data/templates/plugin_config_formatter/param.md.erb +34 -0
  342. data/templates/plugin_config_formatter/section.md.erb +12 -0
  343. data/test/command/test_binlog_reader.rb +362 -0
  344. data/test/command/test_ca_generate.rb +70 -0
  345. data/test/command/test_cap_ctl.rb +100 -0
  346. data/test/command/test_cat.rb +128 -0
  347. data/test/command/test_ctl.rb +56 -0
  348. data/test/command/test_fluentd.rb +1291 -0
  349. data/test/command/test_plugin_config_formatter.rb +397 -0
  350. data/test/command/test_plugin_generator.rb +109 -0
  351. data/test/compat/test_calls_super.rb +166 -0
  352. data/test/compat/test_parser.rb +92 -0
  353. data/test/config/assertions.rb +42 -0
  354. data/test/config/test_config_parser.rb +551 -0
  355. data/test/config/test_configurable.rb +1784 -0
  356. data/test/config/test_configure_proxy.rb +604 -0
  357. data/test/config/test_dsl.rb +415 -0
  358. data/test/config/test_element.rb +518 -0
  359. data/test/config/test_literal_parser.rb +309 -0
  360. data/test/config/test_plugin_configuration.rb +56 -0
  361. data/test/config/test_section.rb +191 -0
  362. data/test/config/test_system_config.rb +195 -0
  363. data/test/config/test_types.rb +408 -0
  364. data/test/counter/test_client.rb +563 -0
  365. data/test/counter/test_error.rb +44 -0
  366. data/test/counter/test_mutex_hash.rb +179 -0
  367. data/test/counter/test_server.rb +589 -0
  368. data/test/counter/test_store.rb +258 -0
  369. data/test/counter/test_validator.rb +137 -0
  370. data/test/helper.rb +155 -0
  371. data/test/helpers/fuzzy_assert.rb +89 -0
  372. data/test/helpers/process_extenstion.rb +33 -0
  373. data/test/log/test_console_adapter.rb +110 -0
  374. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  375. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  376. data/test/plugin/data/2010/01/20100102.log +0 -0
  377. data/test/plugin/data/log/bar +0 -0
  378. data/test/plugin/data/log/foo/bar.log +0 -0
  379. data/test/plugin/data/log/foo/bar2 +0 -0
  380. data/test/plugin/data/log/test.log +0 -0
  381. data/test/plugin/data/sd_file/config +11 -0
  382. data/test/plugin/data/sd_file/config.json +17 -0
  383. data/test/plugin/data/sd_file/config.yaml +11 -0
  384. data/test/plugin/data/sd_file/config.yml +11 -0
  385. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  386. data/test/plugin/in_tail/test_fifo.rb +121 -0
  387. data/test/plugin/in_tail/test_io_handler.rb +150 -0
  388. data/test/plugin/in_tail/test_position_file.rb +346 -0
  389. data/test/plugin/out_forward/test_ack_handler.rb +140 -0
  390. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  391. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  392. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  393. data/test/plugin/out_forward/test_socket_cache.rb +174 -0
  394. data/test/plugin/test_bare_output.rb +131 -0
  395. data/test/plugin/test_base.rb +247 -0
  396. data/test/plugin/test_buf_file.rb +1314 -0
  397. data/test/plugin/test_buf_file_single.rb +898 -0
  398. data/test/plugin/test_buf_memory.rb +42 -0
  399. data/test/plugin/test_buffer.rb +1434 -0
  400. data/test/plugin/test_buffer_chunk.rb +209 -0
  401. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  402. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  403. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  404. data/test/plugin/test_compressable.rb +87 -0
  405. data/test/plugin/test_file_util.rb +96 -0
  406. data/test/plugin/test_filter.rb +368 -0
  407. data/test/plugin/test_filter_grep.rb +697 -0
  408. data/test/plugin/test_filter_parser.rb +731 -0
  409. data/test/plugin/test_filter_record_transformer.rb +577 -0
  410. data/test/plugin/test_filter_stdout.rb +207 -0
  411. data/test/plugin/test_formatter_csv.rb +136 -0
  412. data/test/plugin/test_formatter_hash.rb +38 -0
  413. data/test/plugin/test_formatter_json.rb +61 -0
  414. data/test/plugin/test_formatter_ltsv.rb +70 -0
  415. data/test/plugin/test_formatter_msgpack.rb +28 -0
  416. data/test/plugin/test_formatter_out_file.rb +116 -0
  417. data/test/plugin/test_formatter_single_value.rb +44 -0
  418. data/test/plugin/test_formatter_tsv.rb +76 -0
  419. data/test/plugin/test_in_debug_agent.rb +49 -0
  420. data/test/plugin/test_in_exec.rb +261 -0
  421. data/test/plugin/test_in_forward.rb +1178 -0
  422. data/test/plugin/test_in_gc_stat.rb +62 -0
  423. data/test/plugin/test_in_http.rb +1102 -0
  424. data/test/plugin/test_in_monitor_agent.rb +922 -0
  425. data/test/plugin/test_in_object_space.rb +66 -0
  426. data/test/plugin/test_in_sample.rb +190 -0
  427. data/test/plugin/test_in_syslog.rb +505 -0
  428. data/test/plugin/test_in_tail.rb +3125 -0
  429. data/test/plugin/test_in_tcp.rb +328 -0
  430. data/test/plugin/test_in_udp.rb +296 -0
  431. data/test/plugin/test_in_unix.rb +181 -0
  432. data/test/plugin/test_input.rb +137 -0
  433. data/test/plugin/test_metadata.rb +89 -0
  434. data/test/plugin/test_metrics.rb +294 -0
  435. data/test/plugin/test_metrics_local.rb +96 -0
  436. data/test/plugin/test_multi_output.rb +204 -0
  437. data/test/plugin/test_out_copy.rb +308 -0
  438. data/test/plugin/test_out_exec.rb +312 -0
  439. data/test/plugin/test_out_exec_filter.rb +606 -0
  440. data/test/plugin/test_out_file.rb +1038 -0
  441. data/test/plugin/test_out_forward.rb +1361 -0
  442. data/test/plugin/test_out_http.rb +429 -0
  443. data/test/plugin/test_out_null.rb +105 -0
  444. data/test/plugin/test_out_relabel.rb +28 -0
  445. data/test/plugin/test_out_roundrobin.rb +146 -0
  446. data/test/plugin/test_out_secondary_file.rb +458 -0
  447. data/test/plugin/test_out_stdout.rb +205 -0
  448. data/test/plugin/test_out_stream.rb +103 -0
  449. data/test/plugin/test_output.rb +1334 -0
  450. data/test/plugin/test_output_as_buffered.rb +2024 -0
  451. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  452. data/test/plugin/test_output_as_buffered_compress.rb +179 -0
  453. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  454. data/test/plugin/test_output_as_buffered_retries.rb +966 -0
  455. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  456. data/test/plugin/test_output_as_standard.rb +374 -0
  457. data/test/plugin/test_owned_by.rb +34 -0
  458. data/test/plugin/test_parser.rb +399 -0
  459. data/test/plugin/test_parser_apache.rb +42 -0
  460. data/test/plugin/test_parser_apache2.rb +47 -0
  461. data/test/plugin/test_parser_apache_error.rb +45 -0
  462. data/test/plugin/test_parser_csv.rb +200 -0
  463. data/test/plugin/test_parser_json.rb +138 -0
  464. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  465. data/test/plugin/test_parser_multiline.rb +111 -0
  466. data/test/plugin/test_parser_nginx.rb +88 -0
  467. data/test/plugin/test_parser_none.rb +52 -0
  468. data/test/plugin/test_parser_regexp.rb +284 -0
  469. data/test/plugin/test_parser_syslog.rb +650 -0
  470. data/test/plugin/test_parser_tsv.rb +122 -0
  471. data/test/plugin/test_sd_file.rb +228 -0
  472. data/test/plugin/test_sd_srv.rb +230 -0
  473. data/test/plugin/test_storage.rb +166 -0
  474. data/test/plugin/test_storage_local.rb +335 -0
  475. data/test/plugin/test_string_util.rb +26 -0
  476. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  478. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  479. data/test/plugin_helper/data/cert/cert.pem +19 -0
  480. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  481. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  482. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  483. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  484. data/test/plugin_helper/data/cert/empty.pem +0 -0
  485. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  486. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  489. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  490. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  493. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  494. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  495. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  496. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  497. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  498. data/test/plugin_helper/http_server/test_app.rb +65 -0
  499. data/test/plugin_helper/http_server/test_route.rb +32 -0
  500. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  501. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  502. data/test/plugin_helper/test_cert_option.rb +25 -0
  503. data/test/plugin_helper/test_child_process.rb +862 -0
  504. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  505. data/test/plugin_helper/test_event_emitter.rb +80 -0
  506. data/test/plugin_helper/test_event_loop.rb +52 -0
  507. data/test/plugin_helper/test_extract.rb +194 -0
  508. data/test/plugin_helper/test_formatter.rb +255 -0
  509. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  510. data/test/plugin_helper/test_inject.rb +561 -0
  511. data/test/plugin_helper/test_metrics.rb +137 -0
  512. data/test/plugin_helper/test_parser.rb +264 -0
  513. data/test/plugin_helper/test_record_accessor.rb +238 -0
  514. data/test/plugin_helper/test_retry_state.rb +1006 -0
  515. data/test/plugin_helper/test_server.rb +1895 -0
  516. data/test/plugin_helper/test_service_discovery.rb +165 -0
  517. data/test/plugin_helper/test_socket.rb +146 -0
  518. data/test/plugin_helper/test_storage.rb +542 -0
  519. data/test/plugin_helper/test_thread.rb +164 -0
  520. data/test/plugin_helper/test_timer.rb +130 -0
  521. data/test/scripts/exec_script.rb +32 -0
  522. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  523. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  524. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  525. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  526. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  527. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  528. data/test/test_capability.rb +74 -0
  529. data/test/test_clock.rb +164 -0
  530. data/test/test_config.rb +369 -0
  531. data/test/test_configdsl.rb +148 -0
  532. data/test/test_daemonizer.rb +91 -0
  533. data/test/test_engine.rb +203 -0
  534. data/test/test_event.rb +531 -0
  535. data/test/test_event_router.rb +348 -0
  536. data/test/test_event_time.rb +199 -0
  537. data/test/test_file_wrapper.rb +53 -0
  538. data/test/test_filter.rb +121 -0
  539. data/test/test_fluent_log_event_router.rb +99 -0
  540. data/test/test_formatter.rb +369 -0
  541. data/test/test_input.rb +31 -0
  542. data/test/test_log.rb +1076 -0
  543. data/test/test_match.rb +148 -0
  544. data/test/test_mixin.rb +351 -0
  545. data/test/test_msgpack_factory.rb +50 -0
  546. data/test/test_oj_options.rb +55 -0
  547. data/test/test_output.rb +278 -0
  548. data/test/test_plugin.rb +251 -0
  549. data/test/test_plugin_classes.rb +370 -0
  550. data/test/test_plugin_helper.rb +81 -0
  551. data/test/test_plugin_id.rb +119 -0
  552. data/test/test_process.rb +14 -0
  553. data/test/test_root_agent.rb +951 -0
  554. data/test/test_static_config_analysis.rb +177 -0
  555. data/test/test_supervisor.rb +821 -0
  556. data/test/test_test_drivers.rb +136 -0
  557. data/test/test_time_formatter.rb +301 -0
  558. data/test/test_time_parser.rb +362 -0
  559. data/test/test_tls.rb +65 -0
  560. data/test/test_unique_id.rb +47 -0
  561. data/test/test_variable_store.rb +65 -0
  562. metadata +1183 -0
@@ -0,0 +1,93 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'set'
18
+ require 'fluent/env'
19
+ require 'fluent/variable_store'
20
+
21
+ module Fluent
22
+ module PluginId
23
+
24
+ def initialize
25
+ super
26
+
27
+ @_plugin_id_variable_store = nil
28
+ @_plugin_root_dir = nil
29
+ @id = nil
30
+ end
31
+
32
+ def configure(conf)
33
+ @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:pluing_id, default_value: Set.new)
34
+ @id = conf['@id']
35
+ @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
36
+ if @id
37
+ @id = @id.to_s
38
+ if @_plugin_id_variable_store.include?(@id) && !plugin_id_for_test?
39
+ raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
40
+ end
41
+ @_plugin_id_variable_store.add(@id)
42
+ end
43
+
44
+ super
45
+ end
46
+
47
+ def plugin_id_for_test?
48
+ caller_locations.each do |location|
49
+ # Thread::Backtrace::Location#path returns base filename or absolute path.
50
+ # #absolute_path returns absolute_path always.
51
+ # https://bugs.ruby-lang.org/issues/12159
52
+ if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
53
+ return true
54
+ end
55
+ end
56
+ false
57
+ end
58
+
59
+ def plugin_id_configured?
60
+ if instance_variable_defined?("@_id_configured")
61
+ @_id_configured
62
+ end
63
+ end
64
+
65
+ def plugin_id
66
+ if instance_variable_defined?("@id")
67
+ @id || "object:#{object_id.to_s(16)}"
68
+ else
69
+ "object:#{object_id.to_s(16)}"
70
+ end
71
+ end
72
+
73
+ def plugin_root_dir
74
+ return @_plugin_root_dir if @_plugin_root_dir
75
+ return nil unless system_config.root_dir
76
+ return nil unless plugin_id_configured?
77
+
78
+ # Fluent::Plugin::Base#fluentd_worker_id
79
+ dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
80
+ FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
81
+ @_plugin_root_dir = dir.freeze
82
+ dir
83
+ end
84
+
85
+ def stop
86
+ if @_plugin_id_variable_store
87
+ @_plugin_id_variable_store.delete(@id)
88
+ end
89
+
90
+ super
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,22 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/compat/detach_process_mixin'
18
+
19
+ module Fluent
20
+ DetachProcessMixin = Fluent::Compat::DetachProcessMixin
21
+ DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
22
+ end
@@ -0,0 +1,117 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/error'
18
+
19
+ module Fluent
20
+ class Registry
21
+ DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
22
+ FLUENT_LIB_PATH = File.dirname(File.dirname(DEFAULT_PLUGIN_PATH))
23
+
24
+ def initialize(kind, search_prefix, dir_search_prefix: nil)
25
+ @kind = kind
26
+ @search_prefix = search_prefix
27
+ @dir_search_prefix = dir_search_prefix
28
+ @map = {}
29
+ @paths = []
30
+ end
31
+
32
+ attr_reader :kind, :paths, :map, :dir_search_prefix
33
+
34
+ def register(type, value)
35
+ type = type.to_sym
36
+ @map[type] = value
37
+ end
38
+
39
+ def lookup(type)
40
+ type = type.to_sym
41
+ if value = @map[type]
42
+ return value
43
+ end
44
+ search(type)
45
+ if value = @map[type]
46
+ return value
47
+ end
48
+ raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
49
+ kind: @kind, type: type)
50
+ end
51
+
52
+ def reverse_lookup(value)
53
+ @map.each do |k, v|
54
+ return k if v == value
55
+ end
56
+ nil
57
+ end
58
+
59
+ def search(type)
60
+ # search from additional plugin directories
61
+ if @dir_search_prefix
62
+ path = "#{@dir_search_prefix}#{type}"
63
+ files = @paths.filter_map { |lp|
64
+ lpath = File.expand_path(File.join(lp, "#{path}.rb"))
65
+ File.exist?(lpath) ? lpath : nil
66
+ }
67
+ unless files.empty?
68
+ # prefer newer version
69
+ require files.max
70
+ return
71
+ end
72
+ end
73
+
74
+ path = "#{@search_prefix}#{type}"
75
+
76
+ # prefer LOAD_PATH than gems
77
+ files = $LOAD_PATH.filter_map { |lp|
78
+ if lp == FLUENT_LIB_PATH
79
+ nil
80
+ else
81
+ lpath = File.expand_path(File.join(lp, "#{path}.rb"))
82
+ File.exist?(lpath) ? lpath : nil
83
+ end
84
+ }
85
+ unless files.empty?
86
+ # prefer newer version
87
+ require files.max
88
+ return
89
+ end
90
+
91
+ # Find from gems and prefer newer version
92
+ specs = Gem::Specification.find_all { |spec|
93
+ if spec.name == 'fluentd'.freeze
94
+ false
95
+ else
96
+ spec.contains_requirable_file? path
97
+ end
98
+ }.sort_by { |spec| spec.version }
99
+ if spec = specs.last
100
+ spec.require_paths.each { |lib|
101
+ file = "#{spec.full_gem_path}/#{lib}/#{path}"
102
+ if File.exist?("#{file}.rb")
103
+ require file
104
+ return
105
+ end
106
+ }
107
+ end
108
+
109
+ # Lastly, load built-in plugin
110
+ lpath = File.expand_path(File.join(FLUENT_LIB_PATH, "#{@search_prefix}#{type}.rb"))
111
+ if File.exist?(lpath)
112
+ require lpath
113
+ return
114
+ end
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,372 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'delegate'
18
+
19
+ require 'fluent/config/error'
20
+ require 'fluent/agent'
21
+ require 'fluent/label'
22
+ require 'fluent/plugin'
23
+ require 'fluent/system_config'
24
+ require 'fluent/time'
25
+
26
+ module Fluent
27
+ #
28
+ # Fluentd forms a tree structure to manage plugins:
29
+ #
30
+ # RootAgent
31
+ # |
32
+ # +------------+-------------+-------------+
33
+ # | | | |
34
+ # <label> <source> <filter> <match>
35
+ # |
36
+ # +----+----+
37
+ # | |
38
+ # <filter> <match>
39
+ #
40
+ # Relation:
41
+ # * RootAgent has many <label>, <source>, <filter> and <match>
42
+ # * <label> has many <match> and <filter>
43
+ #
44
+ # Next step: `fluentd/agent.rb`
45
+ # Next step: 'fluentd/label.rb'
46
+ #
47
+ class RootAgent < Agent
48
+ ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
49
+
50
+ def initialize(log:, system_config: SystemConfig.new)
51
+ super(log: log)
52
+
53
+ @labels = {}
54
+ @inputs = []
55
+ @suppress_emit_error_log_interval = 0
56
+ @next_emit_error_log_time = nil
57
+ @without_source = false
58
+ @enable_input_metrics = false
59
+
60
+ suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
61
+ @without_source = system_config.without_source unless system_config.without_source.nil?
62
+ @enable_input_metrics = !!system_config.enable_input_metrics
63
+ end
64
+
65
+ attr_reader :inputs
66
+ attr_reader :labels
67
+
68
+ def configure(conf)
69
+ used_worker_ids = []
70
+ available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
71
+ # initialize <worker> elements
72
+ conf.elements(name: 'worker').each do |e|
73
+ target_worker_id_str = e.arg
74
+ if target_worker_id_str.empty?
75
+ raise Fluent::ConfigError, "Missing worker id on <worker> directive"
76
+ end
77
+
78
+ target_worker_ids = target_worker_id_str.split("-")
79
+ if target_worker_ids.size == 2
80
+ first_worker_id = target_worker_ids.first.to_i
81
+ last_worker_id = target_worker_ids.last.to_i
82
+ if first_worker_id > last_worker_id
83
+ raise Fluent::ConfigError, "greater first_worker_id<#{first_worker_id}> than last_worker_id<#{last_worker_id}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
84
+ end
85
+ target_worker_ids = []
86
+ first_worker_id.step(last_worker_id, 1) do |worker_id|
87
+ target_worker_id = worker_id.to_i
88
+ target_worker_ids << target_worker_id
89
+
90
+ if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
91
+ raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
92
+ end
93
+ available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
94
+ if used_worker_ids.include?(target_worker_id)
95
+ raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
96
+ end
97
+ used_worker_ids << target_worker_id
98
+
99
+ e.elements.each do |elem|
100
+ unless ['source', 'match', 'filter', 'label'].include?(elem.name)
101
+ raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
102
+ end
103
+ end
104
+
105
+ unless target_worker_ids.empty?
106
+ e.set_target_worker_ids(target_worker_ids.uniq)
107
+ end
108
+ end
109
+ else
110
+ target_worker_id = target_worker_id_str.to_i
111
+ if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
112
+ raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
113
+ end
114
+
115
+ e.elements.each do |elem|
116
+ unless ['source', 'match', 'filter', 'label'].include?(elem.name)
117
+ raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
118
+ end
119
+ elem.set_target_worker_id(target_worker_id)
120
+ end
121
+ end
122
+ conf += e
123
+ end
124
+ conf.elements.delete_if{|e| e.name == 'worker'}
125
+
126
+ error_label_config = nil
127
+
128
+ # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
129
+ label_configs = {}
130
+ conf.elements(name: 'label').each { |e|
131
+ if !Fluent::Engine.supervisor_mode && e.for_another_worker?
132
+ next
133
+ end
134
+ name = e.arg
135
+ raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
136
+ raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
137
+
138
+ if name == ERROR_LABEL
139
+ error_label_config = e
140
+ else
141
+ add_label(name)
142
+ label_configs[name] = e
143
+ end
144
+ }
145
+ # Call 'configure' here to avoid 'label not found'
146
+ label_configs.each { |name, e| @labels[name].configure(e) }
147
+ setup_error_label(error_label_config) if error_label_config
148
+
149
+ super
150
+
151
+ # initialize <source> elements
152
+ if @without_source
153
+ log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
154
+ else
155
+ conf.elements(name: 'source').each { |e|
156
+ if !Fluent::Engine.supervisor_mode && e.for_another_worker?
157
+ next
158
+ end
159
+ type = e['@type']
160
+ raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
161
+ add_source(type, e)
162
+ }
163
+ end
164
+ end
165
+
166
+ def setup_error_label(e)
167
+ error_label = add_label(ERROR_LABEL)
168
+ error_label.configure(e)
169
+ @error_collector = error_label.event_router
170
+ end
171
+
172
+ def lifecycle(desc: false, kind_callback: nil)
173
+ kind_or_label_list = if desc
174
+ [:output, :filter, @labels.values.reverse, :output_with_router, :input].flatten
175
+ else
176
+ [:input, :output_with_router, @labels.values, :filter, :output].flatten
177
+ end
178
+ kind_or_label_list.each do |kind|
179
+ if kind.respond_to?(:lifecycle)
180
+ label = kind
181
+ label.lifecycle(desc: desc) do |plugin, display_kind|
182
+ yield plugin, display_kind
183
+ end
184
+ else
185
+ list = if desc
186
+ lifecycle_control_list[kind].reverse
187
+ else
188
+ lifecycle_control_list[kind]
189
+ end
190
+ display_kind = (kind == :output_with_router ? :output : kind)
191
+ list.each do |instance|
192
+ yield instance, display_kind
193
+ end
194
+ end
195
+ if kind_callback
196
+ kind_callback.call
197
+ end
198
+ end
199
+ end
200
+
201
+ def start
202
+ lifecycle(desc: true) do |i| # instance
203
+ i.start unless i.started?
204
+ # Input#start sometimes emits lots of events with in_tail/`read_from_head true` case
205
+ # and it causes deadlock for small buffer/queue output. To avoid such problem,
206
+ # buffer related output threads should be run before `Input#start`.
207
+ # This is why after_start should be called immediately after start call.
208
+ # This depends on `desc: true` because calling plugin order of `desc: true` is
209
+ # Output, Filter, Label, Output with Router, then Input.
210
+ i.after_start unless i.after_started?
211
+ end
212
+ end
213
+
214
+ def flush!
215
+ log.info "flushing all buffer forcedly"
216
+ flushing_threads = []
217
+ lifecycle(desc: true) do |instance|
218
+ if instance.respond_to?(:force_flush)
219
+ t = Thread.new do
220
+ Thread.current.abort_on_exception = true
221
+ begin
222
+ instance.force_flush
223
+ rescue => e
224
+ log.warn "unexpected error while flushing buffer", plugin: instance.class, plugin_id: instance.plugin_id, error: e
225
+ log.warn_backtrace
226
+ end
227
+ end
228
+ flushing_threads << t
229
+ end
230
+ end
231
+ flushing_threads.each{|t| t.join }
232
+ end
233
+
234
+ def shutdown # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
235
+ # These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
236
+ # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
237
+ # Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
238
+
239
+ lifecycle_safe_sequence = ->(method, checker) {
240
+ lifecycle do |instance, kind|
241
+ begin
242
+ log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
243
+ instance.__send__(method) unless instance.__send__(checker)
244
+ rescue Exception => e
245
+ log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
246
+ log.warn_backtrace
247
+ end
248
+ end
249
+ }
250
+
251
+ lifecycle_unsafe_sequence = ->(method, checker) {
252
+ operation = case method
253
+ when :shutdown then "shutting down"
254
+ when :close then "closing"
255
+ else
256
+ raise "BUG: unknown method name '#{method}'"
257
+ end
258
+ operation_threads = []
259
+ callback = ->(){
260
+ operation_threads.each{|t| t.join }
261
+ operation_threads.clear
262
+ }
263
+ lifecycle(kind_callback: callback) do |instance, kind|
264
+ t = Thread.new do
265
+ Thread.current.abort_on_exception = true
266
+ begin
267
+ if method == :shutdown
268
+ # To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.
269
+ # The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown
270
+ operation = "preparing shutdown" # for logging
271
+ log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
272
+ begin
273
+ instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)
274
+ rescue Exception => e
275
+ log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
276
+ log.warn_backtrace
277
+ end
278
+ operation = "shutting down"
279
+ log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
280
+ instance.__send__(:shutdown) unless instance.__send__(:shutdown?)
281
+ else
282
+ log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
283
+ instance.__send__(method) unless instance.__send__(checker)
284
+ end
285
+ rescue Exception => e
286
+ log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
287
+ log.warn_backtrace
288
+ end
289
+ end
290
+ operation_threads << t
291
+ end
292
+ }
293
+
294
+ lifecycle_safe_sequence.call(:stop, :stopped?)
295
+
296
+ # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
297
+ lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
298
+
299
+ lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
300
+
301
+ lifecycle_unsafe_sequence.call(:close, :closed?)
302
+
303
+ lifecycle_safe_sequence.call(:terminate, :terminated?)
304
+ end
305
+
306
+ def suppress_interval(interval_time)
307
+ @suppress_emit_error_log_interval = interval_time
308
+ @next_emit_error_log_time = Time.now.to_i
309
+ end
310
+
311
+ def add_source(type, conf)
312
+ log_type = conf.for_this_worker? ? :default : :worker0
313
+ log.info log_type, "adding source", type: type
314
+
315
+ input = Plugin.new_input(type)
316
+ # <source> emits events to the top-level event router (RootAgent#event_router).
317
+ # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
318
+ # See also 'fluentd/plugin/input.rb'
319
+ input.context_router = @event_router
320
+ input.configure(conf)
321
+ if @enable_input_metrics
322
+ @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
323
+ end
324
+ @inputs << input
325
+
326
+ input
327
+ end
328
+
329
+ def add_label(name)
330
+ label = Label.new(name, log: log)
331
+ raise ConfigError, "Section <label #{name}> appears twice" if @labels[name]
332
+ label.root_agent = self
333
+ @labels[name] = label
334
+ end
335
+
336
+ def find_label(label_name)
337
+ if label = @labels[label_name]
338
+ label
339
+ else
340
+ raise ArgumentError, "#{label_name} label not found"
341
+ end
342
+ end
343
+
344
+ def emit_error_event(tag, time, record, error)
345
+ error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
346
+ if @error_collector
347
+ # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
348
+ log.warn "send an error event to @ERROR:", error_info
349
+ @error_collector.emit(tag, time, record)
350
+ else
351
+ error_info[:record] = record
352
+ log.warn "dump an error event:", error_info
353
+ end
354
+ end
355
+
356
+ def handle_emits_error(tag, es, error)
357
+ error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
358
+ if @error_collector
359
+ log.warn "send an error event stream to @ERROR:", error_info
360
+ @error_collector.emit_stream(tag, es)
361
+ else
362
+ now = Time.now.to_i
363
+ if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
364
+ log.warn "emit transaction failed:", error_info
365
+ log.warn_backtrace
366
+ @next_emit_error_log_time = now + @suppress_emit_error_log_interval
367
+ end
368
+ raise error
369
+ end
370
+ end
371
+ end
372
+ end
data/lib/fluent/rpc.rb ADDED
@@ -0,0 +1,95 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'webrick'
18
+
19
+ module Fluent
20
+ module RPC
21
+ class Server
22
+ def initialize(endpoint, log)
23
+ m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
24
+ raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
25
+ @bind = m[:host]
26
+ @port = m[:port]
27
+ @log = log
28
+
29
+ @server = WEBrick::HTTPServer.new(
30
+ BindAddress: @bind,
31
+ Port: @port,
32
+ Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
33
+ AccessLog: [],
34
+ )
35
+ end
36
+
37
+ def mount(path, servlet, *args)
38
+ @server.mount(path, servlet, *args)
39
+ @log.debug "register #{path} RPC servlet"
40
+ end
41
+
42
+ def mount_proc(path, &block)
43
+ @server.mount_proc(path) { |req, res|
44
+ begin
45
+ code, header, response = block.call(req, res)
46
+ rescue => e
47
+ @log.warn "failed to handle RPC request", path: path, error: e.to_s
48
+ @log.warn_backtrace e.backtrace
49
+
50
+ code = 500
51
+ body = {
52
+ 'message '=> 'Internal Server Error',
53
+ 'error' => "#{e}",
54
+ 'backtrace'=> e.backtrace,
55
+ }
56
+ end
57
+
58
+ code = 200 if code.nil?
59
+ header = {'Content-Type' => 'application/json'} if header.nil?
60
+ body = if response.nil?
61
+ '{"ok":true}'
62
+ else
63
+ response.body['ok'] = code == 200
64
+ response.body.to_json
65
+ end
66
+
67
+ res.status = code
68
+ header.each_pair { |k, v|
69
+ res[k] = v
70
+ }
71
+ res.body = body
72
+ }
73
+ @log.debug "register #{path} RPC handler"
74
+ end
75
+
76
+ def start
77
+ @log.debug "listening RPC http server on http://#{@bind}:#{@port}/"
78
+ @thread = Thread.new {
79
+ @server.start
80
+ }
81
+ end
82
+
83
+ def shutdown
84
+ if @server
85
+ @server.shutdown
86
+ @server = nil
87
+ end
88
+ if @thread
89
+ @thread.join
90
+ @thread = nil
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end