fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,412 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'json'
18
+ require 'webrick'
19
+ require 'cgi'
20
+
21
+ require 'fluent/config/types'
22
+ require 'fluent/plugin/input'
23
+ require 'fluent/plugin/output'
24
+ require 'fluent/plugin/multi_output'
25
+ require 'fluent/plugin/filter'
26
+
27
+ module Fluent::Plugin
28
+ class MonitorAgentInput < Input
29
+ Fluent::Plugin.register_input('monitor_agent', self)
30
+
31
+ helpers :timer, :thread, :http_server
32
+
33
+ desc 'The address to bind to.'
34
+ config_param :bind, :string, default: '0.0.0.0'
35
+ desc 'The port to listen to.'
36
+ config_param :port, :integer, default: 24220
37
+ desc 'The tag with which internal metrics are emitted.'
38
+ config_param :tag, :string, default: nil
39
+ desc 'Determine the rate to emit internal metrics as events.'
40
+ config_param :emit_interval, :time, default: 60
41
+ desc 'Determine whether to include the config information.'
42
+ config_param :include_config, :bool, default: true
43
+ desc 'Determine whether to include the retry information.'
44
+ config_param :include_retry, :bool, default: true
45
+
46
+ class APIHandler
47
+ def initialize(agent)
48
+ @agent = agent
49
+ end
50
+
51
+ def plugins_ltsv(req)
52
+ list = build_object(build_option(req))
53
+
54
+ render_ltsv(list)
55
+ end
56
+
57
+ def plugins_json(req)
58
+ opts = build_option(req)
59
+ obj = build_object(opts)
60
+
61
+ render_json({ 'plugins' => obj }, pretty_json: opts[:pretty_json])
62
+ end
63
+
64
+ def config_ltsv(_req)
65
+ obj = {
66
+ 'pid' => Process.pid,
67
+ 'ppid' => Process.ppid,
68
+ 'version' => Fluent::VERSION,
69
+ }.merge(@agent.fluentd_opts)
70
+
71
+ render_ltsv([obj])
72
+ end
73
+
74
+ def config_json(req)
75
+ obj = {
76
+ 'pid' => Process.pid,
77
+ 'ppid' => Process.ppid,
78
+ 'version' => Fluent::VERSION,
79
+ }.merge(@agent.fluentd_opts)
80
+ opts = build_option(req)
81
+
82
+ render_json(obj, pretty_json: opts[:pretty_json])
83
+ end
84
+
85
+ private
86
+
87
+ def render_error_json(code:, msg:, pretty_json: nil, **additional_params)
88
+ resp = additional_params.merge('message' => msg)
89
+ render_json(resp, code: code, pretty_json: pretty_json)
90
+ end
91
+
92
+ def render_json(obj, code: 200, pretty_json: nil)
93
+ body =
94
+ if pretty_json
95
+ JSON.pretty_generate(obj)
96
+ else
97
+ obj.to_json
98
+ end
99
+
100
+ [code, { 'Content-Type' => 'application/json' }, body]
101
+ end
102
+
103
+ def render_ltsv(obj, code: 200)
104
+ normalized = JSON.parse(obj.to_json)
105
+ text = ''
106
+ normalized.each do |hash|
107
+ row = []
108
+ hash.each do |k, v|
109
+ if v.is_a?(Array)
110
+ row << "#{k}:#{v.join(',')}"
111
+ elsif v.is_a?(Hash)
112
+ next
113
+ else
114
+ row << "#{k}:#{v}"
115
+ end
116
+ end
117
+
118
+ text << row.join("\t") << "\n"
119
+ end
120
+
121
+ [code, { 'Content-Type' => 'text/plain' }, text]
122
+ end
123
+
124
+ def build_object(opts)
125
+ qs = opts[:query]
126
+ if tag = qs['tag'.freeze].first
127
+ # ?tag= to search an output plugin by match pattern
128
+ if obj = @agent.plugin_info_by_tag(tag, opts)
129
+ list = [obj]
130
+ else
131
+ list = []
132
+ end
133
+ elsif plugin_id = (qs['@id'.freeze].first || qs['id'.freeze].first)
134
+ # ?@id= to search a plugin by 'id <plugin_id>' config param
135
+ if obj = @agent.plugin_info_by_id(plugin_id, opts)
136
+ list = [obj]
137
+ else
138
+ list = []
139
+ end
140
+ elsif plugin_type = (qs['@type'.freeze].first || qs['type'.freeze].first)
141
+ # ?@type= to search plugins by 'type <type>' config param
142
+ list = @agent.plugins_info_by_type(plugin_type, opts)
143
+ else
144
+ # otherwise show all plugins
145
+ list = @agent.plugins_info_all(opts)
146
+ end
147
+
148
+ list
149
+ end
150
+
151
+ def build_option(req)
152
+ qs = Hash.new { |_, _| [] }
153
+ # parse ?=query string
154
+ if req.query_string
155
+ qs.merge!(CGI.parse(req.query_string))
156
+ end
157
+
158
+ # if ?debug=1 is set, set :with_debug_info for get_monitor_info
159
+ # and :pretty_json for render_json_error
160
+ opts = { query: qs }
161
+ if qs['debug'.freeze].first
162
+ opts[:with_debug_info] = true
163
+ opts[:pretty_json] = true
164
+ end
165
+
166
+ if ivars = qs['with_ivars'.freeze].first
167
+ opts[:ivars] = ivars.split(',')
168
+ end
169
+
170
+ if with_config = qs['with_config'.freeze].first
171
+ opts[:with_config] = Fluent::Config.bool_value(with_config)
172
+ else
173
+ opts[:with_config] = @agent.include_config
174
+ end
175
+
176
+ if with_retry = qs['with_retry'.freeze].first
177
+ opts[:with_retry] = Fluent::Config.bool_value(with_retry)
178
+ else
179
+ opts[:with_retry] = @agent.include_retry
180
+ end
181
+
182
+ opts
183
+ end
184
+ end
185
+
186
+ def initialize
187
+ super
188
+
189
+ @first_warn = false
190
+ end
191
+
192
+ def configure(conf)
193
+ super
194
+ @port += fluentd_worker_id
195
+ end
196
+
197
+ def multi_workers_ready?
198
+ true
199
+ end
200
+
201
+ class NotFoundJson
202
+ BODY = { 'message' => 'Not found' }.to_json
203
+ def self.call(_req)
204
+ [404, { 'Content-Type' => 'application/json' }, BODY]
205
+ end
206
+ end
207
+
208
+ def start
209
+ super
210
+
211
+ log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
212
+ api_handler = APIHandler.new(self)
213
+ http_server_create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
214
+ serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
215
+ serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }
216
+ serv.get('/api/config') { |req| api_handler.config_ltsv(req) }
217
+ serv.get('/api/config.json') { |req| api_handler.config_json(req) }
218
+ end
219
+
220
+ if @tag
221
+ log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
222
+
223
+ opts = {with_config: false, with_retry: false}
224
+ timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
225
+ es = Fluent::MultiEventStream.new
226
+ now = Fluent::EventTime.now
227
+ plugins_info_all(opts).each { |record|
228
+ es.add(now, record)
229
+ }
230
+ router.emit_stream(@tag, es)
231
+ }
232
+ end
233
+ end
234
+
235
+ # They are deprecated but remain for compatibility
236
+ MONITOR_INFO = {
237
+ 'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
238
+ 'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
239
+ 'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
240
+ 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
241
+ 'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },
242
+ }
243
+
244
+ def all_plugins
245
+ array = []
246
+
247
+ # get all input plugins
248
+ array.concat Fluent::Engine.root_agent.inputs
249
+
250
+ # get all output plugins
251
+ array.concat Fluent::Engine.root_agent.outputs
252
+
253
+ # get all filter plugins
254
+ array.concat Fluent::Engine.root_agent.filters
255
+
256
+ Fluent::Engine.root_agent.labels.each { |name, l|
257
+ # TODO: Add label name to outputs / filters for identifing plugins
258
+ array.concat l.outputs
259
+ array.concat l.filters
260
+ }
261
+
262
+ array
263
+ end
264
+
265
+ # try to match the tag and get the info from the matched output plugin
266
+ # TODO: Support output in label
267
+ def plugin_info_by_tag(tag, opts={})
268
+ matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)
269
+ matches.each { |rule|
270
+ if rule.match?(tag)
271
+ if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)
272
+ return get_monitor_info(rule.collector, opts)
273
+ end
274
+ end
275
+ }
276
+ nil
277
+ end
278
+
279
+ # search a plugin by plugin_id
280
+ def plugin_info_by_id(plugin_id, opts={})
281
+ found = all_plugins.find {|pe|
282
+ pe.respond_to?(:plugin_id) && pe.plugin_id.to_s == plugin_id
283
+ }
284
+ if found
285
+ get_monitor_info(found, opts)
286
+ else
287
+ nil
288
+ end
289
+ end
290
+
291
+ # This method returns an array because
292
+ # multiple plugins could have the same type
293
+ def plugins_info_by_type(type, opts={})
294
+ array = all_plugins.select {|pe|
295
+ (pe.config['@type'] == type) rescue nil
296
+ }
297
+ array.map {|pe|
298
+ get_monitor_info(pe, opts)
299
+ }
300
+ end
301
+
302
+ def plugins_info_all(opts={})
303
+ all_plugins.map {|pe|
304
+ get_monitor_info(pe, opts)
305
+ }
306
+ end
307
+
308
+ IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)
309
+
310
+ # get monitor info from the plugin `pe` and return a hash object
311
+ def get_monitor_info(pe, opts={})
312
+ obj = {}
313
+
314
+ # Common plugin information
315
+ obj['plugin_id'] = pe.plugin_id
316
+ obj['plugin_category'] = plugin_category(pe)
317
+ obj['type'] = pe.config['@type']
318
+ obj['config'] = pe.config if opts[:with_config]
319
+
320
+ # run MONITOR_INFO in plugins' instance context and store the info to obj
321
+ MONITOR_INFO.each_pair {|key,code|
322
+ begin
323
+ catch(:skip) do
324
+ obj[key] = pe.instance_exec(&code)
325
+ end
326
+ rescue NoMethodError => e
327
+ unless @first_warn
328
+ log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
329
+ log.error_backtrace
330
+ @first_warn = true
331
+ end
332
+ rescue => e
333
+ log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
334
+ end
335
+ }
336
+
337
+ if pe.respond_to?(:statistics)
338
+ obj.merge!(pe.statistics.dig('output') || {})
339
+ obj.merge!(pe.statistics.dig('filter') || {})
340
+ obj.merge!(pe.statistics.dig('input') || {})
341
+ end
342
+
343
+ obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
344
+
345
+ # include all instance variables if :with_debug_info is set
346
+ if opts[:with_debug_info]
347
+ iv = {}
348
+ pe.instance_eval do
349
+ instance_variables.each {|sym|
350
+ next if IGNORE_ATTRIBUTES.include?(sym)
351
+ key = sym.to_s[1..-1] # removes first '@'
352
+ iv[key] = instance_variable_get(sym)
353
+ }
354
+ end
355
+ obj['instance_variables'] = iv
356
+ elsif ivars = opts[:ivars]
357
+ iv = {}
358
+ ivars.each {|name|
359
+ iname = "@#{name}"
360
+ iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
361
+ }
362
+ obj['instance_variables'] = iv
363
+ end
364
+
365
+ obj
366
+ end
367
+
368
+ RETRY_INFO = {
369
+ 'start' => '@start',
370
+ 'steps' => '@steps',
371
+ 'next_time' => '@next_time',
372
+ }
373
+
374
+ def get_retry_info(pe_retry)
375
+ retry_variables = {}
376
+
377
+ if pe_retry
378
+ RETRY_INFO.each_pair { |key, param|
379
+ retry_variables[key] = pe_retry.instance_variable_get(param)
380
+ }
381
+ end
382
+
383
+ retry_variables
384
+ end
385
+
386
+ def plugin_category(pe)
387
+ case pe
388
+ when Fluent::Plugin::Input
389
+ 'input'.freeze
390
+ when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
391
+ 'output'.freeze
392
+ when Fluent::Plugin::Filter
393
+ 'filter'.freeze
394
+ else
395
+ 'unknown'.freeze
396
+ end
397
+ end
398
+
399
+ def fluentd_opts
400
+ @fluentd_opts ||= get_fluentd_opts
401
+ end
402
+
403
+ def get_fluentd_opts
404
+ opts = {}
405
+ ObjectSpace.each_object(Fluent::Supervisor) { |obj|
406
+ opts.merge!(obj.options)
407
+ break
408
+ }
409
+ opts
410
+ end
411
+ end
412
+ end
@@ -0,0 +1,93 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'yajl'
18
+
19
+ require 'fluent/plugin/input'
20
+
21
+ module Fluent::Plugin
22
+ class ObjectSpaceInput < Fluent::Plugin::Input
23
+ Fluent::Plugin.register_input('object_space', self)
24
+
25
+ helpers :timer
26
+
27
+ def initialize
28
+ super
29
+ end
30
+
31
+ config_param :emit_interval, :time, default: 60
32
+ config_param :tag, :string
33
+ config_param :top, :integer, default: 15
34
+
35
+ def multi_workers_ready?
36
+ true
37
+ end
38
+
39
+ def start
40
+ super
41
+
42
+ timer_execute(:object_space_input, @emit_interval, &method(:on_timer))
43
+ end
44
+
45
+ class Counter
46
+ def initialize(klass, init_count)
47
+ @klass = klass
48
+ @count = init_count
49
+ end
50
+
51
+ def incr!
52
+ @count += 1
53
+ end
54
+
55
+ def name
56
+ @klass.name
57
+ end
58
+
59
+ attr_reader :count
60
+ end
61
+
62
+ def on_timer
63
+ now = Fluent::EventTime.now
64
+
65
+ array = []
66
+ map = {}
67
+
68
+ ObjectSpace.each_object {|obj|
69
+ klass = obj.class rescue Object
70
+ if c = map[klass]
71
+ c.incr!
72
+ else
73
+ c = Counter.new(klass, 1)
74
+ array << c
75
+ map[klass] = c
76
+ end
77
+ }
78
+
79
+ array.sort_by! {|c| -c.count }
80
+
81
+ record = {}
82
+ array.each_with_index {|c,i|
83
+ break if i >= @top
84
+ record[c.name] = c.count
85
+ }
86
+
87
+ router.emit(@tag, now, record)
88
+ rescue => e
89
+ log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
90
+ log.error_backtrace
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,141 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'json'
18
+
19
+ require 'fluent/plugin/input'
20
+ require 'fluent/config/error'
21
+
22
+ module Fluent::Plugin
23
+ class SampleInput < Input
24
+ Fluent::Plugin.register_input('sample', self)
25
+ Fluent::Plugin.register_input('dummy', self)
26
+
27
+ helpers :thread, :storage
28
+
29
+ BIN_NUM = 10
30
+ DEFAULT_STORAGE_TYPE = 'local'
31
+
32
+ desc "The value is the tag assigned to the generated events."
33
+ config_param :tag, :string
34
+ desc "The number of events in event stream of each emits."
35
+ config_param :size, :integer, default: 1
36
+ desc "It configures how many events to generate per second."
37
+ config_param :rate, :integer, default: 1
38
+ desc "If specified, each generated event has an auto-incremented key field."
39
+ config_param :auto_increment_key, :string, default: nil
40
+ desc "The boolean to suspend-and-resume incremental value after restart"
41
+ config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
+ config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
+ begin
45
+ parsed = JSON.parse(val)
46
+ rescue JSON::ParserError => ex
47
+ # Fluent::ConfigParseError, "got incomplete JSON" will be raised
48
+ # at literal_parser.rb with --use-v1-config, but I had to
49
+ # take care at here for the case of --use-v0-config.
50
+ raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
51
+ end
52
+ sample = parsed.is_a?(Array) ? parsed : [parsed]
53
+ sample.each_with_index do |e, i|
54
+ raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
55
+ end
56
+ sample
57
+ end
58
+
59
+ def initialize
60
+ super
61
+ @storage = nil
62
+ end
63
+
64
+ def configure(conf)
65
+ super
66
+ @sample_index = 0
67
+ config = conf.elements.select{|e| e.name == 'storage' }.first
68
+ @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
+ end
70
+
71
+ def multi_workers_ready?
72
+ true
73
+ end
74
+
75
+ def start
76
+ super
77
+
78
+ @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
79
+ # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
80
+ @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
81
+
82
+ if @auto_increment_key && !@storage.get(:auto_increment_value)
83
+ @storage.put(:auto_increment_value, -1)
84
+ end
85
+
86
+ thread_create(:sample_input, &method(:run))
87
+ end
88
+
89
+ def run
90
+ batch_num = (@rate / BIN_NUM).to_i
91
+ residual_num = (@rate % BIN_NUM)
92
+ while thread_current_running?
93
+ current_time = Time.now.to_i
94
+ BIN_NUM.times do
95
+ break unless (thread_current_running? && Time.now.to_i <= current_time)
96
+ wait(0.1) { emit(batch_num) }
97
+ end
98
+ emit(residual_num) if thread_current_running?
99
+ # wait for next second
100
+ while thread_current_running? && Time.now.to_i <= current_time
101
+ sleep 0.01
102
+ end
103
+ end
104
+ end
105
+
106
+ def emit(num)
107
+ begin
108
+ if @size > 1
109
+ num.times do
110
+ router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
111
+ end
112
+ else
113
+ num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
114
+ end
115
+ rescue => _
116
+ # ignore all errors not to stop emits by emit errors
117
+ end
118
+ end
119
+
120
+ def generate
121
+ d = @sample[@sample_index]
122
+ unless d
123
+ @sample_index = 0
124
+ d = @sample[@sample_index]
125
+ end
126
+ @sample_index += 1
127
+ if @auto_increment_key
128
+ d = d.dup
129
+ d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
+ end
131
+ d
132
+ end
133
+
134
+ def wait(time)
135
+ start_time = Time.now
136
+ yield
137
+ sleep_time = time - (Time.now - start_time)
138
+ sleep sleep_time if sleep_time > 0
139
+ end
140
+ end
141
+ end