fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,326 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/msgpack_factory'
18
+ require 'fluent/plugin/compressable'
19
+
20
+ module Fluent
21
+ class EventStream
22
+ include Enumerable
23
+ include Fluent::Plugin::Compressable
24
+
25
+ # dup does deep copy for event stream
26
+ def dup
27
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
28
+ end
29
+
30
+ def size
31
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
32
+ end
33
+ alias :length :size
34
+
35
+ def empty?
36
+ size == 0
37
+ end
38
+
39
+ # for tests
40
+ def ==(other)
41
+ other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
42
+ end
43
+
44
+ def repeatable?
45
+ false
46
+ end
47
+
48
+ def slice(index, num)
49
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
50
+ end
51
+
52
+ def each(unapcker: nil, &block)
53
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
54
+ end
55
+
56
+ def to_msgpack_stream(time_int: false, packer: nil)
57
+ return to_msgpack_stream_forced_integer(packer: packer) if time_int
58
+ out = packer || Fluent::MessagePackFactory.msgpack_packer
59
+ each {|time,record|
60
+ out.write([time,record])
61
+ }
62
+ out.full_pack
63
+ end
64
+
65
+ def to_compressed_msgpack_stream(time_int: false, packer: nil)
66
+ packed = to_msgpack_stream(time_int: time_int, packer: packer)
67
+ compress(packed)
68
+ end
69
+
70
+ def to_msgpack_stream_forced_integer(packer: nil)
71
+ out = packer || Fluent::MessagePackFactory.msgpack_packer
72
+ each {|time,record|
73
+ out.write([time.to_i,record])
74
+ }
75
+ out.full_pack
76
+ end
77
+ end
78
+
79
+ class OneEventStream < EventStream
80
+ def initialize(time, record)
81
+ @time = time
82
+ @record = record
83
+ end
84
+
85
+ def dup
86
+ OneEventStream.new(@time, @record.dup)
87
+ end
88
+
89
+ def empty?
90
+ false
91
+ end
92
+
93
+ def size
94
+ 1
95
+ end
96
+
97
+ def repeatable?
98
+ true
99
+ end
100
+
101
+ def slice(index, num)
102
+ if index > 0 || num == 0
103
+ ArrayEventStream.new([])
104
+ else
105
+ self.dup
106
+ end
107
+ end
108
+
109
+ def each(unpacker: nil, &block)
110
+ block.call(@time, @record)
111
+ nil
112
+ end
113
+ end
114
+
115
+ # EventStream from entries: Array of [time, record]
116
+ #
117
+ # Use this class for many events data with a tag
118
+ # and its representation is [ [time, record], [time, record], .. ]
119
+ class ArrayEventStream < EventStream
120
+ def initialize(entries)
121
+ @entries = entries
122
+ end
123
+
124
+ def dup
125
+ entries = @entries.map{ |time, record| [time, record.dup] }
126
+ ArrayEventStream.new(entries)
127
+ end
128
+
129
+ def size
130
+ @entries.size
131
+ end
132
+
133
+ def repeatable?
134
+ true
135
+ end
136
+
137
+ def empty?
138
+ @entries.empty?
139
+ end
140
+
141
+ def slice(index, num)
142
+ ArrayEventStream.new(@entries.slice(index, num))
143
+ end
144
+
145
+ def each(unpacker: nil, &block)
146
+ @entries.each(&block)
147
+ nil
148
+ end
149
+ end
150
+
151
+ # EventStream from entries: numbers of pairs of time and record.
152
+ #
153
+ # This class can handle many events more efficiently than ArrayEventStream
154
+ # because this class generate less objects than ArrayEventStream.
155
+ #
156
+ # Use this class as below, in loop of data-enumeration:
157
+ # 1. initialize blank stream:
158
+ # streams[tag] ||= MultiEventStream.new
159
+ # 2. add events
160
+ # stream[tag].add(time, record)
161
+ class MultiEventStream < EventStream
162
+ def initialize(time_array = [], record_array = [])
163
+ @time_array = time_array
164
+ @record_array = record_array
165
+ end
166
+
167
+ def dup
168
+ MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
169
+ end
170
+
171
+ def size
172
+ @time_array.size
173
+ end
174
+
175
+ def add(time, record)
176
+ @time_array << time
177
+ @record_array << record
178
+ end
179
+
180
+ def repeatable?
181
+ true
182
+ end
183
+
184
+ def empty?
185
+ @time_array.empty?
186
+ end
187
+
188
+ def slice(index, num)
189
+ MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
190
+ end
191
+
192
+ def each(unpacker: nil, &block)
193
+ time_array = @time_array
194
+ record_array = @record_array
195
+ for i in 0..time_array.length-1
196
+ block.call(time_array[i], record_array[i])
197
+ end
198
+ nil
199
+ end
200
+ end
201
+
202
+ class MessagePackEventStream < EventStream
203
+ # https://github.com/msgpack/msgpack-ruby/issues/119
204
+
205
+ # Keep cached_unpacker argument for existing plugins
206
+ def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
207
+ @data = data
208
+ @size = size
209
+ @unpacked_times = unpacked_times
210
+ @unpacked_records = unpacked_records
211
+ end
212
+
213
+ def empty?
214
+ @data.empty?
215
+ end
216
+
217
+ def dup
218
+ if @unpacked_times
219
+ self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
220
+ else
221
+ self.class.new(@data.dup, nil, @size)
222
+ end
223
+ end
224
+
225
+ def size
226
+ # @size is unbelievable always when @size == 0
227
+ # If the number of events is really zero, unpacking events takes very short time.
228
+ ensure_unpacked! if @size == 0
229
+ @size
230
+ end
231
+
232
+ def repeatable?
233
+ true
234
+ end
235
+
236
+ def ensure_unpacked!(unpacker: nil)
237
+ return if @unpacked_times && @unpacked_records
238
+ @unpacked_times = []
239
+ @unpacked_records = []
240
+ (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
241
+ @unpacked_times << time
242
+ @unpacked_records << record
243
+ end
244
+ # @size should be updated always right after unpack.
245
+ # The real size of unpacked objects are correct, rather than given size.
246
+ @size = @unpacked_times.size
247
+ end
248
+
249
+ # This method returns MultiEventStream, because there are no reason
250
+ # to surve binary serialized by msgpack.
251
+ def slice(index, num)
252
+ ensure_unpacked!
253
+ MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
254
+ end
255
+
256
+ def each(unpacker: nil, &block)
257
+ ensure_unpacked!(unpacker: unpacker)
258
+ @unpacked_times.each_with_index do |time, i|
259
+ block.call(time, @unpacked_records[i])
260
+ end
261
+ nil
262
+ end
263
+
264
+ def to_msgpack_stream(time_int: false, packer: nil)
265
+ # time_int is always ignored because @data is always packed binary in this class
266
+ @data
267
+ end
268
+ end
269
+
270
+ class CompressedMessagePackEventStream < MessagePackEventStream
271
+ def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
272
+ super
273
+ @decompressed_data = nil
274
+ @compressed_data = data
275
+ end
276
+
277
+ def empty?
278
+ ensure_decompressed!
279
+ super
280
+ end
281
+
282
+ def ensure_unpacked!(unpacker: nil)
283
+ ensure_decompressed!
284
+ super
285
+ end
286
+
287
+ def each(unpacker: nil, &block)
288
+ ensure_decompressed!
289
+ super
290
+ end
291
+
292
+ def to_msgpack_stream(time_int: false, packer: nil)
293
+ ensure_decompressed!
294
+ super
295
+ end
296
+
297
+ def to_compressed_msgpack_stream(time_int: false)
298
+ # time_int is always ignored because @data is always packed binary in this class
299
+ @compressed_data
300
+ end
301
+
302
+ private
303
+
304
+ def ensure_decompressed!
305
+ return if @decompressed_data
306
+ @data = @decompressed_data = decompress(@data)
307
+ end
308
+ end
309
+
310
+ module ChunkMessagePackEventStreamer
311
+ # chunk.extend(ChunkEventStreamer)
312
+ # => chunk.each{|time, record| ... }
313
+ def each(unpacker: nil, &block)
314
+ open do |io|
315
+ (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
316
+ end
317
+ nil
318
+ end
319
+ alias :msgpack_each :each
320
+
321
+ def to_msgpack_stream(time_int: false, packer: nil)
322
+ # time_int is always ignored because data is already packed and written in chunk
323
+ read
324
+ end
325
+ end
326
+ end
@@ -0,0 +1,297 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/match'
18
+ require 'fluent/event'
19
+ require 'fluent/filter'
20
+ require 'fluent/msgpack_factory'
21
+
22
+ module Fluent
23
+ #
24
+ # EventRouter is responsible to route events to a collector.
25
+ #
26
+ # It has a list of MatchPattern and Collector pairs:
27
+ #
28
+ # +----------------+ +-----------------+
29
+ # | MatchPattern | | Collector |
30
+ # +----------------+ +-----------------+
31
+ # | access.** ---------> type forward |
32
+ # | logs.** ---------> type copy |
33
+ # | archive.** ---------> type s3 |
34
+ # +----------------+ +-----------------+
35
+ #
36
+ # EventRouter does:
37
+ #
38
+ # 1) receive an event at `#emit` methods
39
+ # 2) match the event's tag with the MatchPatterns
40
+ # 3) forward the event to the corresponding Collector
41
+ #
42
+ # Collector is either of Output, Filter or other EventRouter.
43
+ #
44
+ class EventRouter
45
+ def initialize(default_collector, emit_error_handler)
46
+ @match_rules = []
47
+ @match_cache = MatchCache.new
48
+ @default_collector = default_collector
49
+ @emit_error_handler = emit_error_handler
50
+ @metric_callbacks = {}
51
+ @caller_plugin_id = nil
52
+ end
53
+
54
+ attr_accessor :default_collector
55
+ attr_accessor :emit_error_handler
56
+
57
+ class Rule
58
+ def initialize(pattern, collector)
59
+ patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
60
+ @pattern = if patterns.length == 1
61
+ patterns[0]
62
+ else
63
+ OrMatchPattern.new(patterns)
64
+ end
65
+ @pattern_str = pattern
66
+ @collector = collector
67
+ end
68
+
69
+ def match?(tag)
70
+ @pattern.match(tag)
71
+ end
72
+
73
+ attr_reader :collector
74
+ attr_reader :pattern_str
75
+ end
76
+
77
+ def suppress_missing_match!
78
+ if @default_collector.respond_to?(:suppress_missing_match!)
79
+ @default_collector.suppress_missing_match!
80
+ end
81
+ end
82
+
83
+ # called by Agent to add new match pattern and collector
84
+ def add_rule(pattern, collector)
85
+ @match_rules << Rule.new(pattern, collector)
86
+ end
87
+
88
+ def add_metric_callbacks(caller_plugin_id, callback)
89
+ @metric_callbacks[caller_plugin_id] = callback
90
+ end
91
+
92
+ def caller_plugin_id=(caller_plugin_id)
93
+ @caller_plugin_id = caller_plugin_id
94
+ end
95
+
96
+ def find_callback
97
+ if @caller_plugin_id
98
+ @metric_callbacks[@caller_plugin_id]
99
+ else
100
+ nil
101
+ end
102
+ end
103
+
104
+ def emit(tag, time, record)
105
+ unless record.nil?
106
+ emit_stream(tag, OneEventStream.new(time, record))
107
+ end
108
+ end
109
+
110
+ def emit_array(tag, array)
111
+ emit_stream(tag, ArrayEventStream.new(array))
112
+ end
113
+
114
+ def emit_stream(tag, es)
115
+ match(tag).emit_events(tag, es)
116
+ if callback = find_callback
117
+ callback.call(es)
118
+ end
119
+ rescue => e
120
+ @emit_error_handler.handle_emits_error(tag, es, e)
121
+ end
122
+
123
+ def emit_error_event(tag, time, record, error)
124
+ @emit_error_handler.emit_error_event(tag, time, record, error)
125
+ end
126
+
127
+ def match?(tag)
128
+ !!find(tag)
129
+ end
130
+
131
+ def match(tag)
132
+ collector = @match_cache.get(tag) {
133
+ find(tag) || @default_collector
134
+ }
135
+ collector
136
+ end
137
+
138
+ class MatchCache
139
+ MATCH_CACHE_SIZE = 1024
140
+
141
+ def initialize
142
+ super
143
+ @map = {}
144
+ @keys = []
145
+ end
146
+
147
+ def get(key)
148
+ if collector = @map[key]
149
+ return collector
150
+ end
151
+ collector = @map[key] = yield
152
+ if @keys.size >= MATCH_CACHE_SIZE
153
+ # expire the oldest key
154
+ @map.delete @keys.shift
155
+ end
156
+ @keys << key
157
+ collector
158
+ end
159
+ end
160
+
161
+ private
162
+
163
+ class Pipeline
164
+ def initialize
165
+ @filters = []
166
+ @output = nil
167
+ @optimizer = FilterOptimizer.new
168
+ end
169
+
170
+ def add_filter(filter)
171
+ @filters << filter
172
+ @optimizer.filters = @filters
173
+ end
174
+
175
+ def set_output(output)
176
+ @output = output
177
+ end
178
+
179
+ def emit_events(tag, es)
180
+ processed = @optimizer.filter_stream(tag, es)
181
+ @output.emit_events(tag, processed)
182
+ end
183
+
184
+ class FilterOptimizer
185
+ def initialize(filters = [])
186
+ @filters = filters
187
+ @optimizable = nil
188
+ end
189
+
190
+ def filters=(filters)
191
+ @filters = filters
192
+ reset_optimization
193
+ end
194
+
195
+ def filter_stream(tag, es)
196
+ if optimizable?
197
+ optimized_filter_stream(tag, es)
198
+ else
199
+ @filters.reduce(es) { |acc, filter|
200
+ filtered_es = filter.filter_stream(tag, acc)
201
+ filter.measure_metrics(filtered_es)
202
+ filtered_es
203
+ }
204
+ end
205
+ end
206
+
207
+ private
208
+
209
+ def optimized_filter_stream(tag, es)
210
+ new_es = MultiEventStream.new
211
+ es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
212
+ filtered_record = record
213
+ filtered_time = time
214
+
215
+ catch :break_loop do
216
+ @filters.each do |filter|
217
+ if filter.has_filter_with_time
218
+ begin
219
+ filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
220
+ throw :break_loop unless filtered_record && filtered_time
221
+ filter.measure_metrics(OneEventStream.new(time, record))
222
+ rescue => e
223
+ filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
224
+ end
225
+ else
226
+ begin
227
+ filtered_record = filter.filter(tag, filtered_time, filtered_record)
228
+ throw :break_loop unless filtered_record
229
+ filter.measure_metrics(OneEventStream.new(time, record))
230
+ rescue => e
231
+ filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
232
+ end
233
+ end
234
+ end
235
+
236
+ new_es.add(filtered_time, filtered_record)
237
+ end
238
+ end
239
+ new_es
240
+ end
241
+
242
+ def optimizable?
243
+ return @optimizable unless @optimizable.nil?
244
+ fs_filters = filters_having_filter_stream
245
+ @optimizable = if fs_filters.empty?
246
+ true
247
+ else
248
+ # skip log message when filter is only 1, because its performance is same as non optimized chain.
249
+ if @filters.size > 1 && fs_filters.size >= 1
250
+ $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
251
+ end
252
+ false
253
+ end
254
+ end
255
+
256
+ def filters_having_filter_stream
257
+ @filters_having_filter_stream ||= @filters.select do |filter|
258
+ filter.class.instance_methods(false).include?(:filter_stream)
259
+ end
260
+ end
261
+
262
+ def reset_optimization
263
+ @optimizable = nil
264
+ @filters_having_filter_stream = nil
265
+ end
266
+ end
267
+ end
268
+
269
+ def find(tag)
270
+ pipeline = nil
271
+ @match_rules.each_with_index { |rule, i|
272
+ if rule.match?(tag)
273
+ if rule.collector.is_a?(Plugin::Filter)
274
+ pipeline ||= Pipeline.new
275
+ pipeline.add_filter(rule.collector)
276
+ else
277
+ if pipeline
278
+ pipeline.set_output(rule.collector)
279
+ else
280
+ # Use Output directly when filter is not matched
281
+ pipeline = rule.collector
282
+ end
283
+ return pipeline
284
+ end
285
+ end
286
+ }
287
+
288
+ if pipeline
289
+ # filter is matched but no match
290
+ pipeline.set_output(@default_collector)
291
+ pipeline
292
+ else
293
+ nil
294
+ end
295
+ end
296
+ end
297
+ end
@@ -0,0 +1,28 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ # To avoid duplicated requirements, extract this logic as file.
18
+
19
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')
20
+ require 'monitor'
21
+ else
22
+ begin
23
+ # monitor_ext is bundled since ruby 2.7.0
24
+ require 'ext_monitor'
25
+ rescue LoadError => _
26
+ require 'monitor'
27
+ end
28
+ end
@@ -0,0 +1,21 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/compat/filter'
18
+
19
+ module Fluent
20
+ Filter = Fluent::Compat::Filter
21
+ end