fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,343 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/types'
18
+ require 'fluent/config/element'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module CompatParameters
23
+ # This plugin helper is to bring old-fashioned buffer/other
24
+ # configuration parameters to v0.14 plugin API configurations.
25
+ # This helper is mainly to convert plugins from v0.12 API
26
+ # to v0.14 API safely, without breaking user deployment.
27
+
28
+ BUFFER_PARAMS = {
29
+ "buffer_type" => "@type",
30
+ "buffer_path" => "path",
31
+ "num_threads" => "flush_thread_count",
32
+ "flush_interval" => "flush_interval",
33
+ "try_flush_interval" => "flush_thread_interval",
34
+ "queued_chunk_flush_interval" => "flush_thread_burst_interval",
35
+ "disable_retry_limit" => "retry_forever",
36
+ "retry_limit" => "retry_max_times",
37
+ "max_retry_wait" => "retry_max_interval",
38
+ "buffer_chunk_limit" => "chunk_limit_size",
39
+ "buffer_queue_limit" => "queue_limit_length",
40
+ "buffer_queue_full_action" => "overflow_action",
41
+ "flush_at_shutdown" => "flush_at_shutdown",
42
+ }
43
+
44
+ BUFFER_TIME_SLICED_PARAMS = {
45
+ "time_slice_format" => nil,
46
+ "time_slice_wait" => "timekey_wait",
47
+ "timezone" => "timekey_zone",
48
+ }
49
+
50
+ PARSER_PARAMS = {
51
+ "format" => nil,
52
+ "types" => nil,
53
+ "types_delimiter" => nil,
54
+ "types_label_delimiter" => nil,
55
+ "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
56
+ "time_key" => "time_key",
57
+ "time_format" => "time_format",
58
+ "localtime" => nil,
59
+ "utc" => nil,
60
+ "delimiter" => "delimiter",
61
+ "keep_time_key" => "keep_time_key",
62
+ "null_empty_string" => "null_empty_string",
63
+ "null_value_pattern" => "null_value_pattern",
64
+ "json_parser" => "json_parser", # JSONParser
65
+ "label_delimiter" => "label_delimiter", # LabeledTSVParser
66
+ "format_firstline" => "format_firstline", # MultilineParser
67
+ "message_key" => "message_key", # NoneParser
68
+ "with_priority" => "with_priority", # SyslogParser
69
+ "message_format" => "message_format", # SyslogParser
70
+ "rfc5424_time_format" => "rfc5424_time_format", # SyslogParser
71
+ # There has been no parsers which can handle timezone in v0.12
72
+ }
73
+
74
+ INJECT_PARAMS = {
75
+ "include_time_key" => nil,
76
+ "time_key" => "time_key",
77
+ "time_format" => "time_format",
78
+ "timezone" => "timezone",
79
+ "include_tag_key" => nil,
80
+ "tag_key" => "tag_key",
81
+ "localtime" => nil,
82
+ "utc" => nil,
83
+ }
84
+
85
+ EXTRACT_PARAMS = {
86
+ "time_key" => "time_key",
87
+ "time_format" => "time_format",
88
+ "timezone" => "timezone",
89
+ "tag_key" => "tag_key",
90
+ "localtime" => nil,
91
+ "utc" => nil,
92
+ }
93
+
94
+ FORMATTER_PARAMS = {
95
+ "format" => "@type",
96
+ "delimiter" => "delimiter",
97
+ "force_quotes" => "force_quotes", # CsvFormatter
98
+ "keys" => "keys", # TSVFormatter
99
+ "fields" => "fields", # CsvFormatter
100
+ "json_parser" => "json_parser", # JSONFormatter
101
+ "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
102
+ "output_time" => "output_time", # OutFileFormatter
103
+ "output_tag" => "output_tag", # OutFileFormatter
104
+ "localtime" => "localtime", # OutFileFormatter
105
+ "utc" => "utc", # OutFileFormatter
106
+ "timezone" => "timezone", # OutFileFormatter
107
+ "message_key" => "message_key", # SingleValueFormatter
108
+ "add_newline" => "add_newline", # SingleValueFormatter
109
+ "output_type" => "output_type", # StdoutFormatter
110
+ }
111
+
112
+ def compat_parameters_convert(conf, *types, **kwargs)
113
+ types.each do |type|
114
+ case type
115
+ when :buffer
116
+ compat_parameters_buffer(conf, **kwargs)
117
+ when :inject
118
+ compat_parameters_inject(conf)
119
+ when :extract
120
+ compat_parameters_extract(conf)
121
+ when :parser
122
+ compat_parameters_parser(conf)
123
+ when :formatter
124
+ compat_parameters_formatter(conf)
125
+ else
126
+ raise "BUG: unknown compat_parameters type: #{type}"
127
+ end
128
+ end
129
+
130
+ conf
131
+ end
132
+
133
+ def compat_parameters_buffer(conf, default_chunk_key: '')
134
+ # return immediately if <buffer> section exists, or any buffer-related parameters don't exist
135
+ return unless conf.elements('buffer').empty?
136
+ return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }
137
+
138
+ # TODO: warn obsolete parameters if these are deprecated
139
+ buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
140
+ hash = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
141
+ if compat_key == 'buffer_queue_full_action' && value == 'exception'
142
+ 'throw_exception'
143
+ else
144
+ value
145
+ end
146
+ end
147
+
148
+ chunk_key = default_chunk_key
149
+
150
+ if conf.has_key?('time_slice_format')
151
+ chunk_key = 'time'
152
+ hash['timekey'] = case conf['time_slice_format']
153
+ when /\%S/ then 1
154
+ when /\%M/ then 60
155
+ when /\%H/ then 3600
156
+ when /\%d/ then 86400
157
+ else
158
+ raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
159
+ end
160
+ if conf.has_key?('localtime') || conf.has_key?('utc')
161
+ if conf.has_key?('localtime') && conf.has_key?('utc')
162
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
163
+ elsif conf.has_key?('localtime')
164
+ hash['timekey_use_utc'] = !(Fluent::Config.bool_value(conf['localtime']))
165
+ elsif conf.has_key?('utc')
166
+ hash['timekey_use_utc'] = Fluent::Config.bool_value(conf['utc'])
167
+ end
168
+ end
169
+ else
170
+ if chunk_key == 'time'
171
+ hash['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
172
+ end
173
+ end
174
+
175
+ e = Fluent::Config::Element.new('buffer', chunk_key, hash, [])
176
+ conf.elements << e
177
+
178
+ conf
179
+ end
180
+
181
+ def compat_parameters_inject(conf)
182
+ return unless conf.elements('inject').empty?
183
+ return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
184
+
185
+ # TODO: warn obsolete parameters if these are deprecated
186
+ hash = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
187
+
188
+ if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
189
+ hash['time_key'] ||= 'time'
190
+ hash['time_type'] ||= 'string'
191
+ end
192
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
193
+ hash['time_key'] ||= 'time'
194
+ hash['time_type'] = 'unixtime'
195
+ end
196
+ if conf.has_key?('localtime') || conf.has_key?('utc')
197
+ utc = to_bool(conf['utc'])
198
+ localtime = to_bool(conf['localtime'])
199
+ if conf.has_key?('localtime') && conf.has_key?('utc') && !(localtime ^ utc)
200
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
201
+ elsif conf.has_key?('localtime')
202
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
203
+ elsif conf.has_key?('utc')
204
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
205
+ # Specifying "localtime false" means using UTC in TimeFormatter
206
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
207
+ # There are difference between "Z" and "+0000" in timezone formatting.
208
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
209
+ end
210
+ end
211
+
212
+ if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
213
+ hash['tag_key'] ||= 'tag'
214
+ end
215
+
216
+ e = Fluent::Config::Element.new('inject', '', hash, [])
217
+ conf.elements << e
218
+
219
+ conf
220
+ end
221
+
222
+ def to_bool(v)
223
+ if v.is_a?(FalseClass) || v == 'false' || v.nil?
224
+ false
225
+ else
226
+ true
227
+ end
228
+ end
229
+
230
+ def compat_parameters_extract(conf)
231
+ return unless conf.elements('extract').empty?
232
+ return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')
233
+
234
+ # TODO: warn obsolete parameters if these are deprecated
235
+ hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)
236
+
237
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
238
+ hash['time_key'] ||= 'time'
239
+ hash['time_type'] = 'unixtime'
240
+ elsif conf.has_key?('format') && conf["format"].start_with?("/") && conf["format"].end_with?("/") # old-style regexp parser
241
+ hash['time_key'] ||= 'time'
242
+ hash['time_type'] ||= 'string'
243
+ end
244
+ if conf.has_key?('localtime') || conf.has_key?('utc')
245
+ if conf.has_key?('localtime') && conf.has_key?('utc')
246
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
247
+ elsif conf.has_key?('localtime')
248
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
249
+ elsif conf.has_key?('utc')
250
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
251
+ # Specifying "localtime false" means using UTC in TimeFormatter
252
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
253
+ # There are difference between "Z" and "+0000" in timezone formatting.
254
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
255
+ end
256
+ end
257
+
258
+ e = Fluent::Config::Element.new('extract', '', hash, [])
259
+ conf.elements << e
260
+
261
+ conf
262
+ end
263
+
264
+ def compat_parameters_parser(conf)
265
+ return unless conf.elements('parse').empty?
266
+ return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
267
+
268
+ # TODO: warn obsolete parameters if these are deprecated
269
+ hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
270
+
271
+ if conf["format"]
272
+ if conf["format"].start_with?("/") && conf["format"].end_with?("/")
273
+ hash["@type"] = "regexp"
274
+ hash["expression"] = conf["format"][1..-2]
275
+ else
276
+ hash["@type"] = conf["format"]
277
+ end
278
+ end
279
+
280
+ if conf["types"]
281
+ delimiter = conf["types_delimiter"] || ','
282
+ label_delimiter = conf["types_label_delimiter"] || ':'
283
+ types = {}
284
+ conf['types'].split(delimiter).each do |pair|
285
+ key, value = pair.split(label_delimiter, 2)
286
+ types[key] = value
287
+ end
288
+ hash["types"] = JSON.dump(types)
289
+ end
290
+ if conf.has_key?('localtime') || conf.has_key?('utc')
291
+ if conf.has_key?('localtime') && conf.has_key?('utc')
292
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
293
+ elsif conf.has_key?('localtime')
294
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
295
+ elsif conf.has_key?('utc')
296
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
297
+ # Specifying "localtime false" means using UTC in TimeFormatter
298
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
299
+ # There are difference between "Z" and "+0000" in timezone formatting.
300
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
301
+ end
302
+ end
303
+
304
+ e = Fluent::Config::Element.new('parse', '', hash, [])
305
+ conf.elements << e
306
+
307
+ conf
308
+ end
309
+
310
+ def compat_parameters_formatter(conf)
311
+ return unless conf.elements('format').empty?
312
+ return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
313
+
314
+ # TODO: warn obsolete parameters if these are deprecated
315
+ hash = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
316
+
317
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
318
+ hash['time_type'] = 'unixtime'
319
+ end
320
+
321
+ e = Fluent::Config::Element.new('format', '', hash, [])
322
+ conf.elements << e
323
+
324
+ conf
325
+ end
326
+
327
+ def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
328
+ hash = {}
329
+ params.each do |compat, current|
330
+ next unless current
331
+ if conf.has_key?(compat)
332
+ if block_given?
333
+ hash[current] = block.call(compat, conf[compat])
334
+ else
335
+ hash[current] = conf[compat]
336
+ end
337
+ end
338
+ end
339
+ hash
340
+ end
341
+ end
342
+ end
343
+ end
@@ -0,0 +1,51 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/counter/client'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module Counter
23
+ def counter_client_create(scope:, loop: Coolio::Loop.new)
24
+ client_conf = system_config.counter_client
25
+ raise Fluent::ConfigError, '<counter_client> is required in <system>' unless client_conf
26
+ counter_client = Fluent::Counter::Client.new(loop, port: client_conf.port, host: client_conf.host, log: log, timeout: client_conf.timeout)
27
+ counter_client.start
28
+ counter_client.establish(scope)
29
+ @_counter_client = counter_client
30
+ counter_client
31
+ end
32
+
33
+ attr_reader :_counter_client
34
+
35
+ def initialize
36
+ super
37
+ @_counter_client = nil
38
+ end
39
+
40
+ def stop
41
+ super
42
+ @_counter_client.stop
43
+ end
44
+
45
+ def terminate
46
+ @_counter_client = nil
47
+ super
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,100 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/time'
18
+
19
+ module Fluent
20
+ module PluginHelper
21
+ module EventEmitter
22
+ # stop : [-]
23
+ # shutdown : disable @router
24
+ # close : [-]
25
+ # terminate: [-]
26
+
27
+ def router
28
+ @_event_emitter_used_actually = true
29
+ if @_event_emitter_lazy_init
30
+ @router = @primary_instance.router
31
+ end
32
+ if @router.respond_to?(:caller_plugin_id=)
33
+ @router.caller_plugin_id = self.plugin_id
34
+ end
35
+ @router
36
+ end
37
+
38
+ def router=(r)
39
+ # not recommended now...
40
+ @router = r
41
+ end
42
+
43
+ def has_router?
44
+ true
45
+ end
46
+
47
+ def event_emitter_used_actually?
48
+ @_event_emitter_used_actually
49
+ end
50
+
51
+ def event_emitter_router(label_name)
52
+ if label_name
53
+ if label_name == "@ROOT"
54
+ Engine.root_agent.event_router
55
+ else
56
+ Engine.root_agent.find_label(label_name).event_router
57
+ end
58
+ elsif self.respond_to?(:as_secondary) && self.as_secondary
59
+ if @primary_instance.has_router?
60
+ @_event_emitter_lazy_init = true
61
+ nil # primary plugin's event router is not initialized yet, here.
62
+ else
63
+ @primary_instance.context_router
64
+ end
65
+ else
66
+ # `Engine.root_agent.event_router` is for testing
67
+ self.context_router || Engine.root_agent.event_router
68
+ end
69
+ end
70
+
71
+ def initialize
72
+ super
73
+ @_event_emitter_used_actually = false
74
+ @_event_emitter_lazy_init = false
75
+ @router = nil
76
+ end
77
+
78
+ def configure(conf)
79
+ require 'fluent/engine'
80
+ super
81
+ @router = event_emitter_router(conf['@label'])
82
+ end
83
+
84
+ def after_shutdown
85
+ @router = nil
86
+ super
87
+ end
88
+
89
+ def close # unset router many times to reduce test cost
90
+ @router = nil
91
+ super
92
+ end
93
+
94
+ def terminate
95
+ @router = nil
96
+ super
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,170 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'fluent/plugin_helper/thread'
19
+ require 'fluent/clock'
20
+
21
+ module Fluent
22
+ module PluginHelper
23
+ module EventLoop
24
+ # Currently this plugin helper is only for other helpers, not plugins.
25
+ # there's no way to create customized watchers to attach event loops.
26
+ include Fluent::PluginHelper::Thread
27
+
28
+ # stop : [-]
29
+ # shutdown : detach all event watchers on event loop
30
+ # close : stop event loop
31
+ # terminate: initialize internal state
32
+
33
+ EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
34
+ EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
35
+
36
+ attr_reader :_event_loop # for tests
37
+
38
+ def event_loop_attach(watcher)
39
+ @_event_loop_mutex.synchronize do
40
+ @_event_loop.attach(watcher)
41
+ @_event_loop_attached_watchers << watcher
42
+ watcher
43
+ end
44
+ end
45
+
46
+ def event_loop_detach(watcher)
47
+ if watcher.attached?
48
+ watcher.detach
49
+ end
50
+ @_event_loop_mutex.synchronize do
51
+ @_event_loop_attached_watchers.delete(watcher)
52
+ end
53
+ end
54
+
55
+ def event_loop_wait_until_start
56
+ sleep(0.1) until event_loop_running?
57
+ end
58
+
59
+ def event_loop_wait_until_stop
60
+ timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
61
+ sleep(0.1) while event_loop_running? && Fluent::Clock.now < timeout_at
62
+ if @_event_loop_running
63
+ puts "terminating event_loop forcedly"
64
+ caller.each{|bt| puts "\t#{bt}" }
65
+ @_event_loop.stop rescue nil
66
+ @_event_loop_running = true
67
+ end
68
+ end
69
+
70
+ def event_loop_running?
71
+ @_event_loop_running
72
+ end
73
+
74
+ def initialize
75
+ super
76
+ @_event_loop = Coolio::Loop.new
77
+ @_event_loop_running = false
78
+ @_event_loop_mutex = Mutex.new
79
+ # plugin MAY configure loop run timeout in #configure
80
+ @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
81
+ @_event_loop_attached_watchers = []
82
+ end
83
+
84
+ def start
85
+ super
86
+
87
+ # event loop does not run here, so mutex lock is not required
88
+ thread_create :event_loop do
89
+ begin
90
+ default_watcher = DefaultWatcher.new
91
+ event_loop_attach(default_watcher)
92
+ @_event_loop_running = true
93
+ @_event_loop.run(@_event_loop_run_timeout) # this method blocks
94
+ ensure
95
+ @_event_loop_running = false
96
+ end
97
+ end
98
+ end
99
+
100
+ def shutdown
101
+ @_event_loop_mutex.synchronize do
102
+ @_event_loop_attached_watchers.reverse.each do |w|
103
+ if w.attached?
104
+ begin
105
+ w.detach
106
+ rescue => e
107
+ log.warn "unexpected error while detaching event loop watcher", error: e
108
+ end
109
+ end
110
+ end
111
+ end
112
+
113
+ super
114
+ end
115
+
116
+ def after_shutdown
117
+ timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
118
+ @_event_loop_mutex.synchronize do
119
+ @_event_loop.watchers.reverse.each do |w|
120
+ begin
121
+ w.detach
122
+ rescue => e
123
+ log.warn "unexpected error while detaching event loop watcher", error: e
124
+ end
125
+ end
126
+ end
127
+ while @_event_loop_running
128
+ if Fluent::Clock.now >= timeout_at
129
+ log.warn "event loop does NOT exit until hard timeout."
130
+ raise "event loop does NOT exit until hard timeout." if @under_plugin_development
131
+ break
132
+ end
133
+ sleep 0.1
134
+ end
135
+
136
+ super
137
+ end
138
+
139
+ def close
140
+ if @_event_loop_running
141
+ begin
142
+ @_event_loop.stop # we cannot check loop is running or not
143
+ rescue RuntimeError => e
144
+ raise unless e.message == 'loop not running'
145
+ end
146
+ end
147
+
148
+ super
149
+ end
150
+
151
+ def terminate
152
+ @_event_loop = nil
153
+ @_event_loop_running = false
154
+ @_event_loop_mutex = nil
155
+ @_event_loop_run_timeout = nil
156
+
157
+ super
158
+ end
159
+
160
+ # watcher to block to run event loop until shutdown
161
+ class DefaultWatcher < Coolio::TimerWatcher
162
+ def initialize
163
+ super(1, true) # interval: 1, repeat: true
164
+ end
165
+ # do nothing
166
+ def on_timer; end
167
+ end
168
+ end
169
+ end
170
+ end