fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
@@ -0,0 +1,319 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ require 'fluent/plugin/output'
17
+ require 'fluent/env'
18
+ require 'fluent/config/error'
19
+
20
+ require 'yajl'
21
+
22
+ module Fluent::Plugin
23
+ class ExecFilterOutput < Output
24
+ Fluent::Plugin.register_output('exec_filter', self)
25
+
26
+ helpers :compat_parameters, :inject, :formatter, :parser, :extract, :child_process, :event_emitter
27
+
28
+ desc 'The command (program) to execute.'
29
+ config_param :command, :string
30
+
31
+ config_param :remove_prefix, :string, default: nil, deprecated: "use @label instead for event routing"
32
+ config_param :add_prefix, :string, default: nil, deprecated: "use @label instead for event routing"
33
+
34
+ config_section :inject do
35
+ config_set_default :time_type, :unixtime
36
+ end
37
+
38
+ config_section :format do
39
+ config_set_default :@type, 'tsv'
40
+ config_set_default :localtime, true
41
+ end
42
+
43
+ config_section :parse do
44
+ config_set_default :@type, 'tsv'
45
+ config_set_default :time_key, nil
46
+ config_set_default :time_format, nil
47
+ config_set_default :localtime, true
48
+ config_set_default :estimate_current_event, false
49
+ end
50
+
51
+ config_section :extract do
52
+ config_set_default :time_type, :float
53
+ end
54
+
55
+ config_section :buffer do
56
+ config_set_default :flush_mode, :interval
57
+ config_set_default :flush_interval, 1
58
+ end
59
+
60
+ config_param :tag, :string, default: nil
61
+
62
+ config_param :tag_key, :string, default: nil, deprecated: "use 'tag_key' in <inject>/<extract> instead"
63
+ config_param :time_key, :string, default: nil, deprecated: "use 'time_key' in <inject>/<extract> instead"
64
+ config_param :time_format, :string, default: nil, deprecated: "use 'time_format' in <inject>/<extract> instead"
65
+
66
+ desc 'The default block size to read if parser requires partial read.'
67
+ config_param :read_block_size, :size, default: 10240 # 10k
68
+
69
+ desc 'The number of spawned process for command.'
70
+ config_param :num_children, :integer, default: 1
71
+
72
+ desc 'Respawn command when command exit. ["none", "inf" or positive integer for times to respawn (default: none)]'
73
+ # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
74
+ config_param :child_respawn, :string, default: nil
75
+
76
+ # 0: output logs for all of messages to emit
77
+ config_param :suppress_error_log_interval, :time, default: 0
78
+
79
+ attr_reader :formatter, :parser # for tests
80
+
81
+ KEYS_FOR_IN_AND_OUT = {
82
+ 'tag_key' => ['in_tag_key', 'out_tag_key'],
83
+ 'time_key' => ['in_time_key', 'out_time_key'],
84
+ 'time_format' => ['in_time_format', 'out_time_format'],
85
+ }
86
+ COMPAT_INJECT_PARAMS = {
87
+ 'in_tag_key' => 'tag_key',
88
+ 'in_time_key' => 'time_key',
89
+ 'in_time_format' => 'time_format',
90
+ }
91
+ COMPAT_FORMAT_PARAMS = {
92
+ 'in_format' => '@type',
93
+ 'in_keys' => 'keys',
94
+ }
95
+ COMPAT_PARSE_PARAMS = {
96
+ 'out_format' => '@type',
97
+ 'out_keys' => 'keys',
98
+ 'out_stream_buffer_size' => 'stream_buffer_size',
99
+ }
100
+ COMPAT_EXTRACT_PARAMS = {
101
+ 'out_tag_key' => 'tag_key',
102
+ 'out_time_key' => 'time_key',
103
+ 'out_time_format' => 'time_format',
104
+ }
105
+
106
+ def exec_filter_compat_parameters_copy_to_subsection!(conf, subsection_name, params)
107
+ return unless conf.elements(subsection_name).empty?
108
+ return unless params.keys.any?{|k| conf.has_key?(k) }
109
+ hash = {}
110
+ params.each_pair do |compat, current|
111
+ hash[current] = conf[compat] if conf.has_key?(compat)
112
+ end
113
+ conf.elements << Fluent::Config::Element.new(subsection_name, '', hash, [])
114
+ end
115
+
116
+ def exec_filter_compat_parameters_convert!(conf)
117
+ KEYS_FOR_IN_AND_OUT.each_pair do |inout, keys|
118
+ if conf.has_key?(inout)
119
+ keys.each do |k|
120
+ conf[k] = conf[inout]
121
+ end
122
+ end
123
+ end
124
+ exec_filter_compat_parameters_copy_to_subsection!(conf, 'inject', COMPAT_INJECT_PARAMS)
125
+ exec_filter_compat_parameters_copy_to_subsection!(conf, 'format', COMPAT_FORMAT_PARAMS)
126
+ exec_filter_compat_parameters_copy_to_subsection!(conf, 'parse', COMPAT_PARSE_PARAMS)
127
+ exec_filter_compat_parameters_copy_to_subsection!(conf, 'extract', COMPAT_EXTRACT_PARAMS)
128
+ end
129
+
130
+ def configure(conf)
131
+ exec_filter_compat_parameters_convert!(conf)
132
+ compat_parameters_convert(conf, :buffer)
133
+
134
+ if inject_section = conf.elements('inject').first
135
+ if inject_section.has_key?('time_format')
136
+ inject_section['time_type'] ||= 'string'
137
+ end
138
+ end
139
+ if extract_section = conf.elements('extract').first
140
+ if extract_section.has_key?('time_format')
141
+ extract_section['time_type'] ||= 'string'
142
+ end
143
+ end
144
+
145
+ super
146
+
147
+ if !@tag && (!@extract_config || !@extract_config.tag_key)
148
+ raise Fluent::ConfigError, "'tag' or '<extract> tag_key </extract>' option is required on exec_filter output"
149
+ end
150
+
151
+ @formatter = formatter_create
152
+ @parser = parser_create
153
+
154
+ if @remove_prefix
155
+ @removed_prefix_string = @remove_prefix + '.'
156
+ @removed_length = @removed_prefix_string.length
157
+ end
158
+ if @add_prefix
159
+ @added_prefix_string = @add_prefix + '.'
160
+ end
161
+
162
+ @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
163
+ 0
164
+ elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
165
+ -1
166
+ elsif @child_respawn =~ /^\d+$/
167
+ @child_respawn.to_i
168
+ else
169
+ raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
170
+ end
171
+
172
+ @suppress_error_log_interval ||= 0
173
+ @next_log_time = Time.now.to_i
174
+ end
175
+
176
+ def multi_workers_ready?
177
+ true
178
+ end
179
+
180
+ ExecutedProcess = Struct.new(:mutex, :pid, :respawns, :readio, :writeio)
181
+
182
+ def start
183
+ super
184
+
185
+ @children_mutex = Mutex.new
186
+ @children = []
187
+ @rr = 0
188
+
189
+ exit_callback = ->(status){
190
+ c = @children.select{|child| child.pid == status.pid }.first
191
+ if c
192
+ unless self.stopped?
193
+ log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
194
+ end
195
+ c.mutex.synchronize do
196
+ (c.writeio && c.writeio.close) rescue nil
197
+ (c.readio && c.readio.close) rescue nil
198
+ c.pid = c.readio = c.writeio = nil
199
+ end
200
+ end
201
+ }
202
+ child_process_callback = ->(index, readio, writeio){
203
+ pid = child_process_id
204
+ c = @children[index]
205
+ writeio.sync = true
206
+ c.mutex.synchronize do
207
+ c.pid = pid
208
+ c.respawns = @respawns
209
+ c.readio = readio
210
+ c.writeio = writeio
211
+ end
212
+
213
+ run(readio)
214
+ }
215
+ execute_child_process = ->(index){
216
+ child_process_execute("out_exec_filter_child#{index}".to_sym, @command, on_exit_callback: exit_callback) do |readio, writeio|
217
+ child_process_callback.call(index, readio, writeio)
218
+ end
219
+ }
220
+
221
+ @children_mutex.synchronize do
222
+ @num_children.times do |i|
223
+ @children << ExecutedProcess.new(Mutex.new, nil, 0, nil, nil)
224
+ execute_child_process.call(i)
225
+ end
226
+ end
227
+
228
+ if @respawns != 0
229
+ thread_create(:out_exec_filter_respawn_monitor) do
230
+ while thread_current_running?
231
+ @children.each_with_index do |c, i|
232
+ if c.mutex && c.mutex.synchronize{ c.pid.nil? && c.respawns != 0 }
233
+ respawns = c.mutex.synchronize do
234
+ c.respawns -= 1 if c.respawns > 0
235
+ c.respawns
236
+ end
237
+ log.info "respawning child process", num: i, respawn_counter: respawns
238
+ execute_child_process.call(i)
239
+ end
240
+ end
241
+ sleep 0.2
242
+ end
243
+ end
244
+ end
245
+ end
246
+
247
+ def terminate
248
+ @children = []
249
+ super
250
+ end
251
+
252
+ def tag_remove_prefix(tag)
253
+ if @remove_prefix
254
+ if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
255
+ tag = tag[@removed_length..-1] || ''
256
+ end
257
+ end
258
+ tag
259
+ end
260
+
261
+ NEWLINE = "\n"
262
+
263
+ def format(tag, time, record)
264
+ tag = tag_remove_prefix(tag)
265
+ record = inject_values_to_record(tag, time, record)
266
+ if @formatter.formatter_type == :text_per_line
267
+ @formatter.format(tag, time, record).chomp + NEWLINE
268
+ else
269
+ @formatter.format(tag, time, record)
270
+ end
271
+ end
272
+
273
+ def write(chunk)
274
+ try_times = 0
275
+ while true
276
+ r = @rr = (@rr + 1) % @children.length
277
+ if @children[r].pid && writeio = @children[r].writeio
278
+ chunk.write_to(writeio)
279
+ break
280
+ end
281
+ try_times += 1
282
+ raise "no healthy child processes exist" if try_times >= @children.length
283
+ end
284
+ end
285
+
286
+ def run(io)
287
+ io.set_encoding(Encoding::ASCII_8BIT)
288
+ case
289
+ when @parser.implement?(:parse_io)
290
+ @parser.parse_io(io, &method(:on_record))
291
+ when @parser.implement?(:parse_partial_data)
292
+ until io.eof?
293
+ @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))
294
+ end
295
+ when @parser.parser_type == :text_per_line
296
+ io.each_line do |line|
297
+ @parser.parse(line.chomp, &method(:on_record))
298
+ end
299
+ else
300
+ @parser.parse(io.read, &method(:on_record))
301
+ end
302
+ end
303
+
304
+ def on_record(time, record)
305
+ tag = extract_tag_from_record(record)
306
+ tag = @added_prefix_string + tag if tag && @add_prefix
307
+ tag ||= @tag
308
+ time ||= extract_time_from_record(record) || Fluent::EventTime.now
309
+ router.emit(tag, time, record)
310
+ rescue => e
311
+ if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
312
+ log.error "exec_filter failed to emit", record: Yajl.dump(record), error: e
313
+ log.error_backtrace e.backtrace
314
+ @next_log_time = Time.now.to_i + @suppress_error_log_interval
315
+ end
316
+ router.emit_error_event(tag, time, record, e) if tag && time && record
317
+ end
318
+ end
319
+ end
@@ -0,0 +1,334 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fileutils'
18
+ require 'zlib'
19
+ require 'time'
20
+
21
+ require 'fluent/plugin/output'
22
+ require 'fluent/config/error'
23
+ # TODO remove ...
24
+ require 'fluent/plugin/file_util'
25
+
26
+ module Fluent::Plugin
27
+ class FileOutput < Output
28
+ Fluent::Plugin.register_output('file', self)
29
+
30
+ helpers :formatter, :inject, :compat_parameters
31
+
32
+ SUPPORTED_COMPRESS = [:text, :gz, :gzip]
33
+ SUPPORTED_COMPRESS_MAP = {
34
+ text: nil,
35
+ gz: :gzip,
36
+ gzip: :gzip,
37
+ }
38
+
39
+ DEFAULT_TIMEKEY = 60 * 60 * 24
40
+
41
+ desc "The Path of the file."
42
+ config_param :path, :string
43
+
44
+ desc "Specify to add file suffix for bare file path or not."
45
+ config_param :add_path_suffix, :bool, default: true
46
+ desc "The file suffix added to bare file path."
47
+ config_param :path_suffix, :string, default: '.log'
48
+ desc "The flushed chunk is appended to existence file or not."
49
+ config_param :append, :bool, default: false
50
+ desc "Compress flushed file."
51
+ config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
52
+ desc "Execute compression again even when buffer chunk is already compressed."
53
+ config_param :recompress, :bool, default: false
54
+ desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
55
+ config_param :symlink_path, :string, default: nil
56
+
57
+ config_section :format do
58
+ config_set_default :@type, 'out_file'
59
+ end
60
+
61
+ config_section :buffer do
62
+ config_set_default :@type, 'file'
63
+ config_set_default :chunk_keys, ['time']
64
+ config_set_default :timekey, DEFAULT_TIMEKEY
65
+ end
66
+
67
+ attr_reader :dir_perm
68
+ attr_accessor :last_written_path # for tests
69
+
70
+ module SymlinkBufferMixin
71
+ def metadata(timekey: nil, tag: nil, variables: nil)
72
+ metadata = super
73
+
74
+ @latest_metadata ||= new_metadata(timekey: 0)
75
+ if metadata.timekey && (metadata.timekey >= @latest_metadata.timekey)
76
+ @latest_metadata = metadata
77
+ end
78
+
79
+ metadata
80
+ end
81
+
82
+ def output_plugin_for_symlink=(output_plugin)
83
+ @_output_plugin_for_symlink = output_plugin
84
+ end
85
+
86
+ def symlink_path=(path)
87
+ @_symlink_path = path
88
+ end
89
+
90
+ def generate_chunk(metadata)
91
+ chunk = super
92
+ # "symlink" feature is to link from symlink_path to the latest file chunk. Records with latest
93
+ # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
94
+ # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
95
+ # These chunks will be enqueued immediately, and will be flushed soon.
96
+ if chunk.metadata == @latest_metadata
97
+ sym_path = @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk)
98
+ FileUtils.mkdir_p(File.dirname(sym_path), mode: @_output_plugin_for_symlink.dir_perm)
99
+ FileUtils.ln_sf(chunk.path, sym_path)
100
+ end
101
+ chunk
102
+ end
103
+ end
104
+
105
+ def configure(conf)
106
+ compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: "time")
107
+
108
+ configured_time_slice_format = conf['time_slice_format']
109
+
110
+ if conf.elements(name: 'buffer').empty?
111
+ conf.add_element('buffer', 'time')
112
+ end
113
+ buffer_conf = conf.elements(name: 'buffer').first
114
+ # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.
115
+ if !buffer_conf.has_key?('path') && !(conf['@id'] && system_config.root_dir)
116
+ # v0.14 file buffer handles path as directory if '*' is missing
117
+ # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
118
+ # but raise it in this plugin.
119
+ buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
120
+ end
121
+
122
+ if conf.has_key?('utc') || conf.has_key?('localtime')
123
+ param_name = conf.has_key?('utc') ? 'utc' : 'localtime'
124
+ log.warn "'#{param_name}' is deprecated for output plugin. This parameter is used for formatter plugin in compatibility layer. If you want to use same feature, use timekey_use_utc parameter in <buffer> directive instead"
125
+ end
126
+
127
+ super
128
+
129
+ @compress_method = SUPPORTED_COMPRESS_MAP[@compress]
130
+
131
+ if @path.include?('*') && !@buffer_config.timekey
132
+ raise Fluent::ConfigError, "path including '*' must be used with buffer chunk key 'time'"
133
+ end
134
+
135
+ path_suffix = @add_path_suffix ? @path_suffix : ''
136
+ path_timekey = if @chunk_key_time
137
+ @as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey
138
+ else
139
+ nil
140
+ end
141
+ @path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
142
+
143
+ if @as_secondary
144
+ # When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.
145
+ # Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.
146
+ placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|
147
+ v.validate!
148
+ end
149
+ else
150
+ placeholder_validate!(:path, @path_template)
151
+
152
+ max_tag_index = get_placeholders_tag(@path_template).max || 1
153
+ max_tag_index = 1 if max_tag_index < 1
154
+ dummy_tag = (['a'] * max_tag_index).join('.')
155
+ dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
156
+ dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
157
+
158
+ test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
159
+ test_path = extract_placeholders(@path_template, test_chunk1)
160
+ unless ::Fluent::FileUtil.writable_p?(test_path)
161
+ raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
162
+ end
163
+ end
164
+
165
+ @formatter = formatter_create
166
+
167
+ if @symlink_path && @buffer.respond_to?(:path)
168
+ if @as_secondary
169
+ raise Fluent::ConfigError, "symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin"
170
+ end
171
+ if Fluent.windows?
172
+ log.warn "symlink_path is unavailable on Windows platform. disabled."
173
+ @symlink_path = nil
174
+ else
175
+ @buffer.extend SymlinkBufferMixin
176
+ @buffer.symlink_path = @symlink_path
177
+ @buffer.output_plugin_for_symlink = self
178
+ end
179
+ end
180
+
181
+ @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
182
+ @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
183
+ @need_lock = system_config.workers > 1
184
+
185
+ # https://github.com/fluent/fluentd/issues/3569
186
+ @need_ruby_on_macos_workaround = false
187
+ if @append && Fluent.macos?
188
+ condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
189
+ @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
190
+ end
191
+ end
192
+
193
+ def multi_workers_ready?
194
+ true
195
+ end
196
+
197
+ def format(tag, time, record)
198
+ r = inject_values_to_record(tag, time, record)
199
+ @formatter.format(tag, time, r)
200
+ end
201
+
202
+ def write(chunk)
203
+ path = extract_placeholders(@path_template, chunk)
204
+ FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
205
+
206
+ writer = case
207
+ when @compress_method.nil?
208
+ method(:write_without_compression)
209
+ when @compress_method == :gzip
210
+ if @buffer.compress != :gzip || @recompress
211
+ method(:write_gzip_with_compression)
212
+ else
213
+ method(:write_gzip_from_gzipped_chunk)
214
+ end
215
+ else
216
+ raise "BUG: unknown compression method #{@compress_method}"
217
+ end
218
+
219
+ if @append
220
+ writer.call(path, chunk)
221
+ else
222
+ find_filepath_available(path, with_lock: @need_lock) do |actual_path|
223
+ writer.call(actual_path, chunk)
224
+ path = actual_path
225
+ end
226
+ end
227
+
228
+ @last_written_path = path
229
+ end
230
+
231
+ def write_without_compression(path, chunk)
232
+ File.open(path, "ab", @file_perm) do |f|
233
+ if @need_ruby_on_macos_workaround
234
+ content = chunk.read()
235
+ f.puts content
236
+ else
237
+ chunk.write_to(f)
238
+ end
239
+ end
240
+ end
241
+
242
+ def write_gzip_with_compression(path, chunk)
243
+ File.open(path, "ab", @file_perm) do |f|
244
+ gz = Zlib::GzipWriter.new(f)
245
+ chunk.write_to(gz, compressed: :text)
246
+ gz.close
247
+ end
248
+ end
249
+
250
+ def write_gzip_from_gzipped_chunk(path, chunk)
251
+ File.open(path, "ab", @file_perm) do |f|
252
+ chunk.write_to(f, compressed: :gzip)
253
+ end
254
+ end
255
+
256
+ def timekey_to_timeformat(timekey)
257
+ case timekey
258
+ when nil then ''
259
+ when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
260
+ when 60...3600 then '%Y%m%d%H%M'
261
+ when 3600...86400 then '%Y%m%d%H'
262
+ else '%Y%m%d'
263
+ end
264
+ end
265
+
266
+ def compression_suffix(compress)
267
+ case compress
268
+ when :gzip then '.gz'
269
+ when nil then ''
270
+ else
271
+ raise ArgumentError, "unknown compression type #{compress}"
272
+ end
273
+ end
274
+
275
+ # /path/to/dir/file.* -> /path/to/dir/file.%Y%m%d
276
+ # /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data
277
+ # /path/to/dir/file -> /path/to/dir/file.%Y%m%d.log
278
+ # %Y%m%d -> %Y%m%d_** (non append)
279
+ # + .gz (gzipped)
280
+ ## TODO: remove time_slice_format when end of support of compat_parameters
281
+ def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)
282
+ comp_suffix = compression_suffix(compress)
283
+ index_placeholder = append ? '' : '_**'
284
+ if original.index('*')
285
+ raise "BUG: configuration error must be raised for path including '*' without timekey" unless timekey
286
+ time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)
287
+ original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix
288
+ else
289
+ if timekey
290
+ if time_slice_format
291
+ "#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}"
292
+ else
293
+ time_placeholders = timekey_to_timeformat(timekey)
294
+ if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }
295
+ raise Fluent::ConfigError, "insufficient timestamp placeholders in path" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }
296
+ "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
297
+ else
298
+ "#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}"
299
+ end
300
+ end
301
+ else
302
+ "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
303
+ end
304
+ end
305
+ end
306
+
307
+ def find_filepath_available(path_with_placeholder, with_lock: false) # for non-append
308
+ raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
309
+ i = 0
310
+ dir_path = locked = nil
311
+ while true
312
+ path = path_with_placeholder.sub('_**', "_#{i}")
313
+ i += 1
314
+ next if File.exist?(path)
315
+
316
+ if with_lock
317
+ dir_path = path + '.lock'
318
+ locked = Dir.mkdir(dir_path) rescue false
319
+ next unless locked
320
+ # ensure that other worker doesn't create a file (and release lock)
321
+ # between previous File.exist? and Dir.mkdir
322
+ next if File.exist?(path)
323
+ end
324
+
325
+ break
326
+ end
327
+ yield path
328
+ ensure
329
+ if dir_path && locked && Dir.exist?(dir_path)
330
+ Dir.rmdir(dir_path) rescue nil
331
+ end
332
+ end
333
+ end
334
+ end