fluentd 1.14.4-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (558) hide show
  1. checksums.yaml +7 -0
  2. data/.deepsource.toml +13 -0
  3. data/.drone.yml +35 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
  5. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  6. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  7. data/.github/ISSUE_TEMPLATE.md +17 -0
  8. data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
  9. data/.github/workflows/issue-auto-closer.yml +12 -0
  10. data/.github/workflows/linux-test.yaml +36 -0
  11. data/.github/workflows/macos-test.yaml +30 -0
  12. data/.github/workflows/stale-actions.yml +22 -0
  13. data/.github/workflows/windows-test.yaml +46 -0
  14. data/.gitignore +30 -0
  15. data/.gitlab-ci.yml +103 -0
  16. data/ADOPTERS.md +5 -0
  17. data/AUTHORS +2 -0
  18. data/CHANGELOG.md +2409 -0
  19. data/CONTRIBUTING.md +45 -0
  20. data/GOVERNANCE.md +55 -0
  21. data/Gemfile +9 -0
  22. data/GithubWorkflow.md +78 -0
  23. data/LICENSE +202 -0
  24. data/MAINTAINERS.md +11 -0
  25. data/README.md +97 -0
  26. data/Rakefile +79 -0
  27. data/SECURITY.md +18 -0
  28. data/bin/fluent-binlog-reader +7 -0
  29. data/bin/fluent-ca-generate +6 -0
  30. data/bin/fluent-cap-ctl +7 -0
  31. data/bin/fluent-cat +5 -0
  32. data/bin/fluent-ctl +7 -0
  33. data/bin/fluent-debug +5 -0
  34. data/bin/fluent-gem +9 -0
  35. data/bin/fluent-plugin-config-format +5 -0
  36. data/bin/fluent-plugin-generate +5 -0
  37. data/bin/fluentd +15 -0
  38. data/code-of-conduct.md +3 -0
  39. data/docs/SECURITY_AUDIT.pdf +0 -0
  40. data/example/copy_roundrobin.conf +39 -0
  41. data/example/counter.conf +18 -0
  42. data/example/filter_stdout.conf +22 -0
  43. data/example/in_forward.conf +14 -0
  44. data/example/in_forward_client.conf +37 -0
  45. data/example/in_forward_shared_key.conf +15 -0
  46. data/example/in_forward_tls.conf +14 -0
  47. data/example/in_forward_users.conf +24 -0
  48. data/example/in_forward_workers.conf +21 -0
  49. data/example/in_http.conf +16 -0
  50. data/example/in_out_forward.conf +17 -0
  51. data/example/in_sample_blocks.conf +17 -0
  52. data/example/in_sample_with_compression.conf +23 -0
  53. data/example/in_syslog.conf +15 -0
  54. data/example/in_tail.conf +14 -0
  55. data/example/in_tcp.conf +13 -0
  56. data/example/in_udp.conf +13 -0
  57. data/example/logevents.conf +25 -0
  58. data/example/multi_filters.conf +61 -0
  59. data/example/out_copy.conf +20 -0
  60. data/example/out_exec_filter.conf +42 -0
  61. data/example/out_file.conf +13 -0
  62. data/example/out_forward.conf +35 -0
  63. data/example/out_forward_buf_file.conf +23 -0
  64. data/example/out_forward_client.conf +109 -0
  65. data/example/out_forward_heartbeat_none.conf +16 -0
  66. data/example/out_forward_sd.conf +17 -0
  67. data/example/out_forward_shared_key.conf +36 -0
  68. data/example/out_forward_tls.conf +18 -0
  69. data/example/out_forward_users.conf +65 -0
  70. data/example/out_null.conf +36 -0
  71. data/example/sd.yaml +8 -0
  72. data/example/secondary_file.conf +42 -0
  73. data/example/suppress_config_dump.conf +7 -0
  74. data/example/v0_12_filter.conf +78 -0
  75. data/example/v1_literal_example.conf +36 -0
  76. data/example/worker_section.conf +36 -0
  77. data/fluent.conf +139 -0
  78. data/fluentd.gemspec +55 -0
  79. data/lib/fluent/agent.rb +168 -0
  80. data/lib/fluent/capability.rb +87 -0
  81. data/lib/fluent/clock.rb +66 -0
  82. data/lib/fluent/command/binlog_reader.rb +244 -0
  83. data/lib/fluent/command/bundler_injection.rb +45 -0
  84. data/lib/fluent/command/ca_generate.rb +184 -0
  85. data/lib/fluent/command/cap_ctl.rb +174 -0
  86. data/lib/fluent/command/cat.rb +365 -0
  87. data/lib/fluent/command/ctl.rb +177 -0
  88. data/lib/fluent/command/debug.rb +103 -0
  89. data/lib/fluent/command/fluentd.rb +374 -0
  90. data/lib/fluent/command/plugin_config_formatter.rb +308 -0
  91. data/lib/fluent/command/plugin_generator.rb +365 -0
  92. data/lib/fluent/compat/call_super_mixin.rb +76 -0
  93. data/lib/fluent/compat/detach_process_mixin.rb +33 -0
  94. data/lib/fluent/compat/exec_util.rb +129 -0
  95. data/lib/fluent/compat/file_util.rb +54 -0
  96. data/lib/fluent/compat/filter.rb +68 -0
  97. data/lib/fluent/compat/formatter.rb +111 -0
  98. data/lib/fluent/compat/formatter_utils.rb +85 -0
  99. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  100. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  101. data/lib/fluent/compat/input.rb +49 -0
  102. data/lib/fluent/compat/output.rb +721 -0
  103. data/lib/fluent/compat/output_chain.rb +60 -0
  104. data/lib/fluent/compat/parser.rb +310 -0
  105. data/lib/fluent/compat/parser_utils.rb +40 -0
  106. data/lib/fluent/compat/propagate_default.rb +62 -0
  107. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  108. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  109. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  110. data/lib/fluent/compat/socket_util.rb +165 -0
  111. data/lib/fluent/compat/string_util.rb +34 -0
  112. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  113. data/lib/fluent/compat/type_converter.rb +90 -0
  114. data/lib/fluent/config/basic_parser.rb +123 -0
  115. data/lib/fluent/config/configure_proxy.rb +424 -0
  116. data/lib/fluent/config/dsl.rb +152 -0
  117. data/lib/fluent/config/element.rb +265 -0
  118. data/lib/fluent/config/error.rb +32 -0
  119. data/lib/fluent/config/literal_parser.rb +286 -0
  120. data/lib/fluent/config/parser.rb +107 -0
  121. data/lib/fluent/config/section.rb +272 -0
  122. data/lib/fluent/config/types.rb +249 -0
  123. data/lib/fluent/config/v1_parser.rb +192 -0
  124. data/lib/fluent/config.rb +76 -0
  125. data/lib/fluent/configurable.rb +201 -0
  126. data/lib/fluent/counter/base_socket.rb +44 -0
  127. data/lib/fluent/counter/client.rb +297 -0
  128. data/lib/fluent/counter/error.rb +86 -0
  129. data/lib/fluent/counter/mutex_hash.rb +163 -0
  130. data/lib/fluent/counter/server.rb +273 -0
  131. data/lib/fluent/counter/store.rb +205 -0
  132. data/lib/fluent/counter/validator.rb +145 -0
  133. data/lib/fluent/counter.rb +23 -0
  134. data/lib/fluent/daemon.rb +15 -0
  135. data/lib/fluent/daemonizer.rb +88 -0
  136. data/lib/fluent/engine.rb +253 -0
  137. data/lib/fluent/env.rb +40 -0
  138. data/lib/fluent/error.rb +34 -0
  139. data/lib/fluent/event.rb +326 -0
  140. data/lib/fluent/event_router.rb +297 -0
  141. data/lib/fluent/ext_monitor_require.rb +28 -0
  142. data/lib/fluent/filter.rb +21 -0
  143. data/lib/fluent/fluent_log_event_router.rb +141 -0
  144. data/lib/fluent/formatter.rb +23 -0
  145. data/lib/fluent/input.rb +21 -0
  146. data/lib/fluent/label.rb +46 -0
  147. data/lib/fluent/load.rb +34 -0
  148. data/lib/fluent/log.rb +713 -0
  149. data/lib/fluent/match.rb +187 -0
  150. data/lib/fluent/mixin.rb +31 -0
  151. data/lib/fluent/msgpack_factory.rb +106 -0
  152. data/lib/fluent/oj_options.rb +62 -0
  153. data/lib/fluent/output.rb +29 -0
  154. data/lib/fluent/output_chain.rb +23 -0
  155. data/lib/fluent/parser.rb +23 -0
  156. data/lib/fluent/plugin/bare_output.rb +104 -0
  157. data/lib/fluent/plugin/base.rb +197 -0
  158. data/lib/fluent/plugin/buf_file.rb +213 -0
  159. data/lib/fluent/plugin/buf_file_single.rb +225 -0
  160. data/lib/fluent/plugin/buf_memory.rb +34 -0
  161. data/lib/fluent/plugin/buffer/chunk.rb +240 -0
  162. data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
  163. data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
  164. data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
  165. data/lib/fluent/plugin/buffer.rb +918 -0
  166. data/lib/fluent/plugin/compressable.rb +96 -0
  167. data/lib/fluent/plugin/exec_util.rb +22 -0
  168. data/lib/fluent/plugin/file_util.rb +22 -0
  169. data/lib/fluent/plugin/file_wrapper.rb +187 -0
  170. data/lib/fluent/plugin/filter.rb +127 -0
  171. data/lib/fluent/plugin/filter_grep.rb +189 -0
  172. data/lib/fluent/plugin/filter_parser.rb +130 -0
  173. data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
  174. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  175. data/lib/fluent/plugin/formatter.rb +75 -0
  176. data/lib/fluent/plugin/formatter_csv.rb +78 -0
  177. data/lib/fluent/plugin/formatter_hash.rb +35 -0
  178. data/lib/fluent/plugin/formatter_json.rb +59 -0
  179. data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
  180. data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
  181. data/lib/fluent/plugin/formatter_out_file.rb +53 -0
  182. data/lib/fluent/plugin/formatter_single_value.rb +36 -0
  183. data/lib/fluent/plugin/formatter_stdout.rb +76 -0
  184. data/lib/fluent/plugin/formatter_tsv.rb +40 -0
  185. data/lib/fluent/plugin/in_debug_agent.rb +71 -0
  186. data/lib/fluent/plugin/in_dummy.rb +18 -0
  187. data/lib/fluent/plugin/in_exec.rb +110 -0
  188. data/lib/fluent/plugin/in_forward.rb +473 -0
  189. data/lib/fluent/plugin/in_gc_stat.rb +72 -0
  190. data/lib/fluent/plugin/in_http.rb +667 -0
  191. data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
  192. data/lib/fluent/plugin/in_object_space.rb +93 -0
  193. data/lib/fluent/plugin/in_sample.rb +141 -0
  194. data/lib/fluent/plugin/in_syslog.rb +276 -0
  195. data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
  196. data/lib/fluent/plugin/in_tail.rb +1228 -0
  197. data/lib/fluent/plugin/in_tcp.rb +181 -0
  198. data/lib/fluent/plugin/in_udp.rb +92 -0
  199. data/lib/fluent/plugin/in_unix.rb +195 -0
  200. data/lib/fluent/plugin/input.rb +75 -0
  201. data/lib/fluent/plugin/metrics.rb +119 -0
  202. data/lib/fluent/plugin/metrics_local.rb +96 -0
  203. data/lib/fluent/plugin/multi_output.rb +195 -0
  204. data/lib/fluent/plugin/out_copy.rb +120 -0
  205. data/lib/fluent/plugin/out_exec.rb +105 -0
  206. data/lib/fluent/plugin/out_exec_filter.rb +319 -0
  207. data/lib/fluent/plugin/out_file.rb +334 -0
  208. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  209. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  210. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  211. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  212. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
  213. data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
  214. data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
  215. data/lib/fluent/plugin/out_forward.rb +826 -0
  216. data/lib/fluent/plugin/out_http.rb +275 -0
  217. data/lib/fluent/plugin/out_null.rb +74 -0
  218. data/lib/fluent/plugin/out_relabel.rb +32 -0
  219. data/lib/fluent/plugin/out_roundrobin.rb +84 -0
  220. data/lib/fluent/plugin/out_secondary_file.rb +131 -0
  221. data/lib/fluent/plugin/out_stdout.rb +74 -0
  222. data/lib/fluent/plugin/out_stream.rb +130 -0
  223. data/lib/fluent/plugin/output.rb +1556 -0
  224. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  225. data/lib/fluent/plugin/parser.rb +275 -0
  226. data/lib/fluent/plugin/parser_apache.rb +28 -0
  227. data/lib/fluent/plugin/parser_apache2.rb +88 -0
  228. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  229. data/lib/fluent/plugin/parser_csv.rb +114 -0
  230. data/lib/fluent/plugin/parser_json.rb +96 -0
  231. data/lib/fluent/plugin/parser_ltsv.rb +51 -0
  232. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  233. data/lib/fluent/plugin/parser_multiline.rb +152 -0
  234. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  235. data/lib/fluent/plugin/parser_none.rb +36 -0
  236. data/lib/fluent/plugin/parser_regexp.rb +68 -0
  237. data/lib/fluent/plugin/parser_syslog.rb +496 -0
  238. data/lib/fluent/plugin/parser_tsv.rb +42 -0
  239. data/lib/fluent/plugin/sd_file.rb +156 -0
  240. data/lib/fluent/plugin/sd_srv.rb +135 -0
  241. data/lib/fluent/plugin/sd_static.rb +58 -0
  242. data/lib/fluent/plugin/service_discovery.rb +65 -0
  243. data/lib/fluent/plugin/socket_util.rb +22 -0
  244. data/lib/fluent/plugin/storage.rb +84 -0
  245. data/lib/fluent/plugin/storage_local.rb +162 -0
  246. data/lib/fluent/plugin/string_util.rb +22 -0
  247. data/lib/fluent/plugin.rb +206 -0
  248. data/lib/fluent/plugin_helper/cert_option.rb +191 -0
  249. data/lib/fluent/plugin_helper/child_process.rb +366 -0
  250. data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
  251. data/lib/fluent/plugin_helper/counter.rb +51 -0
  252. data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
  253. data/lib/fluent/plugin_helper/event_loop.rb +170 -0
  254. data/lib/fluent/plugin_helper/extract.rb +104 -0
  255. data/lib/fluent/plugin_helper/formatter.rb +147 -0
  256. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  257. data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
  258. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  259. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  260. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  261. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  262. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  263. data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
  264. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  265. data/lib/fluent/plugin_helper/http_server.rb +135 -0
  266. data/lib/fluent/plugin_helper/inject.rb +154 -0
  267. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  268. data/lib/fluent/plugin_helper/parser.rb +147 -0
  269. data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
  270. data/lib/fluent/plugin_helper/retry_state.rb +209 -0
  271. data/lib/fluent/plugin_helper/server.rb +801 -0
  272. data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
  273. data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
  274. data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
  275. data/lib/fluent/plugin_helper/socket.rb +277 -0
  276. data/lib/fluent/plugin_helper/socket_option.rb +98 -0
  277. data/lib/fluent/plugin_helper/storage.rb +349 -0
  278. data/lib/fluent/plugin_helper/thread.rb +180 -0
  279. data/lib/fluent/plugin_helper/timer.rb +92 -0
  280. data/lib/fluent/plugin_helper.rb +75 -0
  281. data/lib/fluent/plugin_id.rb +93 -0
  282. data/lib/fluent/process.rb +22 -0
  283. data/lib/fluent/registry.rb +116 -0
  284. data/lib/fluent/root_agent.rb +372 -0
  285. data/lib/fluent/rpc.rb +94 -0
  286. data/lib/fluent/static_config_analysis.rb +194 -0
  287. data/lib/fluent/supervisor.rb +1054 -0
  288. data/lib/fluent/system_config.rb +187 -0
  289. data/lib/fluent/test/base.rb +78 -0
  290. data/lib/fluent/test/driver/base.rb +225 -0
  291. data/lib/fluent/test/driver/base_owned.rb +83 -0
  292. data/lib/fluent/test/driver/base_owner.rb +135 -0
  293. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  294. data/lib/fluent/test/driver/filter.rb +57 -0
  295. data/lib/fluent/test/driver/formatter.rb +30 -0
  296. data/lib/fluent/test/driver/input.rb +31 -0
  297. data/lib/fluent/test/driver/multi_output.rb +53 -0
  298. data/lib/fluent/test/driver/output.rb +102 -0
  299. data/lib/fluent/test/driver/parser.rb +30 -0
  300. data/lib/fluent/test/driver/storage.rb +30 -0
  301. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  302. data/lib/fluent/test/filter_test.rb +77 -0
  303. data/lib/fluent/test/formatter_test.rb +65 -0
  304. data/lib/fluent/test/helpers.rb +134 -0
  305. data/lib/fluent/test/input_test.rb +174 -0
  306. data/lib/fluent/test/log.rb +79 -0
  307. data/lib/fluent/test/output_test.rb +156 -0
  308. data/lib/fluent/test/parser_test.rb +70 -0
  309. data/lib/fluent/test/startup_shutdown.rb +46 -0
  310. data/lib/fluent/test.rb +58 -0
  311. data/lib/fluent/time.rb +512 -0
  312. data/lib/fluent/timezone.rb +171 -0
  313. data/lib/fluent/tls.rb +81 -0
  314. data/lib/fluent/unique_id.rb +39 -0
  315. data/lib/fluent/variable_store.rb +40 -0
  316. data/lib/fluent/version.rb +21 -0
  317. data/lib/fluent/winsvc.rb +103 -0
  318. data/templates/new_gem/Gemfile +3 -0
  319. data/templates/new_gem/README.md.erb +43 -0
  320. data/templates/new_gem/Rakefile +13 -0
  321. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  322. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  323. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  324. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  325. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  326. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  327. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  328. data/templates/new_gem/test/helper.rb.erb +8 -0
  329. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  330. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  331. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  332. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  333. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  334. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  335. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  336. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  337. data/templates/plugin_config_formatter/param.md.erb +34 -0
  338. data/templates/plugin_config_formatter/section.md.erb +12 -0
  339. data/test/command/test_binlog_reader.rb +362 -0
  340. data/test/command/test_ca_generate.rb +70 -0
  341. data/test/command/test_cap_ctl.rb +100 -0
  342. data/test/command/test_cat.rb +128 -0
  343. data/test/command/test_ctl.rb +57 -0
  344. data/test/command/test_fluentd.rb +1106 -0
  345. data/test/command/test_plugin_config_formatter.rb +398 -0
  346. data/test/command/test_plugin_generator.rb +109 -0
  347. data/test/compat/test_calls_super.rb +166 -0
  348. data/test/compat/test_parser.rb +92 -0
  349. data/test/config/assertions.rb +42 -0
  350. data/test/config/test_config_parser.rb +551 -0
  351. data/test/config/test_configurable.rb +1784 -0
  352. data/test/config/test_configure_proxy.rb +604 -0
  353. data/test/config/test_dsl.rb +415 -0
  354. data/test/config/test_element.rb +518 -0
  355. data/test/config/test_literal_parser.rb +309 -0
  356. data/test/config/test_plugin_configuration.rb +56 -0
  357. data/test/config/test_section.rb +191 -0
  358. data/test/config/test_system_config.rb +199 -0
  359. data/test/config/test_types.rb +408 -0
  360. data/test/counter/test_client.rb +563 -0
  361. data/test/counter/test_error.rb +44 -0
  362. data/test/counter/test_mutex_hash.rb +179 -0
  363. data/test/counter/test_server.rb +589 -0
  364. data/test/counter/test_store.rb +258 -0
  365. data/test/counter/test_validator.rb +137 -0
  366. data/test/helper.rb +155 -0
  367. data/test/helpers/fuzzy_assert.rb +89 -0
  368. data/test/helpers/process_extenstion.rb +33 -0
  369. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  370. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  371. data/test/plugin/data/2010/01/20100102.log +0 -0
  372. data/test/plugin/data/log/bar +0 -0
  373. data/test/plugin/data/log/foo/bar.log +0 -0
  374. data/test/plugin/data/log/foo/bar2 +0 -0
  375. data/test/plugin/data/log/test.log +0 -0
  376. data/test/plugin/data/sd_file/config +11 -0
  377. data/test/plugin/data/sd_file/config.json +17 -0
  378. data/test/plugin/data/sd_file/config.yaml +11 -0
  379. data/test/plugin/data/sd_file/config.yml +11 -0
  380. data/test/plugin/data/sd_file/invalid_config.yml +7 -0
  381. data/test/plugin/in_tail/test_fifo.rb +121 -0
  382. data/test/plugin/in_tail/test_io_handler.rb +140 -0
  383. data/test/plugin/in_tail/test_position_file.rb +379 -0
  384. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  385. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  386. data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
  387. data/test/plugin/out_forward/test_load_balancer.rb +106 -0
  388. data/test/plugin/out_forward/test_socket_cache.rb +149 -0
  389. data/test/plugin/test_bare_output.rb +131 -0
  390. data/test/plugin/test_base.rb +115 -0
  391. data/test/plugin/test_buf_file.rb +1275 -0
  392. data/test/plugin/test_buf_file_single.rb +833 -0
  393. data/test/plugin/test_buf_memory.rb +42 -0
  394. data/test/plugin/test_buffer.rb +1383 -0
  395. data/test/plugin/test_buffer_chunk.rb +198 -0
  396. data/test/plugin/test_buffer_file_chunk.rb +871 -0
  397. data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
  398. data/test/plugin/test_buffer_memory_chunk.rb +339 -0
  399. data/test/plugin/test_compressable.rb +87 -0
  400. data/test/plugin/test_file_util.rb +96 -0
  401. data/test/plugin/test_file_wrapper.rb +126 -0
  402. data/test/plugin/test_filter.rb +368 -0
  403. data/test/plugin/test_filter_grep.rb +697 -0
  404. data/test/plugin/test_filter_parser.rb +731 -0
  405. data/test/plugin/test_filter_record_transformer.rb +577 -0
  406. data/test/plugin/test_filter_stdout.rb +207 -0
  407. data/test/plugin/test_formatter_csv.rb +136 -0
  408. data/test/plugin/test_formatter_hash.rb +38 -0
  409. data/test/plugin/test_formatter_json.rb +61 -0
  410. data/test/plugin/test_formatter_ltsv.rb +70 -0
  411. data/test/plugin/test_formatter_msgpack.rb +28 -0
  412. data/test/plugin/test_formatter_out_file.rb +116 -0
  413. data/test/plugin/test_formatter_single_value.rb +44 -0
  414. data/test/plugin/test_formatter_tsv.rb +76 -0
  415. data/test/plugin/test_in_debug_agent.rb +49 -0
  416. data/test/plugin/test_in_exec.rb +261 -0
  417. data/test/plugin/test_in_forward.rb +1180 -0
  418. data/test/plugin/test_in_gc_stat.rb +62 -0
  419. data/test/plugin/test_in_http.rb +1080 -0
  420. data/test/plugin/test_in_monitor_agent.rb +923 -0
  421. data/test/plugin/test_in_object_space.rb +60 -0
  422. data/test/plugin/test_in_sample.rb +190 -0
  423. data/test/plugin/test_in_syslog.rb +505 -0
  424. data/test/plugin/test_in_tail.rb +2363 -0
  425. data/test/plugin/test_in_tcp.rb +243 -0
  426. data/test/plugin/test_in_udp.rb +268 -0
  427. data/test/plugin/test_in_unix.rb +181 -0
  428. data/test/plugin/test_input.rb +137 -0
  429. data/test/plugin/test_metadata.rb +89 -0
  430. data/test/plugin/test_metrics.rb +294 -0
  431. data/test/plugin/test_metrics_local.rb +96 -0
  432. data/test/plugin/test_multi_output.rb +204 -0
  433. data/test/plugin/test_out_copy.rb +308 -0
  434. data/test/plugin/test_out_exec.rb +312 -0
  435. data/test/plugin/test_out_exec_filter.rb +606 -0
  436. data/test/plugin/test_out_file.rb +1037 -0
  437. data/test/plugin/test_out_forward.rb +1348 -0
  438. data/test/plugin/test_out_http.rb +428 -0
  439. data/test/plugin/test_out_null.rb +105 -0
  440. data/test/plugin/test_out_relabel.rb +28 -0
  441. data/test/plugin/test_out_roundrobin.rb +146 -0
  442. data/test/plugin/test_out_secondary_file.rb +458 -0
  443. data/test/plugin/test_out_stdout.rb +205 -0
  444. data/test/plugin/test_out_stream.rb +103 -0
  445. data/test/plugin/test_output.rb +1065 -0
  446. data/test/plugin/test_output_as_buffered.rb +2024 -0
  447. data/test/plugin/test_output_as_buffered_backup.rb +363 -0
  448. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  449. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  450. data/test/plugin/test_output_as_buffered_retries.rb +919 -0
  451. data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
  452. data/test/plugin/test_output_as_standard.rb +374 -0
  453. data/test/plugin/test_owned_by.rb +35 -0
  454. data/test/plugin/test_parser.rb +399 -0
  455. data/test/plugin/test_parser_apache.rb +42 -0
  456. data/test/plugin/test_parser_apache2.rb +47 -0
  457. data/test/plugin/test_parser_apache_error.rb +45 -0
  458. data/test/plugin/test_parser_csv.rb +200 -0
  459. data/test/plugin/test_parser_json.rb +138 -0
  460. data/test/plugin/test_parser_labeled_tsv.rb +160 -0
  461. data/test/plugin/test_parser_multiline.rb +111 -0
  462. data/test/plugin/test_parser_nginx.rb +88 -0
  463. data/test/plugin/test_parser_none.rb +52 -0
  464. data/test/plugin/test_parser_regexp.rb +289 -0
  465. data/test/plugin/test_parser_syslog.rb +650 -0
  466. data/test/plugin/test_parser_tsv.rb +122 -0
  467. data/test/plugin/test_sd_file.rb +228 -0
  468. data/test/plugin/test_sd_srv.rb +230 -0
  469. data/test/plugin/test_storage.rb +167 -0
  470. data/test/plugin/test_storage_local.rb +335 -0
  471. data/test/plugin/test_string_util.rb +26 -0
  472. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  473. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  474. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  475. data/test/plugin_helper/data/cert/cert.pem +19 -0
  476. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  477. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  478. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  479. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  480. data/test/plugin_helper/data/cert/empty.pem +0 -0
  481. data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
  482. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  483. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  484. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  485. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  486. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  487. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  488. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  489. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  490. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  491. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  492. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  493. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  494. data/test/plugin_helper/http_server/test_app.rb +65 -0
  495. data/test/plugin_helper/http_server/test_route.rb +32 -0
  496. data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
  497. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
  498. data/test/plugin_helper/test_cert_option.rb +25 -0
  499. data/test/plugin_helper/test_child_process.rb +840 -0
  500. data/test/plugin_helper/test_compat_parameters.rb +358 -0
  501. data/test/plugin_helper/test_event_emitter.rb +80 -0
  502. data/test/plugin_helper/test_event_loop.rb +52 -0
  503. data/test/plugin_helper/test_extract.rb +194 -0
  504. data/test/plugin_helper/test_formatter.rb +255 -0
  505. data/test/plugin_helper/test_http_server_helper.rb +372 -0
  506. data/test/plugin_helper/test_inject.rb +561 -0
  507. data/test/plugin_helper/test_metrics.rb +137 -0
  508. data/test/plugin_helper/test_parser.rb +264 -0
  509. data/test/plugin_helper/test_record_accessor.rb +238 -0
  510. data/test/plugin_helper/test_retry_state.rb +442 -0
  511. data/test/plugin_helper/test_server.rb +1823 -0
  512. data/test/plugin_helper/test_service_discovery.rb +165 -0
  513. data/test/plugin_helper/test_socket.rb +146 -0
  514. data/test/plugin_helper/test_storage.rb +542 -0
  515. data/test/plugin_helper/test_thread.rb +164 -0
  516. data/test/plugin_helper/test_timer.rb +130 -0
  517. data/test/scripts/exec_script.rb +32 -0
  518. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  519. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  520. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  521. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  522. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  523. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  524. data/test/test_capability.rb +74 -0
  525. data/test/test_clock.rb +164 -0
  526. data/test/test_config.rb +202 -0
  527. data/test/test_configdsl.rb +148 -0
  528. data/test/test_daemonizer.rb +91 -0
  529. data/test/test_engine.rb +203 -0
  530. data/test/test_event.rb +531 -0
  531. data/test/test_event_router.rb +331 -0
  532. data/test/test_event_time.rb +199 -0
  533. data/test/test_filter.rb +121 -0
  534. data/test/test_fluent_log_event_router.rb +99 -0
  535. data/test/test_formatter.rb +366 -0
  536. data/test/test_input.rb +31 -0
  537. data/test/test_log.rb +994 -0
  538. data/test/test_logger_initializer.rb +46 -0
  539. data/test/test_match.rb +148 -0
  540. data/test/test_mixin.rb +351 -0
  541. data/test/test_msgpack_factory.rb +18 -0
  542. data/test/test_oj_options.rb +55 -0
  543. data/test/test_output.rb +278 -0
  544. data/test/test_plugin.rb +251 -0
  545. data/test/test_plugin_classes.rb +370 -0
  546. data/test/test_plugin_helper.rb +81 -0
  547. data/test/test_plugin_id.rb +119 -0
  548. data/test/test_process.rb +14 -0
  549. data/test/test_root_agent.rb +951 -0
  550. data/test/test_static_config_analysis.rb +177 -0
  551. data/test/test_supervisor.rb +601 -0
  552. data/test/test_test_drivers.rb +136 -0
  553. data/test/test_time_formatter.rb +301 -0
  554. data/test/test_time_parser.rb +362 -0
  555. data/test/test_tls.rb +65 -0
  556. data/test/test_unique_id.rb +47 -0
  557. data/test/test_variable_store.rb +65 -0
  558. metadata +1261 -0
data/fluentd.gemspec ADDED
@@ -0,0 +1,55 @@
1
+ require File.expand_path('../lib/fluent/version', __FILE__)
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.name = "fluentd"
5
+ gem.version = Fluent::VERSION # see lib/fluent/version.rb
6
+
7
+ gem.authors = ["Sadayuki Furuhashi"]
8
+ gem.email = ["frsyuki@gmail.com"]
9
+ gem.description = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
10
+ gem.summary = %q{Fluentd event collector}
11
+ gem.homepage = "https://www.fluentd.org/"
12
+
13
+ gem.files = `git ls-files`.split($\)
14
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.require_paths = ["lib"]
17
+ gem.license = "Apache-2.0"
18
+
19
+ gem.required_ruby_version = '>= 2.4'
20
+
21
+ gem.add_runtime_dependency("bundler")
22
+ gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
23
+ gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
+ gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
+ gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
26
+ gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
27
+ gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
+ gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
+ gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
+ gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
31
+ gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
32
+
33
+ # build gem for a certain platform. see also Rakefile
34
+ fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
35
+ gem.platform = fake_platform unless fake_platform.empty?
36
+ if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
37
+ gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
38
+ gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
39
+ gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
40
+ gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
41
+ gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
42
+ gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
43
+ end
44
+
45
+ gem.add_development_dependency("rake", ["~> 13.0"])
46
+ gem.add_development_dependency("flexmock", ["~> 2.0"])
47
+ gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
48
+ gem.add_development_dependency("simplecov", ["~> 0.7"])
49
+ gem.add_development_dependency("rr", ["~> 3.0"])
50
+ gem.add_development_dependency("timecop", ["~> 0.9"])
51
+ gem.add_development_dependency("test-unit", ["~> 3.3"])
52
+ gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
53
+ gem.add_development_dependency("oj", [">= 2.14", "< 4"])
54
+ gem.add_development_dependency("async-http", ">= 0.50.0")
55
+ end
@@ -0,0 +1,168 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/configurable'
18
+ require 'fluent/plugin'
19
+ require 'fluent/output'
20
+ require 'fluent/match'
21
+
22
+ module Fluent
23
+ #
24
+ # Agent is a resource unit who manages emittable plugins
25
+ #
26
+ # Next step: `fluentd/root_agent.rb`
27
+ # Next step: `fluentd/label.rb`
28
+ #
29
+ class Agent
30
+ include Configurable
31
+
32
+ def initialize(log:)
33
+ super()
34
+
35
+ @context = nil
36
+ @outputs = []
37
+ @filters = []
38
+
39
+ @lifecycle_control_list = nil
40
+ # lifecycle_control_list is the list of plugins in this agent, and ordered
41
+ # from plugins which DOES emit, then DOESN'T emit
42
+ # (input -> output w/ router -> filter -> output w/o router)
43
+ # for start: use this order DESC
44
+ # (because plugins which appears later in configurations will receive events from plugins which appears earlier)
45
+ # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
46
+ @lifecycle_cache = nil
47
+
48
+ @log = log
49
+ @event_router = EventRouter.new(NoMatchMatch.new(log), self)
50
+ @error_collector = nil
51
+ end
52
+
53
+ attr_reader :log
54
+ attr_reader :outputs
55
+ attr_reader :filters
56
+ attr_reader :context
57
+ attr_reader :event_router
58
+ attr_reader :error_collector
59
+
60
+ def configure(conf)
61
+ super
62
+
63
+ # initialize <match> and <filter> elements
64
+ conf.elements('filter', 'match').each { |e|
65
+ if !Fluent::Engine.supervisor_mode && e.for_another_worker?
66
+ next
67
+ end
68
+ pattern = e.arg.empty? ? '**' : e.arg
69
+ type = e['@type']
70
+ raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
71
+ if e.name == 'filter'
72
+ add_filter(type, pattern, e)
73
+ else
74
+ add_match(type, pattern, e)
75
+ end
76
+ }
77
+ end
78
+
79
+ def lifecycle_control_list
80
+ return @lifecycle_control_list if @lifecycle_control_list
81
+
82
+ lifecycle_control_list = {
83
+ input: [],
84
+ output_with_router: [],
85
+ filter: [],
86
+ output: [],
87
+ }
88
+ if self.respond_to?(:inputs)
89
+ inputs.each do |i|
90
+ lifecycle_control_list[:input] << i
91
+ end
92
+ end
93
+ outputs.each do |o|
94
+ if o.has_router?
95
+ lifecycle_control_list[:output_with_router] << o
96
+ else
97
+ lifecycle_control_list[:output] << o
98
+ end
99
+ end
100
+ filters.each do |f|
101
+ lifecycle_control_list[:filter] << f
102
+ end
103
+
104
+ @lifecycle_control_list = lifecycle_control_list
105
+ end
106
+
107
+ def lifecycle(desc: false)
108
+ kind_list = if desc
109
+ [:output, :filter, :output_with_router]
110
+ else
111
+ [:output_with_router, :filter, :output]
112
+ end
113
+ kind_list.each do |kind|
114
+ list = if desc
115
+ lifecycle_control_list[kind].reverse
116
+ else
117
+ lifecycle_control_list[kind]
118
+ end
119
+ display_kind = (kind == :output_with_router ? :output : kind)
120
+ list.each do |instance|
121
+ yield instance, display_kind
122
+ end
123
+ end
124
+ end
125
+
126
+ def add_match(type, pattern, conf)
127
+ log_type = conf.for_this_worker? ? :default : :worker0
128
+ log.info log_type, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
129
+
130
+ output = Plugin.new_output(type)
131
+ output.context_router = @event_router
132
+ output.configure(conf)
133
+ @outputs << output
134
+ if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
135
+ # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
136
+ outputs = if output.respond_to?(:static_outputs)
137
+ output.static_outputs
138
+ else
139
+ output.outputs
140
+ end
141
+ @outputs.push(*outputs)
142
+ end
143
+ @event_router.add_rule(pattern, output)
144
+
145
+ output
146
+ end
147
+
148
+ def add_filter(type, pattern, conf)
149
+ log_type = conf.for_this_worker? ? :default : :worker0
150
+ log.info log_type, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
151
+
152
+ filter = Plugin.new_filter(type)
153
+ filter.context_router = @event_router
154
+ filter.configure(conf)
155
+ @filters << filter
156
+ @event_router.add_rule(pattern, filter)
157
+
158
+ filter
159
+ end
160
+
161
+ # For handling invalid record
162
+ def emit_error_event(tag, time, record, error)
163
+ end
164
+
165
+ def handle_emits_error(tag, es, error)
166
+ end
167
+ end
168
+ end
@@ -0,0 +1,87 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "fluent/env"
18
+
19
+ if Fluent.linux?
20
+ begin
21
+ require 'capng'
22
+ rescue LoadError
23
+ end
24
+ end
25
+
26
+ module Fluent
27
+ if defined?(CapNG)
28
+ class Capability
29
+ def initialize(target = nil, pid = nil)
30
+ @capng = CapNG.new(target, pid)
31
+ end
32
+
33
+ def usable?
34
+ true
35
+ end
36
+
37
+ def apply(select_set)
38
+ @capng.apply(select_set)
39
+ end
40
+
41
+ def clear(select_set)
42
+ @capng.clear(select_set)
43
+ end
44
+
45
+ def have_capability?(type, capability)
46
+ @capng.have_capability?(type, capability)
47
+ end
48
+
49
+ def update(action, type, capability_or_capability_array)
50
+ @capng.update(action, type, capability_or_capability_array)
51
+ end
52
+
53
+ def have_capabilities?(select_set)
54
+ @capng.have_capabilities?(select_set)
55
+ end
56
+ end
57
+ else
58
+ class Capability
59
+ def initialize(target = nil, pid = nil)
60
+ end
61
+
62
+ def usable?
63
+ false
64
+ end
65
+
66
+ def apply(select_set)
67
+ false
68
+ end
69
+
70
+ def clear(select_set)
71
+ false
72
+ end
73
+
74
+ def have_capability?(type, capability)
75
+ false
76
+ end
77
+
78
+ def update(action, type, capability_or_capability_array)
79
+ false
80
+ end
81
+
82
+ def have_capabilities?(select_set)
83
+ false
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,66 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module Clock
19
+ CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
20
+
21
+ @@block_level = 0
22
+ @@frozen_clock = nil
23
+
24
+ def self.now
25
+ @@frozen_clock || now_raw
26
+ end
27
+
28
+ def self.freeze(dst = nil, &block)
29
+ return freeze_block(dst, &block) if block_given?
30
+
31
+ dst = dst_clock_from_time(dst) if dst.is_a?(Time)
32
+ @@frozen_clock = dst || now_raw
33
+ end
34
+
35
+ def self.return
36
+ raise "invalid return while running code in blocks" if @@block_level > 0
37
+ @@frozen_clock = nil
38
+ end
39
+
40
+ # internal use
41
+
42
+ def self.now_raw
43
+ Process.clock_gettime(CLOCK_ID)
44
+ end
45
+
46
+ def self.real_now(unit = :second)
47
+ Process.clock_gettime(Process::CLOCK_REALTIME, unit)
48
+ end
49
+
50
+ def self.dst_clock_from_time(time)
51
+ diff_sec = Time.now - time
52
+ now_raw - diff_sec
53
+ end
54
+
55
+ def self.freeze_block(dst)
56
+ dst = dst_clock_from_time(dst) if dst.is_a?(Time)
57
+ pre_frozen_clock = @@frozen_clock
58
+ @@frozen_clock = dst || now_raw
59
+ @@block_level += 1
60
+ yield
61
+ ensure
62
+ @@block_level -= 1
63
+ @@frozen_clock = pre_frozen_clock
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,244 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'optparse'
18
+ require 'msgpack'
19
+
20
+ require 'fluent/msgpack_factory'
21
+ require 'fluent/formatter'
22
+ require 'fluent/plugin'
23
+ require 'fluent/config/element'
24
+ require 'fluent/engine'
25
+ require 'fluent/version'
26
+
27
+ class FluentBinlogReader
28
+ SUBCOMMAND = %w(cat head formats)
29
+ HELP_TEXT = <<HELP
30
+ Usage: fluent-binlog-reader <command> [<args>]
31
+
32
+ Commands of fluent-binlog-reader:
33
+ cat : Read files sequentially, writing them to standard output.
34
+ head : Display the beginning of a text file.
35
+ formats : Display plugins that you can use.
36
+
37
+ See 'fluent-binlog-reader <command> --help' for more information on a specific command.
38
+ HELP
39
+
40
+ def initialize(argv = ARGV)
41
+ @argv = argv
42
+ end
43
+
44
+ def call
45
+ command_class = BinlogReaderCommand.const_get(command)
46
+ command_class.new(@argv).call
47
+ end
48
+
49
+ private
50
+
51
+ def command
52
+ command = @argv.shift
53
+ if command
54
+ if command == '--version'
55
+ puts "#{File.basename($PROGRAM_NAME)} #{Fluent::VERSION}"
56
+ exit 0
57
+ elsif !SUBCOMMAND.include?(command)
58
+ usage "'#{command}' is not supported: Required subcommand : #{SUBCOMMAND.join(' | ')}"
59
+ end
60
+ else
61
+ usage "Required subcommand : #{SUBCOMMAND.join(' | ')}"
62
+ end
63
+
64
+ command.split('_').map(&:capitalize).join('')
65
+ end
66
+
67
+ def usage(msg = nil)
68
+ puts HELP_TEXT
69
+ puts "Error: #{msg}" if msg
70
+ exit 1
71
+ end
72
+ end
73
+
74
+ module BinlogReaderCommand
75
+ class Base
76
+ def initialize(argv = ARGV)
77
+ @argv = argv
78
+
79
+ @options = { plugin: [] }
80
+ @opt_parser = OptionParser.new do |opt|
81
+ opt.version = Fluent::VERSION
82
+ opt.separator 'Options:'
83
+
84
+ opt.on('-p DIR', '--plugin', 'add library directory path') do |v|
85
+ @options[:plugin] << v
86
+ end
87
+ end
88
+ end
89
+
90
+ def call
91
+ raise NotImplementedError, 'BUG: command MUST implement this method'
92
+ end
93
+
94
+ private
95
+
96
+ def usage(msg = nil)
97
+ puts @opt_parser.to_s
98
+ puts "Error: #{msg}" if msg
99
+ exit 1
100
+ end
101
+
102
+ def parse_options!
103
+ @opt_parser.parse!(@argv)
104
+
105
+ unless @options[:plugin].empty?
106
+ if dir = @options[:plugin].find { |d| !Dir.exist?(d) }
107
+ usage "Directory #{dir} doesn't exist"
108
+ else
109
+ @options[:plugin].each do |d|
110
+ Fluent::Plugin.add_plugin_dir(d)
111
+ end
112
+ end
113
+ end
114
+ rescue => e
115
+ usage e
116
+ end
117
+ end
118
+
119
+ module Formattable
120
+ DEFAULT_OPTIONS = {
121
+ format: :out_file
122
+ }
123
+
124
+ def initialize(argv = ARGV)
125
+ super
126
+ @options.merge!(DEFAULT_OPTIONS)
127
+ configure_option_parser
128
+ end
129
+
130
+ private
131
+
132
+ def configure_option_parser
133
+ @options.merge!(config_params: {})
134
+
135
+ @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
136
+
137
+ @opt_parser.on('-f TYPE', '--format', 'configure output format') do |v|
138
+ @options[:format] = v.to_sym
139
+ end
140
+
141
+ @opt_parser.on('-e KEY=VALUE', 'configure formatter config params') do |v|
142
+ key, value = v.split('=')
143
+ usage "#{v} is invalid. valid format is like `key=value`" unless value
144
+ @options[:config_params].merge!(key => value)
145
+ end
146
+ end
147
+
148
+ def lookup_formatter(format, params)
149
+ conf = Fluent::Config::Element.new('ROOT', '', params, [])
150
+ formatter = Fluent::Plugin.new_formatter(format)
151
+
152
+ if formatter.respond_to?(:configure)
153
+ formatter.configure(conf)
154
+ end
155
+ formatter
156
+ rescue => e
157
+ usage e
158
+ end
159
+ end
160
+
161
+ class Head < Base
162
+ include Formattable
163
+
164
+ DEFAULT_HEAD_OPTIONS = {
165
+ count: 5
166
+ }
167
+
168
+ def initialize(argv = ARGV)
169
+ super
170
+ @options.merge!(default_options)
171
+ parse_options!
172
+ end
173
+
174
+ def call
175
+ @formatter = lookup_formatter(@options[:format], @options[:config_params])
176
+
177
+ File.open(@path, 'rb') do |io|
178
+ i = 1
179
+ Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
180
+ print @formatter.format(@path, time, record) # path is used for tag
181
+ break if @options[:count] && i == @options[:count]
182
+ i += 1
183
+ end
184
+ end
185
+ end
186
+
187
+ private
188
+
189
+ def default_options
190
+ DEFAULT_HEAD_OPTIONS
191
+ end
192
+
193
+ def parse_options!
194
+ @opt_parser.on('-n COUNT', 'Set the number of lines to display') do |v|
195
+ @options[:count] = v.to_i
196
+ usage "illegal line count -- #{@options[:count]}" if @options[:count] < 1
197
+ end
198
+
199
+ super
200
+
201
+ usage 'Path is required' if @argv.empty?
202
+ @path = @argv.first
203
+ usage "#{@path} is not found" unless File.exist?(@path)
204
+ end
205
+ end
206
+
207
+ class Cat < Head
208
+ DEFAULT_CAT_OPTIONS = {
209
+ count: nil # Overwrite DEFAULT_HEAD_OPTIONS[:count]
210
+ }
211
+
212
+ def default_options
213
+ DEFAULT_CAT_OPTIONS
214
+ end
215
+ end
216
+
217
+ class Formats < Base
218
+ def initialize(argv = ARGV)
219
+ super
220
+ parse_options!
221
+ end
222
+
223
+ def call
224
+ prefix = Fluent::Plugin::FORMATTER_REGISTRY.dir_search_prefix || 'formatter_'
225
+
226
+ plugin_dirs = @options[:plugin]
227
+ unless plugin_dirs.empty?
228
+ plugin_dirs.each do |d|
229
+ Dir.glob("#{d}/#{prefix}*.rb").each do |path|
230
+ require File.absolute_path(path)
231
+ end
232
+ end
233
+ end
234
+
235
+ $LOAD_PATH.map do |lp|
236
+ Dir.glob("#{lp}/#{prefix}*.rb").each do |path|
237
+ require path
238
+ end
239
+ end
240
+
241
+ puts Fluent::Plugin::FORMATTER_REGISTRY.map.keys
242
+ end
243
+ end
244
+ end
@@ -0,0 +1,45 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'rbconfig'
18
+
19
+ if ENV['BUNDLE_BIN_PATH']
20
+ puts 'error: You seem to use `bundle exec` already.'
21
+ exit 1
22
+ else
23
+ begin
24
+ bundle_bin = Gem::Specification.find_by_name('bundler').bin_file('bundle')
25
+ rescue Gem::LoadError => e
26
+ puts "error: #{e}"
27
+ exit 1
28
+ end
29
+ ruby_bin = RbConfig.ruby
30
+ system("#{ruby_bin} #{bundle_bin} install")
31
+ unless $?.success?
32
+ exit $?.exitstatus
33
+ end
34
+
35
+ cmdline = [
36
+ ruby_bin,
37
+ bundle_bin,
38
+ 'exec',
39
+ ruby_bin,
40
+ File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
41
+ ] + ARGV
42
+
43
+ exec(*cmdline)
44
+ exit! 127
45
+ end