fluentd 1.14.4-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
data/lib/fluent/event.rb
ADDED
@@ -0,0 +1,326 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/msgpack_factory'
|
18
|
+
require 'fluent/plugin/compressable'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
class EventStream
|
22
|
+
include Enumerable
|
23
|
+
include Fluent::Plugin::Compressable
|
24
|
+
|
25
|
+
# dup does deep copy for event stream
|
26
|
+
def dup
|
27
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
28
|
+
end
|
29
|
+
|
30
|
+
def size
|
31
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
32
|
+
end
|
33
|
+
alias :length :size
|
34
|
+
|
35
|
+
def empty?
|
36
|
+
size == 0
|
37
|
+
end
|
38
|
+
|
39
|
+
# for tests
|
40
|
+
def ==(other)
|
41
|
+
other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
|
42
|
+
end
|
43
|
+
|
44
|
+
def repeatable?
|
45
|
+
false
|
46
|
+
end
|
47
|
+
|
48
|
+
def slice(index, num)
|
49
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
50
|
+
end
|
51
|
+
|
52
|
+
def each(unapcker: nil, &block)
|
53
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
54
|
+
end
|
55
|
+
|
56
|
+
def to_msgpack_stream(time_int: false, packer: nil)
|
57
|
+
return to_msgpack_stream_forced_integer(packer: packer) if time_int
|
58
|
+
out = packer || Fluent::MessagePackFactory.msgpack_packer
|
59
|
+
each {|time,record|
|
60
|
+
out.write([time,record])
|
61
|
+
}
|
62
|
+
out.full_pack
|
63
|
+
end
|
64
|
+
|
65
|
+
def to_compressed_msgpack_stream(time_int: false, packer: nil)
|
66
|
+
packed = to_msgpack_stream(time_int: time_int, packer: packer)
|
67
|
+
compress(packed)
|
68
|
+
end
|
69
|
+
|
70
|
+
def to_msgpack_stream_forced_integer(packer: nil)
|
71
|
+
out = packer || Fluent::MessagePackFactory.msgpack_packer
|
72
|
+
each {|time,record|
|
73
|
+
out.write([time.to_i,record])
|
74
|
+
}
|
75
|
+
out.full_pack
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class OneEventStream < EventStream
|
80
|
+
def initialize(time, record)
|
81
|
+
@time = time
|
82
|
+
@record = record
|
83
|
+
end
|
84
|
+
|
85
|
+
def dup
|
86
|
+
OneEventStream.new(@time, @record.dup)
|
87
|
+
end
|
88
|
+
|
89
|
+
def empty?
|
90
|
+
false
|
91
|
+
end
|
92
|
+
|
93
|
+
def size
|
94
|
+
1
|
95
|
+
end
|
96
|
+
|
97
|
+
def repeatable?
|
98
|
+
true
|
99
|
+
end
|
100
|
+
|
101
|
+
def slice(index, num)
|
102
|
+
if index > 0 || num == 0
|
103
|
+
ArrayEventStream.new([])
|
104
|
+
else
|
105
|
+
self.dup
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def each(unpacker: nil, &block)
|
110
|
+
block.call(@time, @record)
|
111
|
+
nil
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
# EventStream from entries: Array of [time, record]
|
116
|
+
#
|
117
|
+
# Use this class for many events data with a tag
|
118
|
+
# and its representation is [ [time, record], [time, record], .. ]
|
119
|
+
class ArrayEventStream < EventStream
|
120
|
+
def initialize(entries)
|
121
|
+
@entries = entries
|
122
|
+
end
|
123
|
+
|
124
|
+
def dup
|
125
|
+
entries = @entries.map{ |time, record| [time, record.dup] }
|
126
|
+
ArrayEventStream.new(entries)
|
127
|
+
end
|
128
|
+
|
129
|
+
def size
|
130
|
+
@entries.size
|
131
|
+
end
|
132
|
+
|
133
|
+
def repeatable?
|
134
|
+
true
|
135
|
+
end
|
136
|
+
|
137
|
+
def empty?
|
138
|
+
@entries.empty?
|
139
|
+
end
|
140
|
+
|
141
|
+
def slice(index, num)
|
142
|
+
ArrayEventStream.new(@entries.slice(index, num))
|
143
|
+
end
|
144
|
+
|
145
|
+
def each(unpacker: nil, &block)
|
146
|
+
@entries.each(&block)
|
147
|
+
nil
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
# EventStream from entries: numbers of pairs of time and record.
|
152
|
+
#
|
153
|
+
# This class can handle many events more efficiently than ArrayEventStream
|
154
|
+
# because this class generate less objects than ArrayEventStream.
|
155
|
+
#
|
156
|
+
# Use this class as below, in loop of data-enumeration:
|
157
|
+
# 1. initialize blank stream:
|
158
|
+
# streams[tag] ||= MultiEventStream.new
|
159
|
+
# 2. add events
|
160
|
+
# stream[tag].add(time, record)
|
161
|
+
class MultiEventStream < EventStream
|
162
|
+
def initialize(time_array = [], record_array = [])
|
163
|
+
@time_array = time_array
|
164
|
+
@record_array = record_array
|
165
|
+
end
|
166
|
+
|
167
|
+
def dup
|
168
|
+
MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
|
169
|
+
end
|
170
|
+
|
171
|
+
def size
|
172
|
+
@time_array.size
|
173
|
+
end
|
174
|
+
|
175
|
+
def add(time, record)
|
176
|
+
@time_array << time
|
177
|
+
@record_array << record
|
178
|
+
end
|
179
|
+
|
180
|
+
def repeatable?
|
181
|
+
true
|
182
|
+
end
|
183
|
+
|
184
|
+
def empty?
|
185
|
+
@time_array.empty?
|
186
|
+
end
|
187
|
+
|
188
|
+
def slice(index, num)
|
189
|
+
MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
|
190
|
+
end
|
191
|
+
|
192
|
+
def each(unpacker: nil, &block)
|
193
|
+
time_array = @time_array
|
194
|
+
record_array = @record_array
|
195
|
+
for i in 0..time_array.length-1
|
196
|
+
block.call(time_array[i], record_array[i])
|
197
|
+
end
|
198
|
+
nil
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
class MessagePackEventStream < EventStream
|
203
|
+
# https://github.com/msgpack/msgpack-ruby/issues/119
|
204
|
+
|
205
|
+
# Keep cached_unpacker argument for existing plugins
|
206
|
+
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
|
207
|
+
@data = data
|
208
|
+
@size = size
|
209
|
+
@unpacked_times = unpacked_times
|
210
|
+
@unpacked_records = unpacked_records
|
211
|
+
end
|
212
|
+
|
213
|
+
def empty?
|
214
|
+
@data.empty?
|
215
|
+
end
|
216
|
+
|
217
|
+
def dup
|
218
|
+
if @unpacked_times
|
219
|
+
self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
|
220
|
+
else
|
221
|
+
self.class.new(@data.dup, nil, @size)
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
def size
|
226
|
+
# @size is unbelievable always when @size == 0
|
227
|
+
# If the number of events is really zero, unpacking events takes very short time.
|
228
|
+
ensure_unpacked! if @size == 0
|
229
|
+
@size
|
230
|
+
end
|
231
|
+
|
232
|
+
def repeatable?
|
233
|
+
true
|
234
|
+
end
|
235
|
+
|
236
|
+
def ensure_unpacked!(unpacker: nil)
|
237
|
+
return if @unpacked_times && @unpacked_records
|
238
|
+
@unpacked_times = []
|
239
|
+
@unpacked_records = []
|
240
|
+
(unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
|
241
|
+
@unpacked_times << time
|
242
|
+
@unpacked_records << record
|
243
|
+
end
|
244
|
+
# @size should be updated always right after unpack.
|
245
|
+
# The real size of unpacked objects are correct, rather than given size.
|
246
|
+
@size = @unpacked_times.size
|
247
|
+
end
|
248
|
+
|
249
|
+
# This method returns MultiEventStream, because there are no reason
|
250
|
+
# to surve binary serialized by msgpack.
|
251
|
+
def slice(index, num)
|
252
|
+
ensure_unpacked!
|
253
|
+
MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
|
254
|
+
end
|
255
|
+
|
256
|
+
def each(unpacker: nil, &block)
|
257
|
+
ensure_unpacked!(unpacker: unpacker)
|
258
|
+
@unpacked_times.each_with_index do |time, i|
|
259
|
+
block.call(time, @unpacked_records[i])
|
260
|
+
end
|
261
|
+
nil
|
262
|
+
end
|
263
|
+
|
264
|
+
def to_msgpack_stream(time_int: false, packer: nil)
|
265
|
+
# time_int is always ignored because @data is always packed binary in this class
|
266
|
+
@data
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
class CompressedMessagePackEventStream < MessagePackEventStream
|
271
|
+
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
|
272
|
+
super
|
273
|
+
@decompressed_data = nil
|
274
|
+
@compressed_data = data
|
275
|
+
end
|
276
|
+
|
277
|
+
def empty?
|
278
|
+
ensure_decompressed!
|
279
|
+
super
|
280
|
+
end
|
281
|
+
|
282
|
+
def ensure_unpacked!(unpacker: nil)
|
283
|
+
ensure_decompressed!
|
284
|
+
super
|
285
|
+
end
|
286
|
+
|
287
|
+
def each(unpacker: nil, &block)
|
288
|
+
ensure_decompressed!
|
289
|
+
super
|
290
|
+
end
|
291
|
+
|
292
|
+
def to_msgpack_stream(time_int: false, packer: nil)
|
293
|
+
ensure_decompressed!
|
294
|
+
super
|
295
|
+
end
|
296
|
+
|
297
|
+
def to_compressed_msgpack_stream(time_int: false)
|
298
|
+
# time_int is always ignored because @data is always packed binary in this class
|
299
|
+
@compressed_data
|
300
|
+
end
|
301
|
+
|
302
|
+
private
|
303
|
+
|
304
|
+
def ensure_decompressed!
|
305
|
+
return if @decompressed_data
|
306
|
+
@data = @decompressed_data = decompress(@data)
|
307
|
+
end
|
308
|
+
end
|
309
|
+
|
310
|
+
module ChunkMessagePackEventStreamer
|
311
|
+
# chunk.extend(ChunkEventStreamer)
|
312
|
+
# => chunk.each{|time, record| ... }
|
313
|
+
def each(unpacker: nil, &block)
|
314
|
+
open do |io|
|
315
|
+
(unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
|
316
|
+
end
|
317
|
+
nil
|
318
|
+
end
|
319
|
+
alias :msgpack_each :each
|
320
|
+
|
321
|
+
def to_msgpack_stream(time_int: false, packer: nil)
|
322
|
+
# time_int is always ignored because data is already packed and written in chunk
|
323
|
+
read
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
@@ -0,0 +1,297 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/match'
|
18
|
+
require 'fluent/event'
|
19
|
+
require 'fluent/filter'
|
20
|
+
require 'fluent/msgpack_factory'
|
21
|
+
|
22
|
+
module Fluent
|
23
|
+
#
|
24
|
+
# EventRouter is responsible to route events to a collector.
|
25
|
+
#
|
26
|
+
# It has a list of MatchPattern and Collector pairs:
|
27
|
+
#
|
28
|
+
# +----------------+ +-----------------+
|
29
|
+
# | MatchPattern | | Collector |
|
30
|
+
# +----------------+ +-----------------+
|
31
|
+
# | access.** ---------> type forward |
|
32
|
+
# | logs.** ---------> type copy |
|
33
|
+
# | archive.** ---------> type s3 |
|
34
|
+
# +----------------+ +-----------------+
|
35
|
+
#
|
36
|
+
# EventRouter does:
|
37
|
+
#
|
38
|
+
# 1) receive an event at `#emit` methods
|
39
|
+
# 2) match the event's tag with the MatchPatterns
|
40
|
+
# 3) forward the event to the corresponding Collector
|
41
|
+
#
|
42
|
+
# Collector is either of Output, Filter or other EventRouter.
|
43
|
+
#
|
44
|
+
class EventRouter
|
45
|
+
def initialize(default_collector, emit_error_handler)
|
46
|
+
@match_rules = []
|
47
|
+
@match_cache = MatchCache.new
|
48
|
+
@default_collector = default_collector
|
49
|
+
@emit_error_handler = emit_error_handler
|
50
|
+
@metric_callbacks = {}
|
51
|
+
@caller_plugin_id = nil
|
52
|
+
end
|
53
|
+
|
54
|
+
attr_accessor :default_collector
|
55
|
+
attr_accessor :emit_error_handler
|
56
|
+
|
57
|
+
class Rule
|
58
|
+
def initialize(pattern, collector)
|
59
|
+
patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
|
60
|
+
@pattern = if patterns.length == 1
|
61
|
+
patterns[0]
|
62
|
+
else
|
63
|
+
OrMatchPattern.new(patterns)
|
64
|
+
end
|
65
|
+
@pattern_str = pattern
|
66
|
+
@collector = collector
|
67
|
+
end
|
68
|
+
|
69
|
+
def match?(tag)
|
70
|
+
@pattern.match(tag)
|
71
|
+
end
|
72
|
+
|
73
|
+
attr_reader :collector
|
74
|
+
attr_reader :pattern_str
|
75
|
+
end
|
76
|
+
|
77
|
+
def suppress_missing_match!
|
78
|
+
if @default_collector.respond_to?(:suppress_missing_match!)
|
79
|
+
@default_collector.suppress_missing_match!
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# called by Agent to add new match pattern and collector
|
84
|
+
def add_rule(pattern, collector)
|
85
|
+
@match_rules << Rule.new(pattern, collector)
|
86
|
+
end
|
87
|
+
|
88
|
+
def add_metric_callbacks(caller_plugin_id, callback)
|
89
|
+
@metric_callbacks[caller_plugin_id] = callback
|
90
|
+
end
|
91
|
+
|
92
|
+
def caller_plugin_id=(caller_plugin_id)
|
93
|
+
@caller_plugin_id = caller_plugin_id
|
94
|
+
end
|
95
|
+
|
96
|
+
def find_callback
|
97
|
+
if @caller_plugin_id
|
98
|
+
@metric_callbacks[@caller_plugin_id]
|
99
|
+
else
|
100
|
+
nil
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def emit(tag, time, record)
|
105
|
+
unless record.nil?
|
106
|
+
emit_stream(tag, OneEventStream.new(time, record))
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def emit_array(tag, array)
|
111
|
+
emit_stream(tag, ArrayEventStream.new(array))
|
112
|
+
end
|
113
|
+
|
114
|
+
def emit_stream(tag, es)
|
115
|
+
match(tag).emit_events(tag, es)
|
116
|
+
if callback = find_callback
|
117
|
+
callback.call(es)
|
118
|
+
end
|
119
|
+
rescue => e
|
120
|
+
@emit_error_handler.handle_emits_error(tag, es, e)
|
121
|
+
end
|
122
|
+
|
123
|
+
def emit_error_event(tag, time, record, error)
|
124
|
+
@emit_error_handler.emit_error_event(tag, time, record, error)
|
125
|
+
end
|
126
|
+
|
127
|
+
def match?(tag)
|
128
|
+
!!find(tag)
|
129
|
+
end
|
130
|
+
|
131
|
+
def match(tag)
|
132
|
+
collector = @match_cache.get(tag) {
|
133
|
+
find(tag) || @default_collector
|
134
|
+
}
|
135
|
+
collector
|
136
|
+
end
|
137
|
+
|
138
|
+
class MatchCache
|
139
|
+
MATCH_CACHE_SIZE = 1024
|
140
|
+
|
141
|
+
def initialize
|
142
|
+
super
|
143
|
+
@map = {}
|
144
|
+
@keys = []
|
145
|
+
end
|
146
|
+
|
147
|
+
def get(key)
|
148
|
+
if collector = @map[key]
|
149
|
+
return collector
|
150
|
+
end
|
151
|
+
collector = @map[key] = yield
|
152
|
+
if @keys.size >= MATCH_CACHE_SIZE
|
153
|
+
# expire the oldest key
|
154
|
+
@map.delete @keys.shift
|
155
|
+
end
|
156
|
+
@keys << key
|
157
|
+
collector
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
private
|
162
|
+
|
163
|
+
class Pipeline
|
164
|
+
def initialize
|
165
|
+
@filters = []
|
166
|
+
@output = nil
|
167
|
+
@optimizer = FilterOptimizer.new
|
168
|
+
end
|
169
|
+
|
170
|
+
def add_filter(filter)
|
171
|
+
@filters << filter
|
172
|
+
@optimizer.filters = @filters
|
173
|
+
end
|
174
|
+
|
175
|
+
def set_output(output)
|
176
|
+
@output = output
|
177
|
+
end
|
178
|
+
|
179
|
+
def emit_events(tag, es)
|
180
|
+
processed = @optimizer.filter_stream(tag, es)
|
181
|
+
@output.emit_events(tag, processed)
|
182
|
+
end
|
183
|
+
|
184
|
+
class FilterOptimizer
|
185
|
+
def initialize(filters = [])
|
186
|
+
@filters = filters
|
187
|
+
@optimizable = nil
|
188
|
+
end
|
189
|
+
|
190
|
+
def filters=(filters)
|
191
|
+
@filters = filters
|
192
|
+
reset_optimization
|
193
|
+
end
|
194
|
+
|
195
|
+
def filter_stream(tag, es)
|
196
|
+
if optimizable?
|
197
|
+
optimized_filter_stream(tag, es)
|
198
|
+
else
|
199
|
+
@filters.reduce(es) { |acc, filter|
|
200
|
+
filtered_es = filter.filter_stream(tag, acc)
|
201
|
+
filter.measure_metrics(filtered_es)
|
202
|
+
filtered_es
|
203
|
+
}
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
private
|
208
|
+
|
209
|
+
def optimized_filter_stream(tag, es)
|
210
|
+
new_es = MultiEventStream.new
|
211
|
+
es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
|
212
|
+
filtered_record = record
|
213
|
+
filtered_time = time
|
214
|
+
|
215
|
+
catch :break_loop do
|
216
|
+
@filters.each do |filter|
|
217
|
+
if filter.has_filter_with_time
|
218
|
+
begin
|
219
|
+
filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
|
220
|
+
throw :break_loop unless filtered_record && filtered_time
|
221
|
+
filter.measure_metrics(OneEventStream.new(time, record))
|
222
|
+
rescue => e
|
223
|
+
filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
|
224
|
+
end
|
225
|
+
else
|
226
|
+
begin
|
227
|
+
filtered_record = filter.filter(tag, filtered_time, filtered_record)
|
228
|
+
throw :break_loop unless filtered_record
|
229
|
+
filter.measure_metrics(OneEventStream.new(time, record))
|
230
|
+
rescue => e
|
231
|
+
filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
new_es.add(filtered_time, filtered_record)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
new_es
|
240
|
+
end
|
241
|
+
|
242
|
+
def optimizable?
|
243
|
+
return @optimizable unless @optimizable.nil?
|
244
|
+
fs_filters = filters_having_filter_stream
|
245
|
+
@optimizable = if fs_filters.empty?
|
246
|
+
true
|
247
|
+
else
|
248
|
+
# skip log message when filter is only 1, because its performance is same as non optimized chain.
|
249
|
+
if @filters.size > 1 && fs_filters.size >= 1
|
250
|
+
$log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
|
251
|
+
end
|
252
|
+
false
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
def filters_having_filter_stream
|
257
|
+
@filters_having_filter_stream ||= @filters.select do |filter|
|
258
|
+
filter.class.instance_methods(false).include?(:filter_stream)
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
def reset_optimization
|
263
|
+
@optimizable = nil
|
264
|
+
@filters_having_filter_stream = nil
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def find(tag)
|
270
|
+
pipeline = nil
|
271
|
+
@match_rules.each_with_index { |rule, i|
|
272
|
+
if rule.match?(tag)
|
273
|
+
if rule.collector.is_a?(Plugin::Filter)
|
274
|
+
pipeline ||= Pipeline.new
|
275
|
+
pipeline.add_filter(rule.collector)
|
276
|
+
else
|
277
|
+
if pipeline
|
278
|
+
pipeline.set_output(rule.collector)
|
279
|
+
else
|
280
|
+
# Use Output directly when filter is not matched
|
281
|
+
pipeline = rule.collector
|
282
|
+
end
|
283
|
+
return pipeline
|
284
|
+
end
|
285
|
+
end
|
286
|
+
}
|
287
|
+
|
288
|
+
if pipeline
|
289
|
+
# filter is matched but no match
|
290
|
+
pipeline.set_output(@default_collector)
|
291
|
+
pipeline
|
292
|
+
else
|
293
|
+
nil
|
294
|
+
end
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
# To avoid duplicated requirements, extract this logic as file.
|
18
|
+
|
19
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')
|
20
|
+
require 'monitor'
|
21
|
+
else
|
22
|
+
begin
|
23
|
+
# monitor_ext is bundled since ruby 2.7.0
|
24
|
+
require 'ext_monitor'
|
25
|
+
rescue LoadError => _
|
26
|
+
require 'monitor'
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/compat/filter'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
Filter = Fluent::Compat::Filter
|
21
|
+
end
|