fluentd 1.14.4-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
@@ -0,0 +1,1556 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/env'
|
18
|
+
require 'fluent/error'
|
19
|
+
require 'fluent/plugin/base'
|
20
|
+
require 'fluent/plugin/buffer'
|
21
|
+
require 'fluent/plugin_helper/record_accessor'
|
22
|
+
require 'fluent/msgpack_factory'
|
23
|
+
require 'fluent/log'
|
24
|
+
require 'fluent/plugin_id'
|
25
|
+
require 'fluent/plugin_helper'
|
26
|
+
require 'fluent/timezone'
|
27
|
+
require 'fluent/unique_id'
|
28
|
+
require 'fluent/clock'
|
29
|
+
require 'fluent/ext_monitor_require'
|
30
|
+
|
31
|
+
require 'time'
|
32
|
+
|
33
|
+
module Fluent
|
34
|
+
module Plugin
|
35
|
+
class Output < Base
|
36
|
+
include PluginId
|
37
|
+
include PluginLoggerMixin
|
38
|
+
include PluginHelper::Mixin
|
39
|
+
include UniqueId::Mixin
|
40
|
+
|
41
|
+
helpers_internal :thread, :retry_state, :metrics
|
42
|
+
|
43
|
+
CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
|
44
|
+
CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
|
45
|
+
CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
|
46
|
+
CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
|
47
|
+
|
48
|
+
CHUNKING_FIELD_WARN_NUM = 4
|
49
|
+
|
50
|
+
config_param :time_as_integer, :bool, default: false
|
51
|
+
desc 'The threshold to show slow flush logs'
|
52
|
+
config_param :slow_flush_log_threshold, :float, default: 20.0
|
53
|
+
|
54
|
+
# `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
|
55
|
+
config_section :buffer, param_name: :buffer_config, init: true, required: false, multi: false, final: true do
|
56
|
+
config_argument :chunk_keys, :array, value_type: :string, default: []
|
57
|
+
config_param :@type, :string, default: 'memory', alias: :type
|
58
|
+
|
59
|
+
config_param :timekey, :time, default: nil # range size to be used: `time.to_i / @timekey`
|
60
|
+
config_param :timekey_wait, :time, default: 600
|
61
|
+
# These are for #extract_placeholders
|
62
|
+
config_param :timekey_use_utc, :bool, default: false # default is localtime
|
63
|
+
config_param :timekey_zone, :string, default: Time.now.strftime('%z') # e.g., "-0700" or "Asia/Tokyo"
|
64
|
+
|
65
|
+
desc 'If true, plugin will try to flush buffer just before shutdown.'
|
66
|
+
config_param :flush_at_shutdown, :bool, default: nil # change default by buffer_plugin.persistent?
|
67
|
+
|
68
|
+
desc 'How to enqueue chunks to be flushed. "interval" flushes per flush_interval, "immediate" flushes just after event arrival.'
|
69
|
+
config_param :flush_mode, :enum, list: [:default, :lazy, :interval, :immediate], default: :default
|
70
|
+
config_param :flush_interval, :time, default: 60, desc: 'The interval between buffer chunk flushes.'
|
71
|
+
|
72
|
+
config_param :flush_thread_count, :integer, default: 1, desc: 'The number of threads to flush the buffer.'
|
73
|
+
|
74
|
+
config_param :flush_thread_interval, :float, default: 1.0, desc: 'Seconds to sleep between checks for buffer flushes in flush threads.'
|
75
|
+
config_param :flush_thread_burst_interval, :float, default: 1.0, desc: 'Seconds to sleep between flushes when many buffer chunks are queued.'
|
76
|
+
|
77
|
+
config_param :delayed_commit_timeout, :time, default: 60, desc: 'Seconds of timeout for buffer chunks to be committed by plugins later.'
|
78
|
+
|
79
|
+
config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :throw_exception, desc: 'The action when the size of buffer exceeds the limit.'
|
80
|
+
|
81
|
+
config_param :retry_forever, :bool, default: false, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry flushing forever.'
|
82
|
+
config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'
|
83
|
+
# 72hours == 17 times with exponential backoff (not to change default behavior)
|
84
|
+
config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
|
85
|
+
|
86
|
+
config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
|
87
|
+
# exponential backoff sequence will be initialized at the time of this threshold
|
88
|
+
|
89
|
+
desc 'How to wait next retry to flush buffer.'
|
90
|
+
config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
|
91
|
+
### Periodic -> fixed :retry_wait
|
92
|
+
### Exponential backoff: k is number of retry times
|
93
|
+
# c: constant factor, @retry_wait
|
94
|
+
# b: base factor, @retry_exponential_backoff_base
|
95
|
+
# k: times
|
96
|
+
# total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
|
97
|
+
config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
|
98
|
+
config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
|
99
|
+
config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
|
100
|
+
|
101
|
+
config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
|
102
|
+
config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
|
103
|
+
end
|
104
|
+
|
105
|
+
config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
|
106
|
+
config_param :@type, :string, default: nil, alias: :type
|
107
|
+
config_section :buffer, required: false, multi: false do
|
108
|
+
# dummy to detect invalid specification for here
|
109
|
+
end
|
110
|
+
config_section :secondary, required: false, multi: false do
|
111
|
+
# dummy to detect invalid specification for here
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def process(tag, es)
|
116
|
+
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
117
|
+
end
|
118
|
+
|
119
|
+
def write(chunk)
|
120
|
+
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
121
|
+
end
|
122
|
+
|
123
|
+
def try_write(chunk)
|
124
|
+
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
125
|
+
end
|
126
|
+
|
127
|
+
def format(tag, time, record)
|
128
|
+
# standard msgpack_event_stream chunk will be used if this method is not implemented in plugin subclass
|
129
|
+
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
130
|
+
end
|
131
|
+
|
132
|
+
def formatted_to_msgpack_binary?
|
133
|
+
# To indicate custom format method (#format) returns msgpack binary or not.
|
134
|
+
# If #format returns msgpack binary, override this method to return true.
|
135
|
+
false
|
136
|
+
end
|
137
|
+
|
138
|
+
# Compatibility for existing plugins
|
139
|
+
def formatted_to_msgpack_binary
|
140
|
+
formatted_to_msgpack_binary?
|
141
|
+
end
|
142
|
+
|
143
|
+
def prefer_buffered_processing
|
144
|
+
# override this method to return false only when all of these are true:
|
145
|
+
# * plugin has both implementation for buffered and non-buffered methods
|
146
|
+
# * plugin is expected to work as non-buffered plugin if no `<buffer>` sections specified
|
147
|
+
true
|
148
|
+
end
|
149
|
+
|
150
|
+
def prefer_delayed_commit
|
151
|
+
# override this method to decide which is used of `write` or `try_write` if both are implemented
|
152
|
+
true
|
153
|
+
end
|
154
|
+
|
155
|
+
def multi_workers_ready?
|
156
|
+
false
|
157
|
+
end
|
158
|
+
|
159
|
+
# Internal states
|
160
|
+
FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)
|
161
|
+
DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
|
162
|
+
def expired?
|
163
|
+
time + timeout < Time.now
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
|
168
|
+
|
169
|
+
# for tests
|
170
|
+
attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
|
171
|
+
attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
|
172
|
+
# output_enqueue_thread_waiting: for test of output.rb itself
|
173
|
+
attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
|
174
|
+
|
175
|
+
def num_errors
|
176
|
+
@num_errors_metrics.get
|
177
|
+
end
|
178
|
+
|
179
|
+
def emit_count
|
180
|
+
@emit_count_metrics.get
|
181
|
+
end
|
182
|
+
|
183
|
+
def emit_size
|
184
|
+
@emit_size_metrics.get
|
185
|
+
end
|
186
|
+
|
187
|
+
def emit_records
|
188
|
+
@emit_records_metrics.get
|
189
|
+
end
|
190
|
+
|
191
|
+
def write_count
|
192
|
+
@write_count_metrics.get
|
193
|
+
end
|
194
|
+
|
195
|
+
def rollback_count
|
196
|
+
@rollback_count_metrics.get
|
197
|
+
end
|
198
|
+
|
199
|
+
def initialize
|
200
|
+
super
|
201
|
+
@counter_mutex = Mutex.new
|
202
|
+
@buffering = false
|
203
|
+
@delayed_commit = false
|
204
|
+
@as_secondary = false
|
205
|
+
@primary_instance = nil
|
206
|
+
|
207
|
+
# TODO: well organized counters
|
208
|
+
@num_errors_metrics = nil
|
209
|
+
@emit_count_metrics = nil
|
210
|
+
@emit_records_metrics = nil
|
211
|
+
@emit_size_metrics = nil
|
212
|
+
@write_count_metrics = nil
|
213
|
+
@rollback_count_metrics = nil
|
214
|
+
@flush_time_count_metrics = nil
|
215
|
+
@slow_flush_count_metrics = nil
|
216
|
+
@enable_size_metrics = false
|
217
|
+
|
218
|
+
# How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
|
219
|
+
if implement?(:synchronous)
|
220
|
+
if implement?(:buffered) || implement?(:delayed_commit)
|
221
|
+
@buffering = nil # do #configure or #start to determine this for full-featured plugins
|
222
|
+
else
|
223
|
+
@buffering = false
|
224
|
+
end
|
225
|
+
else
|
226
|
+
@buffering = true
|
227
|
+
end
|
228
|
+
@custom_format = implement?(:custom_format)
|
229
|
+
@enable_msgpack_streamer = false # decided later
|
230
|
+
|
231
|
+
@buffer = nil
|
232
|
+
@secondary = nil
|
233
|
+
@retry = nil
|
234
|
+
@dequeued_chunks = nil
|
235
|
+
@dequeued_chunks_mutex = nil
|
236
|
+
@output_enqueue_thread = nil
|
237
|
+
@output_flush_threads = nil
|
238
|
+
|
239
|
+
@simple_chunking = nil
|
240
|
+
@chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
|
241
|
+
@flush_mode = nil
|
242
|
+
@timekey_zone = nil
|
243
|
+
|
244
|
+
@retry_for_error_chunk = false
|
245
|
+
end
|
246
|
+
|
247
|
+
def acts_as_secondary(primary)
|
248
|
+
@as_secondary = true
|
249
|
+
@primary_instance = primary
|
250
|
+
@chunk_keys = @primary_instance.chunk_keys || []
|
251
|
+
@chunk_key_tag = @primary_instance.chunk_key_tag || false
|
252
|
+
if @primary_instance.chunk_key_time
|
253
|
+
@chunk_key_time = @primary_instance.chunk_key_time
|
254
|
+
@timekey_zone = @primary_instance.timekey_zone
|
255
|
+
@output_time_formatter_cache = {}
|
256
|
+
end
|
257
|
+
self.context_router = primary.context_router
|
258
|
+
|
259
|
+
singleton_class.module_eval do
|
260
|
+
define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
|
261
|
+
define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) }
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def configure(conf)
|
266
|
+
unless implement?(:synchronous) || implement?(:buffered) || implement?(:delayed_commit)
|
267
|
+
raise "BUG: output plugin must implement some methods. see developer documents."
|
268
|
+
end
|
269
|
+
|
270
|
+
has_buffer_section = (conf.elements(name: 'buffer').size > 0)
|
271
|
+
has_flush_interval = conf.has_key?('flush_interval')
|
272
|
+
|
273
|
+
super
|
274
|
+
|
275
|
+
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
|
276
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
|
277
|
+
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
|
278
|
+
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
|
279
|
+
@write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
|
280
|
+
@rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
|
281
|
+
@flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
|
282
|
+
@slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
|
283
|
+
|
284
|
+
if has_buffer_section
|
285
|
+
unless implement?(:buffered) || implement?(:delayed_commit)
|
286
|
+
raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
|
287
|
+
end
|
288
|
+
@buffering = true
|
289
|
+
else # no buffer sections
|
290
|
+
if implement?(:synchronous)
|
291
|
+
if !implement?(:buffered) && !implement?(:delayed_commit)
|
292
|
+
if @as_secondary
|
293
|
+
raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't."
|
294
|
+
end
|
295
|
+
@buffering = false
|
296
|
+
else
|
297
|
+
if @as_secondary
|
298
|
+
# secondary plugin always works as buffered plugin without buffer instance
|
299
|
+
@buffering = true
|
300
|
+
else
|
301
|
+
# @buffering.nil? shows that enabling buffering or not will be decided in lazy way in #start
|
302
|
+
@buffering = nil
|
303
|
+
end
|
304
|
+
end
|
305
|
+
else # buffered or delayed_commit is supported by `unless` of first line in this method
|
306
|
+
@buffering = true
|
307
|
+
end
|
308
|
+
end
|
309
|
+
# Enable to update record size metrics or not
|
310
|
+
@enable_size_metrics = !!system_config.enable_size_metrics
|
311
|
+
|
312
|
+
if @as_secondary
|
313
|
+
if !@buffering && !@buffering.nil?
|
314
|
+
raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't"
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
if (@buffering || @buffering.nil?) && !@as_secondary
|
319
|
+
# When @buffering.nil?, @buffer_config was initialized with default value for all parameters.
|
320
|
+
# If so, this configuration MUST success.
|
321
|
+
@chunk_keys = @buffer_config.chunk_keys.dup
|
322
|
+
@chunk_key_time = !!@chunk_keys.delete('time')
|
323
|
+
@chunk_key_tag = !!@chunk_keys.delete('tag')
|
324
|
+
if @chunk_keys.any? { |key|
|
325
|
+
begin
|
326
|
+
k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)
|
327
|
+
if k.is_a?(String)
|
328
|
+
k !~ CHUNK_KEY_PATTERN
|
329
|
+
else
|
330
|
+
if key.start_with?('$[')
|
331
|
+
raise Fluent::ConfigError, "in chunk_keys: bracket notation is not allowed"
|
332
|
+
else
|
333
|
+
false
|
334
|
+
end
|
335
|
+
end
|
336
|
+
rescue => e
|
337
|
+
raise Fluent::ConfigError, "in chunk_keys: #{e.message}"
|
338
|
+
end
|
339
|
+
}
|
340
|
+
raise Fluent::ConfigError, "chunk_keys specification includes invalid char"
|
341
|
+
else
|
342
|
+
@chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]
|
343
|
+
end
|
344
|
+
|
345
|
+
if @chunk_key_time
|
346
|
+
raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
|
347
|
+
Fluent::Timezone.validate!(@buffer_config.timekey_zone)
|
348
|
+
@timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
|
349
|
+
@timekey = @buffer_config.timekey
|
350
|
+
if @timekey <= 0
|
351
|
+
raise Fluent::ConfigError, "timekey should be greater than 0. current timekey: #{@timekey}"
|
352
|
+
end
|
353
|
+
@timekey_use_utc = @buffer_config.timekey_use_utc
|
354
|
+
@offset = Fluent::Timezone.utc_offset(@timekey_zone)
|
355
|
+
@calculate_offset = @offset.respond_to?(:call) ? @offset : nil
|
356
|
+
@output_time_formatter_cache = {}
|
357
|
+
end
|
358
|
+
|
359
|
+
if (@chunk_key_tag ? 1 : 0) + @chunk_keys.size >= CHUNKING_FIELD_WARN_NUM
|
360
|
+
log.warn "many chunk keys specified, and it may cause too many chunks on your system."
|
361
|
+
end
|
362
|
+
|
363
|
+
# no chunk keys or only tags (chunking can be done without iterating event stream)
|
364
|
+
@simple_chunking = !@chunk_key_time && @chunk_keys.empty?
|
365
|
+
|
366
|
+
@flush_mode = @buffer_config.flush_mode
|
367
|
+
if @flush_mode == :default
|
368
|
+
if has_flush_interval
|
369
|
+
log.info "'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour"
|
370
|
+
@flush_mode = :interval
|
371
|
+
else
|
372
|
+
@flush_mode = (@chunk_key_time ? :lazy : :interval)
|
373
|
+
end
|
374
|
+
end
|
375
|
+
|
376
|
+
buffer_type = @buffer_config[:@type]
|
377
|
+
buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
|
378
|
+
@buffer = Plugin.new_buffer(buffer_type, parent: self)
|
379
|
+
@buffer.configure(buffer_conf)
|
380
|
+
@buffer.enable_update_timekeys if @chunk_key_time
|
381
|
+
|
382
|
+
@flush_at_shutdown = @buffer_config.flush_at_shutdown
|
383
|
+
if @flush_at_shutdown.nil?
|
384
|
+
@flush_at_shutdown = if @buffer.persistent?
|
385
|
+
false
|
386
|
+
else
|
387
|
+
true # flush_at_shutdown is true in default for on-memory buffer
|
388
|
+
end
|
389
|
+
elsif !@flush_at_shutdown && !@buffer.persistent?
|
390
|
+
buf_type = Plugin.lookup_type_from_class(@buffer.class)
|
391
|
+
log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
|
392
|
+
log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
|
393
|
+
end
|
394
|
+
|
395
|
+
if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
|
396
|
+
if buffer_conf.has_key?('flush_mode')
|
397
|
+
raise Fluent::ConfigError, "'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{@flush_mode}'"
|
398
|
+
else
|
399
|
+
log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
if @buffer.queued_chunks_limit_size.nil?
|
404
|
+
@buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count
|
405
|
+
end
|
406
|
+
end
|
407
|
+
|
408
|
+
if @secondary_config
|
409
|
+
raise Fluent::ConfigError, "Invalid <secondary> section for non-buffered plugin" unless @buffering
|
410
|
+
raise Fluent::ConfigError, "<secondary> section cannot have <buffer> section" if @secondary_config.buffer
|
411
|
+
raise Fluent::ConfigError, "<secondary> section cannot have <secondary> section" if @secondary_config.secondary
|
412
|
+
if @buffer_config.retry_forever
|
413
|
+
log.warn "<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary"
|
414
|
+
end
|
415
|
+
|
416
|
+
secondary_type = @secondary_config[:@type]
|
417
|
+
unless secondary_type
|
418
|
+
secondary_type = conf['@type'] # primary plugin type
|
419
|
+
end
|
420
|
+
secondary_conf = conf.elements(name: 'secondary').first
|
421
|
+
@secondary = Plugin.new_output(secondary_type)
|
422
|
+
unless @secondary.respond_to?(:acts_as_secondary)
|
423
|
+
raise Fluent::ConfigError, "Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output"
|
424
|
+
end
|
425
|
+
@secondary.acts_as_secondary(self)
|
426
|
+
@secondary.configure(secondary_conf)
|
427
|
+
if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
|
428
|
+
log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
|
429
|
+
end
|
430
|
+
else
|
431
|
+
@secondary = nil
|
432
|
+
end
|
433
|
+
|
434
|
+
self
|
435
|
+
end
|
436
|
+
|
437
|
+
def start
|
438
|
+
super
|
439
|
+
|
440
|
+
if @buffering.nil?
|
441
|
+
@buffering = prefer_buffered_processing
|
442
|
+
if !@buffering && @buffer
|
443
|
+
@buffer.terminate # it's not started, so terminate will be enough
|
444
|
+
# At here, this plugin works as non-buffered plugin.
|
445
|
+
# Un-assign @buffer not to show buffering metrics (e.g., in_monitor_agent)
|
446
|
+
@buffer = nil
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
450
|
+
if @buffering
|
451
|
+
m = method(:emit_buffered)
|
452
|
+
singleton_class.module_eval do
|
453
|
+
define_method(:emit_events, m)
|
454
|
+
end
|
455
|
+
|
456
|
+
@custom_format = implement?(:custom_format)
|
457
|
+
@enable_msgpack_streamer = @custom_format ? formatted_to_msgpack_binary : true
|
458
|
+
@delayed_commit = if implement?(:buffered) && implement?(:delayed_commit)
|
459
|
+
prefer_delayed_commit
|
460
|
+
else
|
461
|
+
implement?(:delayed_commit)
|
462
|
+
end
|
463
|
+
@delayed_commit_timeout = @buffer_config.delayed_commit_timeout
|
464
|
+
else # !@buffering
|
465
|
+
m = method(:emit_sync)
|
466
|
+
singleton_class.module_eval do
|
467
|
+
define_method(:emit_events, m)
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
if @buffering && !@as_secondary
|
472
|
+
@retry = nil
|
473
|
+
@retry_mutex = Mutex.new
|
474
|
+
|
475
|
+
@buffer.start
|
476
|
+
|
477
|
+
@output_enqueue_thread = nil
|
478
|
+
@output_enqueue_thread_running = true
|
479
|
+
|
480
|
+
@output_flush_threads = []
|
481
|
+
@output_flush_threads_mutex = Mutex.new
|
482
|
+
@output_flush_threads_running = true
|
483
|
+
|
484
|
+
# mainly for test: detect enqueue works as code below:
|
485
|
+
# @output.interrupt_flushes
|
486
|
+
# # emits
|
487
|
+
# @output.enqueue_thread_wait
|
488
|
+
@output_flush_interrupted = false
|
489
|
+
@output_enqueue_thread_mutex = Mutex.new
|
490
|
+
@output_enqueue_thread_waiting = false
|
491
|
+
|
492
|
+
@dequeued_chunks = []
|
493
|
+
@dequeued_chunks_mutex = Mutex.new
|
494
|
+
|
495
|
+
@buffer_config.flush_thread_count.times do |i|
|
496
|
+
thread_title = "flush_thread_#{i}".to_sym
|
497
|
+
thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
|
498
|
+
thread = thread_create(thread_title) do
|
499
|
+
flush_thread_run(thread_state)
|
500
|
+
end
|
501
|
+
thread_state.thread = thread
|
502
|
+
@output_flush_threads_mutex.synchronize do
|
503
|
+
@output_flush_threads << thread_state
|
504
|
+
end
|
505
|
+
end
|
506
|
+
@output_flush_thread_current_position = 0
|
507
|
+
|
508
|
+
if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
|
509
|
+
@output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
|
510
|
+
end
|
511
|
+
end
|
512
|
+
@secondary.start if @secondary
|
513
|
+
end
|
514
|
+
|
515
|
+
def after_start
|
516
|
+
super
|
517
|
+
@secondary.after_start if @secondary
|
518
|
+
end
|
519
|
+
|
520
|
+
def stop
|
521
|
+
@secondary.stop if @secondary
|
522
|
+
@buffer.stop if @buffering && @buffer
|
523
|
+
|
524
|
+
super
|
525
|
+
end
|
526
|
+
|
527
|
+
def before_shutdown
|
528
|
+
@secondary.before_shutdown if @secondary
|
529
|
+
|
530
|
+
if @buffering && @buffer
|
531
|
+
if @flush_at_shutdown
|
532
|
+
force_flush
|
533
|
+
end
|
534
|
+
@buffer.before_shutdown
|
535
|
+
# Need to ensure to stop enqueueing ... after #shutdown, we cannot write any data
|
536
|
+
@output_enqueue_thread_running = false
|
537
|
+
if @output_enqueue_thread && @output_enqueue_thread.alive?
|
538
|
+
@output_enqueue_thread.wakeup
|
539
|
+
@output_enqueue_thread.join
|
540
|
+
end
|
541
|
+
end
|
542
|
+
|
543
|
+
super
|
544
|
+
end
|
545
|
+
|
546
|
+
def shutdown
|
547
|
+
@secondary.shutdown if @secondary
|
548
|
+
@buffer.shutdown if @buffering && @buffer
|
549
|
+
|
550
|
+
super
|
551
|
+
end
|
552
|
+
|
553
|
+
def after_shutdown
|
554
|
+
try_rollback_all if @buffering && !@as_secondary # rollback regardless with @delayed_commit, because secondary may do it
|
555
|
+
@secondary.after_shutdown if @secondary
|
556
|
+
|
557
|
+
if @buffering && @buffer
|
558
|
+
@buffer.after_shutdown
|
559
|
+
|
560
|
+
@output_flush_threads_running = false
|
561
|
+
if @output_flush_threads && !@output_flush_threads.empty?
|
562
|
+
@output_flush_threads.each do |state|
|
563
|
+
# to wakeup thread and make it to stop by itself
|
564
|
+
state.mutex.synchronize {
|
565
|
+
if state.thread && state.thread.status
|
566
|
+
state.next_clock = 0
|
567
|
+
state.cond_var.signal
|
568
|
+
end
|
569
|
+
}
|
570
|
+
Thread.pass
|
571
|
+
state.thread.join
|
572
|
+
end
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
super
|
577
|
+
end
|
578
|
+
|
579
|
+
def close
|
580
|
+
@buffer.close if @buffering && @buffer
|
581
|
+
@secondary.close if @secondary
|
582
|
+
|
583
|
+
super
|
584
|
+
end
|
585
|
+
|
586
|
+
def terminate
|
587
|
+
@buffer.terminate if @buffering && @buffer
|
588
|
+
@secondary.terminate if @secondary
|
589
|
+
|
590
|
+
super
|
591
|
+
end
|
592
|
+
|
593
|
+
def support_in_v12_style?(feature)
|
594
|
+
# for plugins written in v0.12 styles
|
595
|
+
case feature
|
596
|
+
when :synchronous then false
|
597
|
+
when :buffered then false
|
598
|
+
when :delayed_commit then false
|
599
|
+
when :custom_format then false
|
600
|
+
else
|
601
|
+
raise ArgumentError, "unknown feature: #{feature}"
|
602
|
+
end
|
603
|
+
end
|
604
|
+
|
605
|
+
def implement?(feature)
|
606
|
+
methods_of_plugin = self.class.instance_methods(false)
|
607
|
+
case feature
|
608
|
+
when :synchronous then methods_of_plugin.include?(:process) || support_in_v12_style?(:synchronous)
|
609
|
+
when :buffered then methods_of_plugin.include?(:write) || support_in_v12_style?(:buffered)
|
610
|
+
when :delayed_commit then methods_of_plugin.include?(:try_write)
|
611
|
+
when :custom_format then methods_of_plugin.include?(:format) || support_in_v12_style?(:custom_format)
|
612
|
+
else
|
613
|
+
raise ArgumentError, "Unknown feature for output plugin: #{feature}"
|
614
|
+
end
|
615
|
+
end
|
616
|
+
|
617
|
+
def placeholder_validate!(name, str)
|
618
|
+
placeholder_validators(name, str).each do |v|
|
619
|
+
v.validate!
|
620
|
+
end
|
621
|
+
end
|
622
|
+
|
623
|
+
def placeholder_validators(name, str, time_key = (@chunk_key_time && @buffer_config.timekey), tag_key = @chunk_key_tag, chunk_keys = @chunk_keys)
|
624
|
+
validators = []
|
625
|
+
|
626
|
+
sec, title, example = get_placeholders_time(str)
|
627
|
+
if sec || time_key
|
628
|
+
validators << PlaceholderValidator.new(name, str, :time, {sec: sec, title: title, example: example, timekey: time_key})
|
629
|
+
end
|
630
|
+
|
631
|
+
parts = get_placeholders_tag(str)
|
632
|
+
if tag_key || !parts.empty?
|
633
|
+
validators << PlaceholderValidator.new(name, str, :tag, {parts: parts, tagkey: tag_key})
|
634
|
+
end
|
635
|
+
|
636
|
+
keys = get_placeholders_keys(str)
|
637
|
+
if chunk_keys && !chunk_keys.empty? || !keys.empty?
|
638
|
+
validators << PlaceholderValidator.new(name, str, :keys, {keys: keys, chunkkeys: chunk_keys})
|
639
|
+
end
|
640
|
+
|
641
|
+
validators
|
642
|
+
end
|
643
|
+
|
644
|
+
class PlaceholderValidator
|
645
|
+
attr_reader :name, :string, :type, :argument
|
646
|
+
|
647
|
+
def initialize(name, str, type, arg)
|
648
|
+
@name = name
|
649
|
+
@string = str
|
650
|
+
@type = type
|
651
|
+
raise ArgumentError, "invalid type:#{type}" if @type != :time && @type != :tag && @type != :keys
|
652
|
+
@argument = arg
|
653
|
+
end
|
654
|
+
|
655
|
+
def time?
|
656
|
+
@type == :time
|
657
|
+
end
|
658
|
+
|
659
|
+
def tag?
|
660
|
+
@type == :tag
|
661
|
+
end
|
662
|
+
|
663
|
+
def keys?
|
664
|
+
@type == :keys
|
665
|
+
end
|
666
|
+
|
667
|
+
def validate!
|
668
|
+
case @type
|
669
|
+
when :time then validate_time!
|
670
|
+
when :tag then validate_tag!
|
671
|
+
when :keys then validate_keys!
|
672
|
+
end
|
673
|
+
end
|
674
|
+
|
675
|
+
def validate_time!
|
676
|
+
sec = @argument[:sec]
|
677
|
+
title = @argument[:title]
|
678
|
+
example = @argument[:example]
|
679
|
+
timekey = @argument[:timekey]
|
680
|
+
if !sec && timekey
|
681
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
|
682
|
+
end
|
683
|
+
if sec && !timekey
|
684
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has timestamp placeholders, but chunk key 'time' is not configured"
|
685
|
+
end
|
686
|
+
if sec && timekey && timekey < sec
|
687
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
|
688
|
+
end
|
689
|
+
end
|
690
|
+
|
691
|
+
def validate_tag!
|
692
|
+
parts = @argument[:parts]
|
693
|
+
tagkey = @argument[:tagkey]
|
694
|
+
if tagkey && parts.empty?
|
695
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have tag placeholder"
|
696
|
+
end
|
697
|
+
if !tagkey && !parts.empty?
|
698
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has tag placeholders, but chunk key 'tag' is not configured"
|
699
|
+
end
|
700
|
+
end
|
701
|
+
|
702
|
+
def validate_keys!
|
703
|
+
keys = @argument[:keys]
|
704
|
+
chunk_keys = @argument[:chunkkeys]
|
705
|
+
if (chunk_keys - keys).size > 0
|
706
|
+
not_specified = (chunk_keys - keys).sort
|
707
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
|
708
|
+
end
|
709
|
+
if (keys - chunk_keys).size > 0
|
710
|
+
not_satisfied = (keys - chunk_keys).sort
|
711
|
+
raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
|
712
|
+
end
|
713
|
+
end
|
714
|
+
end
|
715
|
+
|
716
|
+
TIME_KEY_PLACEHOLDER_THRESHOLDS = [
|
717
|
+
[1, :second, '%S'],
|
718
|
+
[60, :minute, '%M'],
|
719
|
+
[3600, :hour, '%H'],
|
720
|
+
[86400, :day, '%d'],
|
721
|
+
]
|
722
|
+
TIMESTAMP_CHECK_BASE_TIME = Time.parse("2016-01-01 00:00:00 UTC")
|
723
|
+
# it's not validated to use timekey larger than 1 day
|
724
|
+
def get_placeholders_time(str)
|
725
|
+
base_str = TIMESTAMP_CHECK_BASE_TIME.strftime(str)
|
726
|
+
TIME_KEY_PLACEHOLDER_THRESHOLDS.each do |triple|
|
727
|
+
sec = triple.first
|
728
|
+
return triple if (TIMESTAMP_CHECK_BASE_TIME + sec).strftime(str) != base_str
|
729
|
+
end
|
730
|
+
nil
|
731
|
+
end
|
732
|
+
|
733
|
+
# -1 means whole tag
|
734
|
+
def get_placeholders_tag(str)
|
735
|
+
# [["tag"],["tag[0]"]]
|
736
|
+
parts = []
|
737
|
+
str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
|
738
|
+
if ph == "tag"
|
739
|
+
parts << -1
|
740
|
+
elsif ph =~ /^tag\[(-?\d+)\]$/
|
741
|
+
parts << $1.to_i
|
742
|
+
end
|
743
|
+
end
|
744
|
+
parts.sort
|
745
|
+
end
|
746
|
+
|
747
|
+
def get_placeholders_keys(str)
|
748
|
+
str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map(&:first).reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
|
749
|
+
end
|
750
|
+
|
751
|
+
# TODO: optimize this code
|
752
|
+
def extract_placeholders(str, chunk)
|
753
|
+
metadata = if chunk.is_a?(Fluent::Plugin::Buffer::Chunk)
|
754
|
+
chunk_passed = true
|
755
|
+
chunk.metadata
|
756
|
+
else
|
757
|
+
chunk_passed = false
|
758
|
+
# For existing plugins. Old plugin passes Chunk.metadata instead of Chunk
|
759
|
+
chunk
|
760
|
+
end
|
761
|
+
if metadata.empty?
|
762
|
+
str.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
763
|
+
if chunk_passed
|
764
|
+
dump_unique_id_hex(chunk.unique_id)
|
765
|
+
else
|
766
|
+
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
767
|
+
end
|
768
|
+
}
|
769
|
+
else
|
770
|
+
rvalue = str.dup
|
771
|
+
# strftime formatting
|
772
|
+
if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
|
773
|
+
@output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
|
774
|
+
rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
|
775
|
+
end
|
776
|
+
# ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
|
777
|
+
if @chunk_key_tag
|
778
|
+
if str.include?('${tag}')
|
779
|
+
rvalue = rvalue.gsub('${tag}', metadata.tag)
|
780
|
+
end
|
781
|
+
if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
|
782
|
+
hash = {}
|
783
|
+
tag_parts = metadata.tag.split('.')
|
784
|
+
tag_parts.each_with_index do |part, i|
|
785
|
+
hash["${tag[#{i}]}"] = part
|
786
|
+
hash["${tag[#{i-tag_parts.size}]}"] = part
|
787
|
+
end
|
788
|
+
rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
|
789
|
+
end
|
790
|
+
if rvalue =~ CHUNK_TAG_PLACEHOLDER_PATTERN
|
791
|
+
log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
|
792
|
+
end
|
793
|
+
end
|
794
|
+
|
795
|
+
# First we replace ${chunk_id} with chunk.unique_id (hexlified).
|
796
|
+
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
797
|
+
if chunk_passed
|
798
|
+
dump_unique_id_hex(chunk.unique_id)
|
799
|
+
else
|
800
|
+
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
801
|
+
end
|
802
|
+
}
|
803
|
+
|
804
|
+
# Then, replace other ${chunk_key}s.
|
805
|
+
if !@chunk_keys.empty? && metadata.variables
|
806
|
+
hash = {'${tag}' => '${tag}'} # not to erase this wrongly
|
807
|
+
@chunk_keys.each do |key|
|
808
|
+
hash["${#{key}}"] = metadata.variables[key.to_sym]
|
809
|
+
end
|
810
|
+
|
811
|
+
rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
|
812
|
+
hash.fetch(matched) do
|
813
|
+
log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
|
814
|
+
''
|
815
|
+
end
|
816
|
+
end
|
817
|
+
end
|
818
|
+
|
819
|
+
if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
|
820
|
+
log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
|
821
|
+
end
|
822
|
+
|
823
|
+
rvalue
|
824
|
+
end
|
825
|
+
end
|
826
|
+
|
827
|
+
def emit_events(tag, es)
|
828
|
+
# actually this method will be overwritten by #configure
|
829
|
+
if @buffering
|
830
|
+
emit_buffered(tag, es)
|
831
|
+
else
|
832
|
+
emit_sync(tag, es)
|
833
|
+
end
|
834
|
+
end
|
835
|
+
|
836
|
+
def emit_sync(tag, es)
|
837
|
+
@emit_count_metrics.inc
|
838
|
+
begin
|
839
|
+
process(tag, es)
|
840
|
+
@emit_records_metrics.add(es.size)
|
841
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
842
|
+
rescue
|
843
|
+
@num_errors_metrics.inc
|
844
|
+
raise
|
845
|
+
end
|
846
|
+
end
|
847
|
+
|
848
|
+
def emit_buffered(tag, es)
|
849
|
+
@emit_count_metrics.inc
|
850
|
+
begin
|
851
|
+
execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
|
852
|
+
if !@retry && @buffer.queued?(nil, optimistic: true)
|
853
|
+
submit_flush_once
|
854
|
+
end
|
855
|
+
rescue
|
856
|
+
# TODO: separate number of errors into emit errors and write/flush errors
|
857
|
+
@num_errors_metrics.inc
|
858
|
+
raise
|
859
|
+
end
|
860
|
+
end
|
861
|
+
|
862
|
+
# TODO: optimize this code
|
863
|
+
def metadata(tag, time, record)
|
864
|
+
# this arguments are ordered in output plugin's rule
|
865
|
+
# Metadata 's argument order is different from this one (timekey, tag, variables)
|
866
|
+
|
867
|
+
raise ArgumentError, "tag must be a String: #{tag.class}" unless tag.nil? || tag.is_a?(String)
|
868
|
+
raise ArgumentError, "time must be a Fluent::EventTime (or Integer): #{time.class}" unless time.nil? || time.is_a?(Fluent::EventTime) || time.is_a?(Integer)
|
869
|
+
raise ArgumentError, "record must be a Hash: #{record.class}" unless record.nil? || record.is_a?(Hash)
|
870
|
+
|
871
|
+
if @chunk_keys.nil? && @chunk_key_time.nil? && @chunk_key_tag.nil?
|
872
|
+
# for tests
|
873
|
+
return Struct.new(:timekey, :tag, :variables).new
|
874
|
+
end
|
875
|
+
|
876
|
+
# timekey is int from epoch, and `timekey - timekey % 60` is assumed to mach with 0s of each minutes.
|
877
|
+
# it's wrong if timezone is configured as one which supports leap second, but it's very rare and
|
878
|
+
# we can ignore it (especially in production systems).
|
879
|
+
if @chunk_keys.empty?
|
880
|
+
if !@chunk_key_time && !@chunk_key_tag
|
881
|
+
@buffer.metadata()
|
882
|
+
elsif @chunk_key_time && @chunk_key_tag
|
883
|
+
timekey = calculate_timekey(time)
|
884
|
+
@buffer.metadata(timekey: timekey, tag: tag)
|
885
|
+
elsif @chunk_key_time
|
886
|
+
timekey = calculate_timekey(time)
|
887
|
+
@buffer.metadata(timekey: timekey)
|
888
|
+
else
|
889
|
+
@buffer.metadata(tag: tag)
|
890
|
+
end
|
891
|
+
else
|
892
|
+
timekey = if @chunk_key_time
|
893
|
+
calculate_timekey(time)
|
894
|
+
else
|
895
|
+
nil
|
896
|
+
end
|
897
|
+
pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
|
898
|
+
@buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
|
899
|
+
end
|
900
|
+
end
|
901
|
+
|
902
|
+
def calculate_timekey(time)
|
903
|
+
time_int = time.to_i
|
904
|
+
if @timekey_use_utc
|
905
|
+
(time_int - (time_int % @timekey)).to_i
|
906
|
+
else
|
907
|
+
offset = @calculate_offset ? @calculate_offset.call(time) : @offset
|
908
|
+
(time_int - ((time_int + offset)% @timekey)).to_i
|
909
|
+
end
|
910
|
+
end
|
911
|
+
|
912
|
+
def chunk_for_test(tag, time, record)
|
913
|
+
require 'fluent/plugin/buffer/memory_chunk'
|
914
|
+
|
915
|
+
m = metadata(tag, time, record)
|
916
|
+
Fluent::Plugin::Buffer::MemoryChunk.new(m)
|
917
|
+
end
|
918
|
+
|
919
|
+
def execute_chunking(tag, es, enqueue: false)
|
920
|
+
if @simple_chunking
|
921
|
+
handle_stream_simple(tag, es, enqueue: enqueue)
|
922
|
+
elsif @custom_format
|
923
|
+
handle_stream_with_custom_format(tag, es, enqueue: enqueue)
|
924
|
+
else
|
925
|
+
handle_stream_with_standard_format(tag, es, enqueue: enqueue)
|
926
|
+
end
|
927
|
+
end
|
928
|
+
|
929
|
+
def write_guard(&block)
|
930
|
+
begin
|
931
|
+
block.call
|
932
|
+
rescue Fluent::Plugin::Buffer::BufferOverflowError
|
933
|
+
log.warn "failed to write data into buffer by buffer overflow", action: @buffer_config.overflow_action
|
934
|
+
case @buffer_config.overflow_action
|
935
|
+
when :throw_exception
|
936
|
+
raise
|
937
|
+
when :block
|
938
|
+
log.debug "buffer.write is now blocking"
|
939
|
+
until @buffer.storable?
|
940
|
+
if self.stopped?
|
941
|
+
log.error "breaking block behavior to shutdown Fluentd"
|
942
|
+
# to break infinite loop to exit Fluentd process
|
943
|
+
raise
|
944
|
+
end
|
945
|
+
log.trace "sleeping until buffer can store more data"
|
946
|
+
sleep 1
|
947
|
+
end
|
948
|
+
log.debug "retrying buffer.write after blocked operation"
|
949
|
+
retry
|
950
|
+
when :drop_oldest_chunk
|
951
|
+
begin
|
952
|
+
oldest = @buffer.dequeue_chunk
|
953
|
+
if oldest
|
954
|
+
log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
|
955
|
+
@buffer.purge_chunk(oldest.unique_id)
|
956
|
+
else
|
957
|
+
log.error "no queued chunks to be dropped for drop_oldest_chunk"
|
958
|
+
end
|
959
|
+
rescue
|
960
|
+
# ignore any errors
|
961
|
+
end
|
962
|
+
raise unless @buffer.storable?
|
963
|
+
retry
|
964
|
+
else
|
965
|
+
raise "BUG: unknown overflow_action '#{@buffer_config.overflow_action}'"
|
966
|
+
end
|
967
|
+
end
|
968
|
+
end
|
969
|
+
|
970
|
+
FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
|
971
|
+
FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
|
972
|
+
FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
|
973
|
+
FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
|
974
|
+
|
975
|
+
def generate_format_proc
|
976
|
+
if @buffer && @buffer.compress == :gzip
|
977
|
+
@time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
|
978
|
+
else
|
979
|
+
@time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
|
980
|
+
end
|
981
|
+
end
|
982
|
+
|
983
|
+
# metadata_and_data is a Hash of:
|
984
|
+
# (standard format) metadata => event stream
|
985
|
+
# (custom format) metadata => array of formatted event
|
986
|
+
# For standard format, formatting should be done for whole event stream, but
|
987
|
+
# "whole event stream" may be a split of "es" here when it's bigger than chunk_limit_size.
|
988
|
+
# `@buffer.write` will do this splitting.
|
989
|
+
# For custom format, formatting will be done here. Custom formatting always requires
|
990
|
+
# iteration of event stream, and it should be done just once even if total event stream size
|
991
|
+
# is bigger than chunk_limit_size because of performance.
|
992
|
+
def handle_stream_with_custom_format(tag, es, enqueue: false)
|
993
|
+
meta_and_data = {}
|
994
|
+
records = 0
|
995
|
+
es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
|
996
|
+
meta = metadata(tag, time, record)
|
997
|
+
meta_and_data[meta] ||= []
|
998
|
+
res = format(tag, time, record)
|
999
|
+
if res
|
1000
|
+
meta_and_data[meta] << res
|
1001
|
+
records += 1
|
1002
|
+
end
|
1003
|
+
end
|
1004
|
+
write_guard do
|
1005
|
+
@buffer.write(meta_and_data, enqueue: enqueue)
|
1006
|
+
end
|
1007
|
+
@emit_records_metrics.add(es.size)
|
1008
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
1009
|
+
true
|
1010
|
+
end
|
1011
|
+
|
1012
|
+
def handle_stream_with_standard_format(tag, es, enqueue: false)
|
1013
|
+
format_proc = generate_format_proc
|
1014
|
+
meta_and_data = {}
|
1015
|
+
records = 0
|
1016
|
+
es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
|
1017
|
+
meta = metadata(tag, time, record)
|
1018
|
+
meta_and_data[meta] ||= MultiEventStream.new
|
1019
|
+
meta_and_data[meta].add(time, record)
|
1020
|
+
records += 1
|
1021
|
+
end
|
1022
|
+
write_guard do
|
1023
|
+
@buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
|
1024
|
+
end
|
1025
|
+
@emit_records_metrics.add(es.size)
|
1026
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
1027
|
+
true
|
1028
|
+
end
|
1029
|
+
|
1030
|
+
def handle_stream_simple(tag, es, enqueue: false)
|
1031
|
+
format_proc = nil
|
1032
|
+
meta = metadata((@chunk_key_tag ? tag : nil), nil, nil)
|
1033
|
+
records = es.size
|
1034
|
+
if @custom_format
|
1035
|
+
records = 0
|
1036
|
+
data = []
|
1037
|
+
es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
|
1038
|
+
res = format(tag, time, record)
|
1039
|
+
if res
|
1040
|
+
data << res
|
1041
|
+
records += 1
|
1042
|
+
end
|
1043
|
+
end
|
1044
|
+
else
|
1045
|
+
format_proc = generate_format_proc
|
1046
|
+
data = es
|
1047
|
+
end
|
1048
|
+
write_guard do
|
1049
|
+
@buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
|
1050
|
+
end
|
1051
|
+
@emit_records_metrics.add(es.size)
|
1052
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
1053
|
+
true
|
1054
|
+
end
|
1055
|
+
|
1056
|
+
def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
|
1057
|
+
log.on_trace { log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }
|
1058
|
+
|
1059
|
+
if delayed
|
1060
|
+
@dequeued_chunks_mutex.synchronize do
|
1061
|
+
@dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
|
1062
|
+
end
|
1063
|
+
end
|
1064
|
+
@buffer.purge_chunk(chunk_id)
|
1065
|
+
|
1066
|
+
@retry_mutex.synchronize do
|
1067
|
+
if @retry # success to flush chunks in retries
|
1068
|
+
if secondary
|
1069
|
+
log.warn "retry succeeded by secondary.", chunk_id: dump_unique_id_hex(chunk_id)
|
1070
|
+
else
|
1071
|
+
log.warn "retry succeeded.", chunk_id: dump_unique_id_hex(chunk_id)
|
1072
|
+
end
|
1073
|
+
@retry = nil
|
1074
|
+
end
|
1075
|
+
end
|
1076
|
+
end
|
1077
|
+
|
1078
|
+
# update_retry parameter is for preventing busy loop by async write
|
1079
|
+
# We will remove this parameter by re-design retry_state management between threads.
|
1080
|
+
def rollback_write(chunk_id, update_retry: true)
|
1081
|
+
# This API is to rollback chunks explicitly from plugins.
|
1082
|
+
# 3rd party plugins can depend it on automatic rollback of #try_rollback_write
|
1083
|
+
@dequeued_chunks_mutex.synchronize do
|
1084
|
+
@dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
|
1085
|
+
end
|
1086
|
+
# returns true if chunk was rollbacked as expected
|
1087
|
+
# false if chunk was already flushed and couldn't be rollbacked unexpectedly
|
1088
|
+
# in many cases, false can be just ignored
|
1089
|
+
if @buffer.takeback_chunk(chunk_id)
|
1090
|
+
@rollback_count_metrics.inc
|
1091
|
+
if update_retry
|
1092
|
+
primary = @as_secondary ? @primary_instance : self
|
1093
|
+
primary.update_retry_state(chunk_id, @as_secondary)
|
1094
|
+
end
|
1095
|
+
true
|
1096
|
+
else
|
1097
|
+
false
|
1098
|
+
end
|
1099
|
+
end
|
1100
|
+
|
1101
|
+
def try_rollback_write
|
1102
|
+
@dequeued_chunks_mutex.synchronize do
|
1103
|
+
while @dequeued_chunks.first && @dequeued_chunks.first.expired?
|
1104
|
+
info = @dequeued_chunks.shift
|
1105
|
+
if @buffer.takeback_chunk(info.chunk_id)
|
1106
|
+
@rollback_count_metrics.inc
|
1107
|
+
log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
|
1108
|
+
primary = @as_secondary ? @primary_instance : self
|
1109
|
+
primary.update_retry_state(info.chunk_id, @as_secondary)
|
1110
|
+
end
|
1111
|
+
end
|
1112
|
+
end
|
1113
|
+
end
|
1114
|
+
|
1115
|
+
def try_rollback_all
|
1116
|
+
return unless @dequeued_chunks
|
1117
|
+
@dequeued_chunks_mutex.synchronize do
|
1118
|
+
until @dequeued_chunks.empty?
|
1119
|
+
info = @dequeued_chunks.shift
|
1120
|
+
if @buffer.takeback_chunk(info.chunk_id)
|
1121
|
+
@rollback_count_metrics.inc
|
1122
|
+
log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
|
1123
|
+
primary = @as_secondary ? @primary_instance : self
|
1124
|
+
primary.update_retry_state(info.chunk_id, @as_secondary)
|
1125
|
+
end
|
1126
|
+
end
|
1127
|
+
end
|
1128
|
+
end
|
1129
|
+
|
1130
|
+
def next_flush_time
|
1131
|
+
if @buffer.queued?
|
1132
|
+
@retry_mutex.synchronize do
|
1133
|
+
@retry ? @retry.next_time : Time.now + @buffer_config.flush_thread_burst_interval
|
1134
|
+
end
|
1135
|
+
else
|
1136
|
+
Time.now + @buffer_config.flush_thread_interval
|
1137
|
+
end
|
1138
|
+
end
|
1139
|
+
|
1140
|
+
UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError]
|
1141
|
+
|
1142
|
+
def try_flush
|
1143
|
+
chunk = @buffer.dequeue_chunk
|
1144
|
+
return unless chunk
|
1145
|
+
|
1146
|
+
log.on_trace { log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id) }
|
1147
|
+
|
1148
|
+
output = self
|
1149
|
+
using_secondary = false
|
1150
|
+
if @retry_mutex.synchronize{ @retry && @retry.secondary? }
|
1151
|
+
output = @secondary
|
1152
|
+
using_secondary = true
|
1153
|
+
end
|
1154
|
+
|
1155
|
+
if @enable_msgpack_streamer
|
1156
|
+
chunk.extend ChunkMessagePackEventStreamer
|
1157
|
+
end
|
1158
|
+
|
1159
|
+
begin
|
1160
|
+
chunk_write_start = Fluent::Clock.now
|
1161
|
+
|
1162
|
+
if output.delayed_commit
|
1163
|
+
log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
|
1164
|
+
@write_count_metrics.inc
|
1165
|
+
@dequeued_chunks_mutex.synchronize do
|
1166
|
+
# delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
|
1167
|
+
@dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
|
1168
|
+
end
|
1169
|
+
|
1170
|
+
output.try_write(chunk)
|
1171
|
+
check_slow_flush(chunk_write_start)
|
1172
|
+
else # output plugin without delayed purge
|
1173
|
+
chunk_id = chunk.unique_id
|
1174
|
+
dump_chunk_id = dump_unique_id_hex(chunk_id)
|
1175
|
+
log.trace "adding write count", instance: self.object_id
|
1176
|
+
@write_count_metrics.inc
|
1177
|
+
log.trace "executing sync write", chunk: dump_chunk_id
|
1178
|
+
|
1179
|
+
output.write(chunk)
|
1180
|
+
check_slow_flush(chunk_write_start)
|
1181
|
+
|
1182
|
+
log.trace "write operation done, committing", chunk: dump_chunk_id
|
1183
|
+
commit_write(chunk_id, delayed: false, secondary: using_secondary)
|
1184
|
+
log.trace "done to commit a chunk", chunk: dump_chunk_id
|
1185
|
+
end
|
1186
|
+
rescue *UNRECOVERABLE_ERRORS => e
|
1187
|
+
if @secondary
|
1188
|
+
if using_secondary
|
1189
|
+
log.warn "got unrecoverable error in secondary.", error: e
|
1190
|
+
log.warn_backtrace
|
1191
|
+
backup_chunk(chunk, using_secondary, output.delayed_commit)
|
1192
|
+
else
|
1193
|
+
if (self.class == @secondary.class)
|
1194
|
+
log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
|
1195
|
+
log.warn_backtrace
|
1196
|
+
backup_chunk(chunk, using_secondary, output.delayed_commit)
|
1197
|
+
else
|
1198
|
+
# Call secondary output directly without retry update.
|
1199
|
+
# In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
|
1200
|
+
if @secondary.delayed_commit
|
1201
|
+
log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
|
1202
|
+
log.warn_backtrace
|
1203
|
+
backup_chunk(chunk, using_secondary, output.delayed_commit)
|
1204
|
+
else
|
1205
|
+
log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
|
1206
|
+
log.warn_backtrace
|
1207
|
+
begin
|
1208
|
+
@secondary.write(chunk)
|
1209
|
+
commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
|
1210
|
+
rescue => e
|
1211
|
+
log.warn "got an error in secondary for unrecoverable error", error: e
|
1212
|
+
log.warn_backtrace
|
1213
|
+
backup_chunk(chunk, using_secondary, output.delayed_commit)
|
1214
|
+
end
|
1215
|
+
end
|
1216
|
+
end
|
1217
|
+
end
|
1218
|
+
else
|
1219
|
+
log.warn "got unrecoverable error in primary and no secondary", error: e
|
1220
|
+
log.warn_backtrace
|
1221
|
+
backup_chunk(chunk, using_secondary, output.delayed_commit)
|
1222
|
+
end
|
1223
|
+
rescue => e
|
1224
|
+
log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
|
1225
|
+
if output.delayed_commit
|
1226
|
+
@dequeued_chunks_mutex.synchronize do
|
1227
|
+
@dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
|
1228
|
+
end
|
1229
|
+
end
|
1230
|
+
|
1231
|
+
if @buffer.takeback_chunk(chunk.unique_id)
|
1232
|
+
@rollback_count_metrics.inc
|
1233
|
+
end
|
1234
|
+
|
1235
|
+
update_retry_state(chunk.unique_id, using_secondary, e)
|
1236
|
+
|
1237
|
+
raise if @under_plugin_development && !@retry_for_error_chunk
|
1238
|
+
end
|
1239
|
+
end
|
1240
|
+
|
1241
|
+
def backup_chunk(chunk, using_secondary, delayed_commit)
|
1242
|
+
if @buffer_config.disable_chunk_backup
|
1243
|
+
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
|
1244
|
+
else
|
1245
|
+
unique_id = dump_unique_id_hex(chunk.unique_id)
|
1246
|
+
safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
|
1247
|
+
backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
|
1248
|
+
backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
|
1249
|
+
backup_dir = File.dirname(backup_file)
|
1250
|
+
|
1251
|
+
log.warn "bad chunk is moved to #{backup_file}"
|
1252
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
1253
|
+
File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
|
1254
|
+
chunk.write_to(f)
|
1255
|
+
}
|
1256
|
+
end
|
1257
|
+
commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
|
1258
|
+
end
|
1259
|
+
|
1260
|
+
def check_slow_flush(start)
|
1261
|
+
elapsed_time = Fluent::Clock.now - start
|
1262
|
+
elapsed_millsec = (elapsed_time * 1000).to_i
|
1263
|
+
@flush_time_count_metrics.add(elapsed_millsec)
|
1264
|
+
if elapsed_time > @slow_flush_log_threshold
|
1265
|
+
@slow_flush_count_metrics.inc
|
1266
|
+
log.warn "buffer flush took longer time than slow_flush_log_threshold:",
|
1267
|
+
elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
|
1268
|
+
end
|
1269
|
+
end
|
1270
|
+
|
1271
|
+
def update_retry_state(chunk_id, using_secondary, error = nil)
|
1272
|
+
@retry_mutex.synchronize do
|
1273
|
+
@num_errors_metrics.inc
|
1274
|
+
chunk_id_hex = dump_unique_id_hex(chunk_id)
|
1275
|
+
|
1276
|
+
unless @retry
|
1277
|
+
@retry = retry_state(@buffer_config.retry_randomize)
|
1278
|
+
if error
|
1279
|
+
log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1280
|
+
log.warn_backtrace error.backtrace
|
1281
|
+
end
|
1282
|
+
return
|
1283
|
+
end
|
1284
|
+
|
1285
|
+
# @retry exists
|
1286
|
+
|
1287
|
+
if @retry.limit?
|
1288
|
+
if error
|
1289
|
+
records = @buffer.queued_records
|
1290
|
+
msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
|
1291
|
+
log.error msg, retry_times: @retry.steps, records: records, error: error
|
1292
|
+
log.error_backtrace error.backtrace
|
1293
|
+
end
|
1294
|
+
@buffer.clear_queue!
|
1295
|
+
log.debug "buffer queue cleared"
|
1296
|
+
@retry = nil
|
1297
|
+
else
|
1298
|
+
# Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
|
1299
|
+
# @retry.step is called almost as many times as the number of flush threads in a short time.
|
1300
|
+
if Time.now >= @retry.next_time
|
1301
|
+
@retry.step
|
1302
|
+
else
|
1303
|
+
@retry.recalc_next_time # to prevent all flush threads from retrying at the same time
|
1304
|
+
end
|
1305
|
+
if error
|
1306
|
+
if using_secondary
|
1307
|
+
msg = "failed to flush the buffer with secondary output."
|
1308
|
+
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1309
|
+
log.warn_backtrace error.backtrace
|
1310
|
+
else
|
1311
|
+
msg = "failed to flush the buffer."
|
1312
|
+
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1313
|
+
log.warn_backtrace error.backtrace
|
1314
|
+
end
|
1315
|
+
end
|
1316
|
+
end
|
1317
|
+
end
|
1318
|
+
end
|
1319
|
+
|
1320
|
+
def retry_state(randomize)
|
1321
|
+
if @secondary
|
1322
|
+
retry_state_create(
|
1323
|
+
:output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,
|
1324
|
+
forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,
|
1325
|
+
max_interval: @buffer_config.retry_max_interval,
|
1326
|
+
secondary: true, secondary_threshold: @buffer_config.retry_secondary_threshold,
|
1327
|
+
randomize: randomize
|
1328
|
+
)
|
1329
|
+
else
|
1330
|
+
retry_state_create(
|
1331
|
+
:output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout,
|
1332
|
+
forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_exponential_backoff_base,
|
1333
|
+
max_interval: @buffer_config.retry_max_interval,
|
1334
|
+
randomize: randomize
|
1335
|
+
)
|
1336
|
+
end
|
1337
|
+
end
|
1338
|
+
|
1339
|
+
def submit_flush_once
|
1340
|
+
# Without locks: it is rough but enough to select "next" writer selection
|
1341
|
+
@output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
|
1342
|
+
state = @output_flush_threads[@output_flush_thread_current_position]
|
1343
|
+
state.mutex.synchronize {
|
1344
|
+
if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
|
1345
|
+
state.next_clock = 0
|
1346
|
+
state.cond_var.signal
|
1347
|
+
else
|
1348
|
+
log.warn "thread is already dead"
|
1349
|
+
end
|
1350
|
+
}
|
1351
|
+
Thread.pass
|
1352
|
+
end
|
1353
|
+
|
1354
|
+
def force_flush
|
1355
|
+
if @buffering
|
1356
|
+
@buffer.enqueue_all(true)
|
1357
|
+
submit_flush_all
|
1358
|
+
end
|
1359
|
+
end
|
1360
|
+
|
1361
|
+
def submit_flush_all
|
1362
|
+
while !@retry && @buffer.queued?
|
1363
|
+
submit_flush_once
|
1364
|
+
sleep @buffer_config.flush_thread_burst_interval
|
1365
|
+
end
|
1366
|
+
end
|
1367
|
+
|
1368
|
+
# only for tests of output plugin
|
1369
|
+
def interrupt_flushes
|
1370
|
+
@output_flush_interrupted = true
|
1371
|
+
end
|
1372
|
+
|
1373
|
+
# only for tests of output plugin
|
1374
|
+
def enqueue_thread_wait
|
1375
|
+
@output_enqueue_thread_mutex.synchronize do
|
1376
|
+
@output_flush_interrupted = false
|
1377
|
+
@output_enqueue_thread_waiting = true
|
1378
|
+
end
|
1379
|
+
require 'timeout'
|
1380
|
+
Timeout.timeout(10) do
|
1381
|
+
Thread.pass while @output_enqueue_thread_waiting
|
1382
|
+
end
|
1383
|
+
end
|
1384
|
+
|
1385
|
+
# only for tests of output plugin
|
1386
|
+
def flush_thread_wakeup
|
1387
|
+
@output_flush_threads.each do |state|
|
1388
|
+
state.mutex.synchronize {
|
1389
|
+
if state.thread && state.thread.status
|
1390
|
+
state.next_clock = 0
|
1391
|
+
state.cond_var.signal
|
1392
|
+
end
|
1393
|
+
}
|
1394
|
+
Thread.pass
|
1395
|
+
end
|
1396
|
+
end
|
1397
|
+
|
1398
|
+
def enqueue_thread_run
|
1399
|
+
value_for_interval = nil
|
1400
|
+
if @flush_mode == :interval
|
1401
|
+
value_for_interval = @buffer_config.flush_interval
|
1402
|
+
end
|
1403
|
+
if @chunk_key_time
|
1404
|
+
if !value_for_interval || @buffer_config.timekey < value_for_interval
|
1405
|
+
value_for_interval = [@buffer_config.timekey, @buffer_config.timekey_wait].min
|
1406
|
+
end
|
1407
|
+
end
|
1408
|
+
unless value_for_interval
|
1409
|
+
raise "BUG: both of flush_interval and timekey are disabled"
|
1410
|
+
end
|
1411
|
+
interval = value_for_interval / 11.0
|
1412
|
+
if interval < @buffer_config.flush_thread_interval
|
1413
|
+
interval = @buffer_config.flush_thread_interval
|
1414
|
+
end
|
1415
|
+
|
1416
|
+
while !self.after_started? && !self.stopped?
|
1417
|
+
sleep 0.5
|
1418
|
+
end
|
1419
|
+
log.debug "enqueue_thread actually running"
|
1420
|
+
|
1421
|
+
begin
|
1422
|
+
while @output_enqueue_thread_running
|
1423
|
+
now_int = Time.now.to_i
|
1424
|
+
if @output_flush_interrupted
|
1425
|
+
sleep interval
|
1426
|
+
next
|
1427
|
+
end
|
1428
|
+
|
1429
|
+
@output_enqueue_thread_mutex.lock
|
1430
|
+
begin
|
1431
|
+
if @flush_mode == :interval
|
1432
|
+
flush_interval = @buffer_config.flush_interval.to_i
|
1433
|
+
# This block should be done by integer values.
|
1434
|
+
# If both of flush_interval & flush_thread_interval are 1s, expected actual flush timing is 1.5s.
|
1435
|
+
# If we use integered values for this comparison, expected actual flush timing is 1.0s.
|
1436
|
+
@buffer.enqueue_all{ |metadata, chunk| chunk.raw_create_at + flush_interval <= now_int }
|
1437
|
+
end
|
1438
|
+
|
1439
|
+
if @chunk_key_time
|
1440
|
+
timekey_unit = @buffer_config.timekey
|
1441
|
+
timekey_wait = @buffer_config.timekey_wait
|
1442
|
+
current_timekey = now_int - now_int % timekey_unit
|
1443
|
+
@buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_timekey && metadata.timekey + timekey_unit + timekey_wait <= now_int }
|
1444
|
+
end
|
1445
|
+
rescue => e
|
1446
|
+
raise if @under_plugin_development
|
1447
|
+
log.error "unexpected error while checking flushed chunks. ignored.", error: e
|
1448
|
+
log.error_backtrace
|
1449
|
+
ensure
|
1450
|
+
@output_enqueue_thread_waiting = false
|
1451
|
+
@output_enqueue_thread_mutex.unlock
|
1452
|
+
end
|
1453
|
+
sleep interval
|
1454
|
+
end
|
1455
|
+
rescue => e
|
1456
|
+
# normal errors are rescued by inner begin-rescue clause.
|
1457
|
+
log.error "error on enqueue thread", error: e
|
1458
|
+
log.error_backtrace
|
1459
|
+
raise
|
1460
|
+
end
|
1461
|
+
end
|
1462
|
+
|
1463
|
+
def flush_thread_run(state)
|
1464
|
+
flush_thread_interval = @buffer_config.flush_thread_interval
|
1465
|
+
|
1466
|
+
state.next_clock = Fluent::Clock.now + flush_thread_interval
|
1467
|
+
|
1468
|
+
while !self.after_started? && !self.stopped?
|
1469
|
+
sleep 0.5
|
1470
|
+
end
|
1471
|
+
log.debug "flush_thread actually running"
|
1472
|
+
|
1473
|
+
state.mutex.lock
|
1474
|
+
begin
|
1475
|
+
# This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
|
1476
|
+
while @output_flush_threads_running
|
1477
|
+
current_clock = Fluent::Clock.now
|
1478
|
+
next_retry_time = nil
|
1479
|
+
|
1480
|
+
@retry_mutex.synchronize do
|
1481
|
+
next_retry_time = @retry ? @retry.next_time : nil
|
1482
|
+
end
|
1483
|
+
|
1484
|
+
if state.next_clock > current_clock
|
1485
|
+
interval = state.next_clock - current_clock
|
1486
|
+
elsif next_retry_time && next_retry_time > Time.now
|
1487
|
+
interval = next_retry_time.to_f - Time.now.to_f
|
1488
|
+
else
|
1489
|
+
state.mutex.unlock
|
1490
|
+
begin
|
1491
|
+
try_flush
|
1492
|
+
# next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
|
1493
|
+
interval = next_flush_time.to_f - Time.now.to_f
|
1494
|
+
# TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
|
1495
|
+
# because @retry still exists (#commit_write is not called yet in #try_flush)
|
1496
|
+
# @retry should be cleared if delayed commit is enabled? Or any other solution?
|
1497
|
+
state.next_clock = Fluent::Clock.now + interval
|
1498
|
+
ensure
|
1499
|
+
state.mutex.lock
|
1500
|
+
end
|
1501
|
+
end
|
1502
|
+
|
1503
|
+
if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
|
1504
|
+
unless @output_flush_interrupted
|
1505
|
+
state.mutex.unlock
|
1506
|
+
begin
|
1507
|
+
try_rollback_write
|
1508
|
+
ensure
|
1509
|
+
state.mutex.lock
|
1510
|
+
end
|
1511
|
+
end
|
1512
|
+
end
|
1513
|
+
|
1514
|
+
state.cond_var.wait(state.mutex, interval) if interval > 0
|
1515
|
+
end
|
1516
|
+
rescue => e
|
1517
|
+
# normal errors are rescued by output plugins in #try_flush
|
1518
|
+
# so this rescue section is for critical & unrecoverable errors
|
1519
|
+
log.error "error on output thread", error: e
|
1520
|
+
log.error_backtrace
|
1521
|
+
raise
|
1522
|
+
ensure
|
1523
|
+
state.mutex.unlock
|
1524
|
+
end
|
1525
|
+
end
|
1526
|
+
|
1527
|
+
BUFFER_STATS_KEYS = {}
|
1528
|
+
Fluent::Plugin::Buffer::STATS_KEYS.each { |key|
|
1529
|
+
BUFFER_STATS_KEYS[key] = "buffer_#{key}"
|
1530
|
+
}
|
1531
|
+
|
1532
|
+
def statistics
|
1533
|
+
stats = {
|
1534
|
+
'emit_records' => @emit_records_metrics.get,
|
1535
|
+
'emit_size' => @emit_size_metrics.get,
|
1536
|
+
# Respect original name
|
1537
|
+
# https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
|
1538
|
+
'retry_count' => @num_errors_metrics.get,
|
1539
|
+
'emit_count' => @emit_count_metrics.get,
|
1540
|
+
'write_count' => @write_count_metrics.get,
|
1541
|
+
'rollback_count' => @rollback_count_metrics.get,
|
1542
|
+
'slow_flush_count' => @slow_flush_count_metrics.get,
|
1543
|
+
'flush_time_count' => @flush_time_count_metrics.get,
|
1544
|
+
}
|
1545
|
+
|
1546
|
+
if @buffer && @buffer.respond_to?(:statistics)
|
1547
|
+
(@buffer.statistics['buffer'] || {}).each do |k, v|
|
1548
|
+
stats[BUFFER_STATS_KEYS[k]] = v
|
1549
|
+
end
|
1550
|
+
end
|
1551
|
+
|
1552
|
+
{ 'output' => stats }
|
1553
|
+
end
|
1554
|
+
end
|
1555
|
+
end
|
1556
|
+
end
|