fluentd 1.14.4-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
@@ -0,0 +1,96 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin'
|
18
|
+
require 'fluent/plugin/metrics'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
module Plugin
|
22
|
+
class LocalMetrics < Metrics
|
23
|
+
Fluent::Plugin.register_metrics('local', self)
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
super
|
27
|
+
@store = 0
|
28
|
+
@monitor = Monitor.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def configure(conf)
|
32
|
+
super
|
33
|
+
|
34
|
+
if use_gauge_metric
|
35
|
+
class << self
|
36
|
+
alias_method :dec, :dec_gauge
|
37
|
+
alias_method :set, :set_gauge
|
38
|
+
alias_method :sub, :sub_gauge
|
39
|
+
end
|
40
|
+
else
|
41
|
+
class << self
|
42
|
+
alias_method :set, :set_counter
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def multi_workers_ready?
|
48
|
+
true
|
49
|
+
end
|
50
|
+
|
51
|
+
def get
|
52
|
+
@monitor.synchronize do
|
53
|
+
@store
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def inc
|
58
|
+
@monitor.synchronize do
|
59
|
+
@store += 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def dec_gauge
|
64
|
+
@monitor.synchronize do
|
65
|
+
@store -= 1
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def add(value)
|
70
|
+
@monitor.synchronize do
|
71
|
+
@store += value
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def sub_gauge(value)
|
76
|
+
@monitor.synchronize do
|
77
|
+
@store -= value
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def set_counter(value)
|
82
|
+
return if @store > value
|
83
|
+
|
84
|
+
@monitor.synchronize do
|
85
|
+
@store = value
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_gauge(value)
|
90
|
+
@monitor.synchronize do
|
91
|
+
@store = value
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/base'
|
18
|
+
require 'fluent/log'
|
19
|
+
require 'fluent/plugin_id'
|
20
|
+
require 'fluent/plugin_helper'
|
21
|
+
|
22
|
+
module Fluent
|
23
|
+
module Plugin
|
24
|
+
class MultiOutput < Base
|
25
|
+
include PluginId
|
26
|
+
include PluginLoggerMixin
|
27
|
+
include PluginHelper::Mixin # for event_emitter
|
28
|
+
|
29
|
+
helpers :event_emitter # to get router from agent, which will be supplied to child plugins
|
30
|
+
helpers_internal :metrics
|
31
|
+
|
32
|
+
config_section :store, param_name: :stores, multi: true, required: true do
|
33
|
+
config_argument :arg, :string, default: ''
|
34
|
+
config_param :@type, :string, default: nil
|
35
|
+
end
|
36
|
+
|
37
|
+
attr_reader :outputs, :outputs_statically_created
|
38
|
+
|
39
|
+
def process(tag, es)
|
40
|
+
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
41
|
+
end
|
42
|
+
|
43
|
+
def initialize
|
44
|
+
super
|
45
|
+
@outputs = []
|
46
|
+
@outputs_statically_created = false
|
47
|
+
|
48
|
+
@counter_mutex = Mutex.new
|
49
|
+
# TODO: well organized counters
|
50
|
+
@num_errors_metrics = nil
|
51
|
+
@emit_count_metrics = nil
|
52
|
+
@emit_records_metrics = nil
|
53
|
+
@emit_size_metrics = nil
|
54
|
+
# @write_count = 0
|
55
|
+
# @rollback_count = 0
|
56
|
+
@enable_size_metrics = false
|
57
|
+
end
|
58
|
+
|
59
|
+
def num_errors
|
60
|
+
@num_errors_metrics.get
|
61
|
+
end
|
62
|
+
|
63
|
+
def emit_count
|
64
|
+
@emit_count_metrics.get
|
65
|
+
end
|
66
|
+
|
67
|
+
def emit_size
|
68
|
+
@emit_size_metrics.get
|
69
|
+
end
|
70
|
+
|
71
|
+
def emit_records
|
72
|
+
@emit_records_metrics.get
|
73
|
+
end
|
74
|
+
|
75
|
+
def statistics
|
76
|
+
stats = {
|
77
|
+
'num_errors' => @num_errors_metrics.get,
|
78
|
+
'emit_records' => @emit_records_metrics.get,
|
79
|
+
'emit_count' => @emit_count_metrics.get,
|
80
|
+
'emit_size' => @emit_size_metrics.get,
|
81
|
+
}
|
82
|
+
|
83
|
+
{ 'multi_output' => stats }
|
84
|
+
end
|
85
|
+
|
86
|
+
def multi_output?
|
87
|
+
true
|
88
|
+
end
|
89
|
+
|
90
|
+
def configure(conf)
|
91
|
+
super
|
92
|
+
|
93
|
+
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
|
94
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
|
95
|
+
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
|
96
|
+
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
|
97
|
+
@enable_size_metrics = !!system_config.enable_size_metrics
|
98
|
+
|
99
|
+
@stores.each do |store|
|
100
|
+
store_conf = store.corresponding_config_element
|
101
|
+
type = store_conf['@type']
|
102
|
+
unless type
|
103
|
+
raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
|
104
|
+
end
|
105
|
+
|
106
|
+
log.debug "adding store", type: type
|
107
|
+
|
108
|
+
output = Fluent::Plugin.new_output(type)
|
109
|
+
output.context_router = self.context_router
|
110
|
+
output.configure(store_conf)
|
111
|
+
@outputs << output
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def static_outputs
|
116
|
+
@outputs_statically_created = true
|
117
|
+
@outputs
|
118
|
+
end
|
119
|
+
|
120
|
+
# Child plugin's lifecycles are controlled by agent automatically.
|
121
|
+
# It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
|
122
|
+
# * `start` of this plugin will be called after child plugins
|
123
|
+
# * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
|
124
|
+
|
125
|
+
# But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find
|
126
|
+
# sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.
|
127
|
+
# TODO: this hack will be removed at v2.
|
128
|
+
def call_lifecycle_method(method_name, checker_name)
|
129
|
+
return if @outputs_statically_created
|
130
|
+
@outputs.each do |o|
|
131
|
+
begin
|
132
|
+
log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
|
133
|
+
o.__send__(method_name) unless o.__send__(checker_name)
|
134
|
+
rescue Exception => e
|
135
|
+
log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
|
136
|
+
log.warn_backtrace
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def start
|
142
|
+
super
|
143
|
+
call_lifecycle_method(:start, :started?)
|
144
|
+
end
|
145
|
+
|
146
|
+
def after_start
|
147
|
+
super
|
148
|
+
call_lifecycle_method(:after_start, :after_started?)
|
149
|
+
end
|
150
|
+
|
151
|
+
def stop
|
152
|
+
super
|
153
|
+
call_lifecycle_method(:stop, :stopped?)
|
154
|
+
end
|
155
|
+
|
156
|
+
def before_shutdown
|
157
|
+
super
|
158
|
+
call_lifecycle_method(:before_shutdown, :before_shutdown?)
|
159
|
+
end
|
160
|
+
|
161
|
+
def shutdown
|
162
|
+
super
|
163
|
+
call_lifecycle_method(:shutdown, :shutdown?)
|
164
|
+
end
|
165
|
+
|
166
|
+
def after_shutdown
|
167
|
+
super
|
168
|
+
call_lifecycle_method(:after_shutdown, :after_shutdown?)
|
169
|
+
end
|
170
|
+
|
171
|
+
def close
|
172
|
+
super
|
173
|
+
call_lifecycle_method(:close, :closed?)
|
174
|
+
end
|
175
|
+
|
176
|
+
def terminate
|
177
|
+
super
|
178
|
+
call_lifecycle_method(:terminate, :terminated?)
|
179
|
+
end
|
180
|
+
|
181
|
+
def emit_sync(tag, es)
|
182
|
+
@emit_count_metrics.inc
|
183
|
+
begin
|
184
|
+
process(tag, es)
|
185
|
+
@emit_records_metrics.add(es.size)
|
186
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
187
|
+
rescue
|
188
|
+
@num_errors_metrics.inc
|
189
|
+
raise
|
190
|
+
end
|
191
|
+
end
|
192
|
+
alias :emit_events :emit_sync
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/multi_output'
|
18
|
+
require 'fluent/config/error'
|
19
|
+
require 'fluent/event'
|
20
|
+
|
21
|
+
module Fluent::Plugin
|
22
|
+
class CopyOutput < MultiOutput
|
23
|
+
Fluent::Plugin.register_output('copy', self)
|
24
|
+
|
25
|
+
desc 'If true, pass different record to each `store` plugin.'
|
26
|
+
config_param :deep_copy, :bool, default: false, deprecated: "use 'copy_mode' parameter instead"
|
27
|
+
desc 'Pass different record to each `store` plugin by specified method'
|
28
|
+
config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
|
29
|
+
|
30
|
+
attr_reader :ignore_errors, :ignore_if_prev_successes
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
super
|
34
|
+
@ignore_errors = []
|
35
|
+
@ignore_if_prev_successes = []
|
36
|
+
end
|
37
|
+
|
38
|
+
def configure(conf)
|
39
|
+
super
|
40
|
+
|
41
|
+
@copy_proc = gen_copy_proc
|
42
|
+
@stores.each_with_index { |store, i|
|
43
|
+
if i == 0 && store.arg.include?('ignore_if_prev_success')
|
44
|
+
raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
|
45
|
+
end
|
46
|
+
@ignore_errors << (store.arg.include?('ignore_error'))
|
47
|
+
@ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
|
48
|
+
}
|
49
|
+
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
|
50
|
+
log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def multi_workers_ready?
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
58
|
+
def process(tag, es)
|
59
|
+
unless es.repeatable?
|
60
|
+
m = Fluent::MultiEventStream.new
|
61
|
+
es.each {|time,record|
|
62
|
+
m.add(time, record)
|
63
|
+
}
|
64
|
+
es = m
|
65
|
+
end
|
66
|
+
success = Array.new(outputs.size)
|
67
|
+
outputs.each_with_index do |output, i|
|
68
|
+
begin
|
69
|
+
if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
|
70
|
+
log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
|
71
|
+
else
|
72
|
+
output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
|
73
|
+
success[i] = true
|
74
|
+
end
|
75
|
+
rescue => e
|
76
|
+
if @ignore_errors[i]
|
77
|
+
log.error "ignore emit error in #{output.plugin_id}", error: e
|
78
|
+
else
|
79
|
+
raise e
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def gen_copy_proc
|
88
|
+
@copy_mode = :shallow if @deep_copy
|
89
|
+
|
90
|
+
case @copy_mode
|
91
|
+
when :no_copy
|
92
|
+
nil
|
93
|
+
when :shallow
|
94
|
+
Proc.new { |es| es.dup }
|
95
|
+
when :deep
|
96
|
+
Proc.new { |es|
|
97
|
+
packer = Fluent::MessagePackFactory.msgpack_packer
|
98
|
+
times = []
|
99
|
+
records = []
|
100
|
+
es.each { |time, record|
|
101
|
+
times << time
|
102
|
+
packer.pack(record)
|
103
|
+
}
|
104
|
+
Fluent::MessagePackFactory.msgpack_unpacker.feed_each(packer.full_pack) { |record|
|
105
|
+
records << record
|
106
|
+
}
|
107
|
+
Fluent::MultiEventStream.new(times, records)
|
108
|
+
}
|
109
|
+
when :marshal
|
110
|
+
Proc.new { |es|
|
111
|
+
new_es = Fluent::MultiEventStream.new
|
112
|
+
es.each { |time, record|
|
113
|
+
new_es.add(time, Marshal.load(Marshal.dump(record)))
|
114
|
+
}
|
115
|
+
new_es
|
116
|
+
}
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'tempfile'
|
18
|
+
|
19
|
+
require 'fluent/plugin/output'
|
20
|
+
require 'fluent/config/error'
|
21
|
+
|
22
|
+
module Fluent::Plugin
|
23
|
+
class ExecOutput < Output
|
24
|
+
Fluent::Plugin.register_output('exec', self)
|
25
|
+
|
26
|
+
helpers :inject, :formatter, :compat_parameters, :child_process
|
27
|
+
|
28
|
+
desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argument'
|
29
|
+
config_param :command, :string
|
30
|
+
|
31
|
+
config_param :command_timeout, :time, default: 270 # 4min 30sec
|
32
|
+
|
33
|
+
config_section :format do
|
34
|
+
config_set_default :@type, 'tsv'
|
35
|
+
end
|
36
|
+
|
37
|
+
config_section :inject do
|
38
|
+
config_set_default :time_type, :string
|
39
|
+
config_set_default :localtime, false
|
40
|
+
end
|
41
|
+
|
42
|
+
config_section :buffer do
|
43
|
+
config_set_default :delayed_commit_timeout, 300 # 5 min
|
44
|
+
end
|
45
|
+
|
46
|
+
attr_reader :formatter # for tests
|
47
|
+
|
48
|
+
def configure(conf)
|
49
|
+
compat_parameters_convert(conf, :inject, :formatter, :buffer, default_chunk_key: 'time')
|
50
|
+
super
|
51
|
+
@formatter = formatter_create
|
52
|
+
end
|
53
|
+
|
54
|
+
def multi_workers_ready?
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
58
|
+
NEWLINE = "\n"
|
59
|
+
|
60
|
+
def format(tag, time, record)
|
61
|
+
record = inject_values_to_record(tag, time, record)
|
62
|
+
if @formatter.formatter_type == :text_per_line
|
63
|
+
@formatter.format(tag, time, record).chomp + NEWLINE
|
64
|
+
else
|
65
|
+
@formatter.format(tag, time, record)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def try_write(chunk)
|
70
|
+
tmpfile = nil
|
71
|
+
prog = if chunk.respond_to?(:path)
|
72
|
+
"#{@command} #{chunk.path}"
|
73
|
+
else
|
74
|
+
tmpfile = Tempfile.new("fluent-plugin-out-exec-")
|
75
|
+
tmpfile.binmode
|
76
|
+
chunk.write_to(tmpfile)
|
77
|
+
tmpfile.close
|
78
|
+
"#{@command} #{tmpfile.path}"
|
79
|
+
end
|
80
|
+
chunk_id = chunk.unique_id
|
81
|
+
callback = ->(status){
|
82
|
+
begin
|
83
|
+
if tmpfile
|
84
|
+
tmpfile.delete rescue nil
|
85
|
+
end
|
86
|
+
if status && status.success?
|
87
|
+
commit_write(chunk_id)
|
88
|
+
elsif status
|
89
|
+
# #rollback_write will be done automatically if it isn't called at here.
|
90
|
+
# But it's after command_timeout, and this timeout should be longer than users expectation.
|
91
|
+
# So here, this plugin calls it explicitly.
|
92
|
+
rollback_write(chunk_id)
|
93
|
+
log.warn "command exits with error code", prog: prog, status: status.exitstatus, signal: status.termsig
|
94
|
+
else
|
95
|
+
rollback_write(chunk_id)
|
96
|
+
log.warn "command unexpectedly exits without exit status", prog: prog
|
97
|
+
end
|
98
|
+
rescue => e
|
99
|
+
log.error "unexpected error in child process callback", error: e
|
100
|
+
end
|
101
|
+
}
|
102
|
+
child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: @command_timeout, on_exit_callback: callback)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|