fluentd 1.14.4-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
@@ -0,0 +1,161 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
require 'fluent/plugin_helper/socket'
|
19
|
+
require 'fluent/engine'
|
20
|
+
require 'fluent/clock'
|
21
|
+
|
22
|
+
module Fluent::Plugin
|
23
|
+
class ForwardOutput < Output
|
24
|
+
class AckHandler
|
25
|
+
module Result
|
26
|
+
SUCCESS = :success
|
27
|
+
FAILED = :failed
|
28
|
+
CHUNKID_UNMATCHED = :chunkid_unmatched
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize(timeout:, log:, read_length:)
|
32
|
+
@mutex = Mutex.new
|
33
|
+
@ack_waitings = []
|
34
|
+
@timeout = timeout
|
35
|
+
@log = log
|
36
|
+
@read_length = read_length
|
37
|
+
@unpacker = Fluent::MessagePackFactory.msgpack_unpacker
|
38
|
+
end
|
39
|
+
|
40
|
+
def collect_response(select_interval)
|
41
|
+
now = Fluent::Clock.now
|
42
|
+
sockets = []
|
43
|
+
results = []
|
44
|
+
begin
|
45
|
+
new_list = []
|
46
|
+
@mutex.synchronize do
|
47
|
+
@ack_waitings.each do |info|
|
48
|
+
if info.expired?(now)
|
49
|
+
# There are 2 types of cases when no response has been received from socket:
|
50
|
+
# (1) the node does not support sending responses
|
51
|
+
# (2) the node does support sending response but responses have not arrived for some reasons.
|
52
|
+
@log.warn 'no response from node. regard it as unavailable.', host: info.node.host, port: info.node.port
|
53
|
+
results << [info, Result::FAILED]
|
54
|
+
else
|
55
|
+
sockets << info.sock
|
56
|
+
new_list << info
|
57
|
+
end
|
58
|
+
end
|
59
|
+
@ack_waitings = new_list
|
60
|
+
end
|
61
|
+
|
62
|
+
readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
|
63
|
+
if readable_sockets
|
64
|
+
readable_sockets.each do |sock|
|
65
|
+
results << read_ack_from_sock(sock)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
results.each do |info, ret|
|
70
|
+
if info.nil?
|
71
|
+
yield nil, nil, nil, ret
|
72
|
+
else
|
73
|
+
yield info.chunk_id, info.node, info.sock, ret
|
74
|
+
end
|
75
|
+
end
|
76
|
+
rescue => e
|
77
|
+
@log.error 'unexpected error while receiving ack', error: e
|
78
|
+
@log.error_backtrace
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do
|
83
|
+
def expired?(now)
|
84
|
+
expired_time < now
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
Ack = Struct.new(:chunk_id, :node, :handler) do
|
89
|
+
def enqueue(sock)
|
90
|
+
handler.enqueue(node, sock, chunk_id)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def create_ack(chunk_id, node)
|
95
|
+
Ack.new(chunk_id, node, self)
|
96
|
+
end
|
97
|
+
|
98
|
+
def enqueue(node, sock, cid)
|
99
|
+
info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout)
|
100
|
+
@mutex.synchronize do
|
101
|
+
@ack_waitings << info
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def read_ack_from_sock(sock)
|
108
|
+
begin
|
109
|
+
raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
|
110
|
+
rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
|
111
|
+
raw_data = ''
|
112
|
+
end
|
113
|
+
|
114
|
+
info = find(sock)
|
115
|
+
|
116
|
+
# When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
|
117
|
+
# If this happens we assume the data wasn't delivered and retry it.
|
118
|
+
if raw_data.empty?
|
119
|
+
@log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
|
120
|
+
# info.node.disable!
|
121
|
+
return info, Result::FAILED
|
122
|
+
else
|
123
|
+
@unpacker.feed(raw_data)
|
124
|
+
res = @unpacker.read
|
125
|
+
@log.trace 'getting response from destination', host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
|
126
|
+
if res['ack'] != info.chunk_id_base64
|
127
|
+
# Some errors may have occurred when ack and chunk id is different, so send the chunk again.
|
128
|
+
@log.warn 'ack in response and chunk id in sent data are different', chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
|
129
|
+
return info, Result::CHUNKID_UNMATCHED
|
130
|
+
else
|
131
|
+
@log.trace 'got a correct ack response', chunk_id: dump_unique_id_hex(info.chunk_id)
|
132
|
+
end
|
133
|
+
|
134
|
+
return info, Result::SUCCESS
|
135
|
+
end
|
136
|
+
rescue => e
|
137
|
+
@log.error 'unexpected error while receiving ack message', error: e
|
138
|
+
@log.error_backtrace
|
139
|
+
[nil, Result::FAILED]
|
140
|
+
ensure
|
141
|
+
delete(info)
|
142
|
+
end
|
143
|
+
|
144
|
+
def dump_unique_id_hex(unique_id)
|
145
|
+
Fluent::UniqueId.hex(unique_id)
|
146
|
+
end
|
147
|
+
|
148
|
+
def find(sock)
|
149
|
+
@mutex.synchronize do
|
150
|
+
@ack_waitings.find { |info| info.sock == sock }
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def delete(info)
|
155
|
+
@mutex.synchronize do
|
156
|
+
@ack_waitings.delete(info)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
|
19
|
+
module Fluent::Plugin
|
20
|
+
class ForwardOutput < Output
|
21
|
+
class ConnectionManager
|
22
|
+
RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
|
23
|
+
|
24
|
+
# @param log [Logger]
|
25
|
+
# @param secure [Boolean]
|
26
|
+
# @param connection_factory [Proc]
|
27
|
+
# @param SocketCache [Fluent::ForwardOutput::SocketCache]
|
28
|
+
def initialize(log:, secure:, connection_factory:, socket_cache:)
|
29
|
+
@log = log
|
30
|
+
@secure = secure
|
31
|
+
@connection_factory = connection_factory
|
32
|
+
@socket_cache = socket_cache
|
33
|
+
end
|
34
|
+
|
35
|
+
def stop
|
36
|
+
@socket_cache && @socket_cache.clear
|
37
|
+
end
|
38
|
+
|
39
|
+
# @param ack [Fluent::Plugin::ForwardOutput::AckHander::Ack|nil]
|
40
|
+
def connect(host:, port:, hostname:, ack: nil, &block)
|
41
|
+
if @socket_cache
|
42
|
+
return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
@log.debug('connect new socket')
|
46
|
+
socket = @connection_factory.call(host, port, hostname)
|
47
|
+
request_info = RequestInfo.new(@secure ? :helo : :established)
|
48
|
+
|
49
|
+
unless block_given?
|
50
|
+
return [socket, request_info]
|
51
|
+
end
|
52
|
+
|
53
|
+
begin
|
54
|
+
yield(socket, request_info)
|
55
|
+
ensure
|
56
|
+
if ack
|
57
|
+
ack.enqueue(socket)
|
58
|
+
else
|
59
|
+
socket.close_write rescue nil
|
60
|
+
socket.close rescue nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def purge_obsolete_socks
|
66
|
+
unless @socket_cache
|
67
|
+
raise "Do not call this method without keepalive option"
|
68
|
+
end
|
69
|
+
@socket_cache.purge_obsolete_socks
|
70
|
+
end
|
71
|
+
|
72
|
+
def close(sock)
|
73
|
+
if @socket_cache
|
74
|
+
@socket_cache.checkin(sock)
|
75
|
+
else
|
76
|
+
sock.close_write rescue nil
|
77
|
+
sock.close rescue nil
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def connect_keepalive(host:, port:, hostname:, ack: nil)
|
84
|
+
request_info = RequestInfo.new(:established)
|
85
|
+
socket = @socket_cache.checkout_or([host, port, hostname]) do
|
86
|
+
s = @connection_factory.call(host, port, hostname)
|
87
|
+
request_info = RequestInfo.new(@secure ? :helo : :established) # overwrite if new connection
|
88
|
+
s
|
89
|
+
end
|
90
|
+
|
91
|
+
unless block_given?
|
92
|
+
return [socket, request_info]
|
93
|
+
end
|
94
|
+
|
95
|
+
ret = nil
|
96
|
+
begin
|
97
|
+
ret = yield(socket, request_info)
|
98
|
+
rescue
|
99
|
+
@socket_cache.revoke(socket)
|
100
|
+
raise
|
101
|
+
else
|
102
|
+
if ack
|
103
|
+
ack.enqueue(socket)
|
104
|
+
else
|
105
|
+
@socket_cache.checkin(socket)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
ret
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
|
19
|
+
module Fluent::Plugin
|
20
|
+
class ForwardOutput < Output
|
21
|
+
class Error < StandardError; end
|
22
|
+
class NoNodesAvailable < Error; end
|
23
|
+
class ConnectionClosedError < Error; end
|
24
|
+
class HandshakeError < Error; end
|
25
|
+
class HeloError < HandshakeError; end
|
26
|
+
class PingpongError < HandshakeError; end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
|
19
|
+
module Fluent::Plugin
|
20
|
+
class ForwardOutput < Output
|
21
|
+
class FailureDetector
|
22
|
+
PHI_FACTOR = 1.0 / Math.log(10.0)
|
23
|
+
SAMPLE_SIZE = 1000
|
24
|
+
|
25
|
+
def initialize(heartbeat_interval, hard_timeout, init_last)
|
26
|
+
@heartbeat_interval = heartbeat_interval
|
27
|
+
@last = init_last
|
28
|
+
@hard_timeout = hard_timeout
|
29
|
+
|
30
|
+
# microsec
|
31
|
+
@init_gap = (heartbeat_interval * 1e6).to_i
|
32
|
+
@window = [@init_gap]
|
33
|
+
end
|
34
|
+
|
35
|
+
def hard_timeout?(now)
|
36
|
+
now - @last > @hard_timeout
|
37
|
+
end
|
38
|
+
|
39
|
+
def add(now)
|
40
|
+
if @window.empty?
|
41
|
+
@window << @init_gap
|
42
|
+
@last = now
|
43
|
+
else
|
44
|
+
gap = now - @last
|
45
|
+
@window << (gap * 1e6).to_i
|
46
|
+
@window.shift if @window.length > SAMPLE_SIZE
|
47
|
+
@last = now
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def phi(now)
|
52
|
+
size = @window.size
|
53
|
+
return 0.0 if size == 0
|
54
|
+
|
55
|
+
# Calculate weighted moving average
|
56
|
+
mean_usec = 0
|
57
|
+
fact = 0
|
58
|
+
@window.each_with_index {|gap,i|
|
59
|
+
mean_usec += gap * (1+i)
|
60
|
+
fact += (1+i)
|
61
|
+
}
|
62
|
+
mean_usec = mean_usec / fact
|
63
|
+
|
64
|
+
# Normalize arrive intervals into 1sec
|
65
|
+
mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1
|
66
|
+
|
67
|
+
# Calculate phi of the phi accrual failure detector
|
68
|
+
t = now - @last - @heartbeat_interval + 1
|
69
|
+
phi = PHI_FACTOR * t / mean
|
70
|
+
|
71
|
+
return phi
|
72
|
+
end
|
73
|
+
|
74
|
+
def sample_size
|
75
|
+
@window.size
|
76
|
+
end
|
77
|
+
|
78
|
+
def clear
|
79
|
+
@window.clear
|
80
|
+
@last = 0
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
require 'fluent/plugin/out_forward/error'
|
19
|
+
require 'digest'
|
20
|
+
|
21
|
+
module Fluent::Plugin
|
22
|
+
class ForwardOutput < Output
|
23
|
+
class HandshakeProtocol
|
24
|
+
def initialize(log:, hostname:, shared_key:, password:, username:)
|
25
|
+
@log = log
|
26
|
+
@hostname = hostname
|
27
|
+
@shared_key = shared_key
|
28
|
+
@password = password
|
29
|
+
@username = username
|
30
|
+
@shared_key_salt = generate_salt
|
31
|
+
end
|
32
|
+
|
33
|
+
def invoke(sock, ri, data)
|
34
|
+
@log.trace __callee__
|
35
|
+
|
36
|
+
case ri.state
|
37
|
+
when :helo
|
38
|
+
unless check_helo(ri, data)
|
39
|
+
raise HeloError, 'received invalid helo message'
|
40
|
+
end
|
41
|
+
|
42
|
+
sock.write(generate_ping(ri).to_msgpack)
|
43
|
+
ri.state = :pingpong
|
44
|
+
when :pingpong
|
45
|
+
succeeded, reason = check_pong(ri, data)
|
46
|
+
unless succeeded
|
47
|
+
raise PingpongError, reason
|
48
|
+
end
|
49
|
+
|
50
|
+
ri.state = :established
|
51
|
+
else
|
52
|
+
raise "BUG: unknown session state: #{ri.state}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def check_pong(ri, message)
|
59
|
+
@log.debug('checking pong')
|
60
|
+
# ['PONG', bool(authentication result), 'reason if authentication failed',
|
61
|
+
# self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
|
62
|
+
unless message.size == 5 && message[0] == 'PONG'
|
63
|
+
return false, 'invalid format for PONG message'
|
64
|
+
end
|
65
|
+
_pong, auth_result, reason, hostname, shared_key_hexdigest = message
|
66
|
+
|
67
|
+
unless auth_result
|
68
|
+
return false, 'authentication failed: ' + reason
|
69
|
+
end
|
70
|
+
|
71
|
+
if hostname == @hostname
|
72
|
+
return false, 'same hostname between input and output: invalid configuration'
|
73
|
+
end
|
74
|
+
|
75
|
+
clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest
|
76
|
+
unless shared_key_hexdigest == clientside
|
77
|
+
return false, 'shared key mismatch'
|
78
|
+
end
|
79
|
+
|
80
|
+
[true, nil]
|
81
|
+
end
|
82
|
+
|
83
|
+
def check_helo(ri, message)
|
84
|
+
@log.debug('checking helo')
|
85
|
+
# ['HELO', options(hash)]
|
86
|
+
unless message.size == 2 && message[0] == 'HELO'
|
87
|
+
return false
|
88
|
+
end
|
89
|
+
|
90
|
+
opts = message[1] || {}
|
91
|
+
# make shared_key_check failed (instead of error) if protocol version mismatch exist
|
92
|
+
ri.shared_key_nonce = opts['nonce'] || ''
|
93
|
+
ri.auth = opts['auth'] || ''
|
94
|
+
true
|
95
|
+
end
|
96
|
+
|
97
|
+
def generate_ping(ri)
|
98
|
+
@log.debug('generating ping')
|
99
|
+
# ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
|
100
|
+
# username || '', sha512\_hex(auth\_salt + username + password) || '']
|
101
|
+
shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
|
102
|
+
.update(@hostname)
|
103
|
+
.update(ri.shared_key_nonce)
|
104
|
+
.update(@shared_key)
|
105
|
+
.hexdigest
|
106
|
+
ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]
|
107
|
+
if !ri.auth.empty?
|
108
|
+
if @username.nil? || @password.nil?
|
109
|
+
raise PingpongError, "username and password are required"
|
110
|
+
end
|
111
|
+
|
112
|
+
password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
|
113
|
+
ping.push(@username, password_hexdigest)
|
114
|
+
else
|
115
|
+
ping.push('', '')
|
116
|
+
end
|
117
|
+
ping
|
118
|
+
end
|
119
|
+
|
120
|
+
def generate_salt
|
121
|
+
SecureRandom.hex(16)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
require 'fluent/plugin/out_forward/error'
|
19
|
+
|
20
|
+
module Fluent::Plugin
|
21
|
+
class ForwardOutput < Output
|
22
|
+
class LoadBalancer
|
23
|
+
def initialize(log)
|
24
|
+
@log = log
|
25
|
+
@weight_array = []
|
26
|
+
@rand_seed = Random.new.seed
|
27
|
+
@rr = 0
|
28
|
+
@mutex = Mutex.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def select_healthy_node
|
32
|
+
error = nil
|
33
|
+
|
34
|
+
# Don't care about the change of @weight_array's size while looping since
|
35
|
+
# it's only used for determining the number of loops and it is not so important.
|
36
|
+
wlen = @weight_array.size
|
37
|
+
wlen.times do
|
38
|
+
node = @mutex.synchronize do
|
39
|
+
r = @rr % @weight_array.size
|
40
|
+
@rr = (r + 1) % @weight_array.size
|
41
|
+
@weight_array[r]
|
42
|
+
end
|
43
|
+
next unless node.available?
|
44
|
+
|
45
|
+
begin
|
46
|
+
ret = yield node
|
47
|
+
return ret, node
|
48
|
+
rescue
|
49
|
+
# for load balancing during detecting crashed servers
|
50
|
+
error = $! # use the latest error
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
raise error if error
|
55
|
+
raise NoNodesAvailable, "no nodes are available"
|
56
|
+
end
|
57
|
+
|
58
|
+
def rebuild_weight_array(nodes)
|
59
|
+
standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
|
60
|
+
n.standby?
|
61
|
+
}
|
62
|
+
|
63
|
+
lost_weight = 0
|
64
|
+
regular_nodes.each {|n|
|
65
|
+
unless n.available?
|
66
|
+
lost_weight += n.weight
|
67
|
+
end
|
68
|
+
}
|
69
|
+
@log.debug("rebuilding weight array", lost_weight: lost_weight)
|
70
|
+
|
71
|
+
if lost_weight > 0
|
72
|
+
standby_nodes.each {|n|
|
73
|
+
if n.available?
|
74
|
+
regular_nodes << n
|
75
|
+
@log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
|
76
|
+
lost_weight -= n.weight
|
77
|
+
break if lost_weight <= 0
|
78
|
+
end
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
weight_array = []
|
83
|
+
if regular_nodes.empty?
|
84
|
+
@log.warn('No nodes are available')
|
85
|
+
@mutex.synchronize do
|
86
|
+
@weight_array = weight_array
|
87
|
+
end
|
88
|
+
return @weight_array
|
89
|
+
end
|
90
|
+
|
91
|
+
gcd = regular_nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
|
92
|
+
regular_nodes.each {|n|
|
93
|
+
(n.weight / gcd).times {
|
94
|
+
weight_array << n
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
98
|
+
# for load balancing during detecting crashed servers
|
99
|
+
coe = (regular_nodes.size * 6) / weight_array.size
|
100
|
+
weight_array *= coe if coe > 1
|
101
|
+
|
102
|
+
r = Random.new(@rand_seed)
|
103
|
+
weight_array.sort_by! { r.rand }
|
104
|
+
|
105
|
+
@mutex.synchronize do
|
106
|
+
@weight_array = weight_array
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
alias select_service select_healthy_node
|
111
|
+
alias rebalance rebuild_weight_array
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|