fluentd 1.14.4-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
@@ -0,0 +1,1054 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fileutils'
|
18
|
+
require 'open3'
|
19
|
+
|
20
|
+
require 'fluent/config'
|
21
|
+
require 'fluent/counter'
|
22
|
+
require 'fluent/env'
|
23
|
+
require 'fluent/engine'
|
24
|
+
require 'fluent/error'
|
25
|
+
require 'fluent/log'
|
26
|
+
require 'fluent/plugin'
|
27
|
+
require 'fluent/rpc'
|
28
|
+
require 'fluent/system_config'
|
29
|
+
require 'fluent/msgpack_factory'
|
30
|
+
require 'fluent/variable_store'
|
31
|
+
require 'serverengine'
|
32
|
+
|
33
|
+
if Fluent.windows?
|
34
|
+
require 'windows/library'
|
35
|
+
require 'windows/synchronize'
|
36
|
+
require 'windows/system_info'
|
37
|
+
include Windows::Library
|
38
|
+
include Windows::Synchronize
|
39
|
+
include Windows::SystemInfo
|
40
|
+
require 'win32/ipc'
|
41
|
+
require 'win32/event'
|
42
|
+
end
|
43
|
+
|
44
|
+
module Fluent
|
45
|
+
module ServerModule
|
46
|
+
def before_run
|
47
|
+
@fluentd_conf = config[:fluentd_conf]
|
48
|
+
@rpc_endpoint = nil
|
49
|
+
@rpc_server = nil
|
50
|
+
@counter = nil
|
51
|
+
|
52
|
+
if config[:rpc_endpoint]
|
53
|
+
@rpc_endpoint = config[:rpc_endpoint]
|
54
|
+
@enable_get_dump = config[:enable_get_dump]
|
55
|
+
run_rpc_server
|
56
|
+
end
|
57
|
+
|
58
|
+
if Fluent.windows?
|
59
|
+
install_windows_event_handler
|
60
|
+
else
|
61
|
+
install_supervisor_signal_handlers
|
62
|
+
end
|
63
|
+
|
64
|
+
if counter = config[:counter_server]
|
65
|
+
run_counter_server(counter)
|
66
|
+
end
|
67
|
+
|
68
|
+
if config[:disable_shared_socket]
|
69
|
+
$log.info "shared socket for multiple workers is disabled"
|
70
|
+
else
|
71
|
+
socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
72
|
+
ServerEngine::SocketManager::Server.open(socket_manager_path)
|
73
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def after_run
|
78
|
+
stop_windows_event_thread if Fluent.windows?
|
79
|
+
stop_rpc_server if @rpc_endpoint
|
80
|
+
stop_counter_server if @counter
|
81
|
+
Fluent::Supervisor.cleanup_resources
|
82
|
+
end
|
83
|
+
|
84
|
+
def run_rpc_server
|
85
|
+
@rpc_server = RPC::Server.new(@rpc_endpoint, $log)
|
86
|
+
|
87
|
+
# built-in RPC for signals
|
88
|
+
@rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
|
89
|
+
$log.debug "fluentd RPC got /api/processes.interruptWorkers request"
|
90
|
+
Process.kill :INT, $$
|
91
|
+
nil
|
92
|
+
}
|
93
|
+
@rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
|
94
|
+
$log.debug "fluentd RPC got /api/processes.killWorkers request"
|
95
|
+
Process.kill :TERM, $$
|
96
|
+
nil
|
97
|
+
}
|
98
|
+
@rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
|
99
|
+
$log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
|
100
|
+
if Fluent.windows?
|
101
|
+
supervisor_sigusr1_handler
|
102
|
+
stop(true)
|
103
|
+
else
|
104
|
+
Process.kill :USR1, $$
|
105
|
+
Process.kill :TERM, $$
|
106
|
+
end
|
107
|
+
nil
|
108
|
+
}
|
109
|
+
@rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
|
110
|
+
$log.debug "fluentd RPC got /api/plugins.flushBuffers request"
|
111
|
+
if Fluent.windows?
|
112
|
+
supervisor_sigusr1_handler
|
113
|
+
else
|
114
|
+
Process.kill :USR1, $$
|
115
|
+
end
|
116
|
+
nil
|
117
|
+
}
|
118
|
+
@rpc_server.mount_proc('/api/config.reload') { |req, res|
|
119
|
+
$log.debug "fluentd RPC got /api/config.reload request"
|
120
|
+
if Fluent.windows?
|
121
|
+
# restart worker with auto restarting by killing
|
122
|
+
kill_worker
|
123
|
+
else
|
124
|
+
Process.kill :HUP, $$
|
125
|
+
end
|
126
|
+
nil
|
127
|
+
}
|
128
|
+
@rpc_server.mount_proc('/api/config.dump') { |req, res|
|
129
|
+
$log.debug "fluentd RPC got /api/config.dump request"
|
130
|
+
$log.info "dump in-memory config"
|
131
|
+
supervisor_dump_config_handler
|
132
|
+
nil
|
133
|
+
}
|
134
|
+
|
135
|
+
@rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
|
136
|
+
$log.debug "fluentd RPC got /api/config.gracefulReload request"
|
137
|
+
if Fluent.windows?
|
138
|
+
supervisor_sigusr2_handler
|
139
|
+
else
|
140
|
+
Process.kill :USR2, $$
|
141
|
+
end
|
142
|
+
|
143
|
+
nil
|
144
|
+
}
|
145
|
+
|
146
|
+
@rpc_server.mount_proc('/api/config.getDump') { |req, res|
|
147
|
+
$log.debug "fluentd RPC got /api/config.getDump request"
|
148
|
+
$log.info "get dump in-memory config via HTTP"
|
149
|
+
res.body = supervisor_get_dump_config_handler
|
150
|
+
[nil, nil, res]
|
151
|
+
} if @enable_get_dump
|
152
|
+
|
153
|
+
@rpc_server.start
|
154
|
+
end
|
155
|
+
|
156
|
+
def stop_rpc_server
|
157
|
+
@rpc_server.shutdown
|
158
|
+
end
|
159
|
+
|
160
|
+
def run_counter_server(counter_conf)
|
161
|
+
@counter = Fluent::Counter::Server.new(
|
162
|
+
counter_conf.scope,
|
163
|
+
{host: counter_conf.bind, port: counter_conf.port, log: $log, path: counter_conf.backup_path}
|
164
|
+
)
|
165
|
+
@counter.start
|
166
|
+
end
|
167
|
+
|
168
|
+
def stop_counter_server
|
169
|
+
@counter.stop
|
170
|
+
end
|
171
|
+
|
172
|
+
def install_supervisor_signal_handlers
|
173
|
+
return if Fluent.windows?
|
174
|
+
|
175
|
+
trap :HUP do
|
176
|
+
$log.debug "fluentd supervisor process get SIGHUP"
|
177
|
+
supervisor_sighup_handler
|
178
|
+
end
|
179
|
+
|
180
|
+
trap :USR1 do
|
181
|
+
$log.debug "fluentd supervisor process get SIGUSR1"
|
182
|
+
supervisor_sigusr1_handler
|
183
|
+
end
|
184
|
+
|
185
|
+
trap :USR2 do
|
186
|
+
$log.debug 'fluentd supervisor process got SIGUSR2'
|
187
|
+
supervisor_sigusr2_handler
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
if Fluent.windows?
|
192
|
+
# Override some methods of ServerEngine::MultiSpawnWorker
|
193
|
+
# Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
|
194
|
+
# handlers (see install_supervisor_signal_handlers), they should be
|
195
|
+
# disabled also on Windows, just send commands to workers instead.
|
196
|
+
def restart(graceful)
|
197
|
+
@monitors.each do |m|
|
198
|
+
m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def reload
|
203
|
+
@monitors.each do |m|
|
204
|
+
m.send_command("RELOAD\n")
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def install_windows_event_handler
|
210
|
+
return unless Fluent.windows?
|
211
|
+
|
212
|
+
@pid_signame = "fluentd_#{$$}"
|
213
|
+
@signame = config[:signame]
|
214
|
+
|
215
|
+
Thread.new do
|
216
|
+
ipc = Win32::Ipc.new(nil)
|
217
|
+
events = [
|
218
|
+
Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
|
219
|
+
Win32::Event.new("#{@pid_signame}"),
|
220
|
+
Win32::Event.new("#{@pid_signame}_HUP"),
|
221
|
+
Win32::Event.new("#{@pid_signame}_USR1"),
|
222
|
+
Win32::Event.new("#{@pid_signame}_USR2"),
|
223
|
+
]
|
224
|
+
if @signame
|
225
|
+
signame_events = [
|
226
|
+
Win32::Event.new("#{@signame}"),
|
227
|
+
Win32::Event.new("#{@signame}_HUP"),
|
228
|
+
Win32::Event.new("#{@signame}_USR1"),
|
229
|
+
Win32::Event.new("#{@signame}_USR2"),
|
230
|
+
]
|
231
|
+
events.concat(signame_events)
|
232
|
+
end
|
233
|
+
begin
|
234
|
+
loop do
|
235
|
+
idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
|
236
|
+
if idx > 0 && idx <= events.length
|
237
|
+
$log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
|
238
|
+
else
|
239
|
+
$log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
|
240
|
+
end
|
241
|
+
case idx
|
242
|
+
when 2, 6
|
243
|
+
stop(true)
|
244
|
+
when 3, 7
|
245
|
+
supervisor_sighup_handler
|
246
|
+
when 4, 8
|
247
|
+
supervisor_sigusr1_handler
|
248
|
+
when 5, 9
|
249
|
+
supervisor_sigusr2_handler
|
250
|
+
when 1
|
251
|
+
break
|
252
|
+
end
|
253
|
+
end
|
254
|
+
ensure
|
255
|
+
events.each { |event| event.close }
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
def stop_windows_event_thread
|
261
|
+
if Fluent.windows?
|
262
|
+
ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
|
263
|
+
ev.set
|
264
|
+
ev.close
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
def supervisor_sighup_handler
|
269
|
+
kill_worker
|
270
|
+
end
|
271
|
+
|
272
|
+
def supervisor_sigusr1_handler
|
273
|
+
reopen_log
|
274
|
+
send_signal_to_workers(:USR1)
|
275
|
+
end
|
276
|
+
|
277
|
+
def supervisor_sigusr2_handler
|
278
|
+
conf = nil
|
279
|
+
t = Thread.new do
|
280
|
+
$log.info 'Reloading new config'
|
281
|
+
|
282
|
+
# Validate that loading config is valid at first
|
283
|
+
conf = Fluent::Config.build(
|
284
|
+
config_path: config[:config_path],
|
285
|
+
encoding: config[:conf_encoding],
|
286
|
+
additional_config: config[:inline_config],
|
287
|
+
use_v1_config: config[:use_v1_config],
|
288
|
+
)
|
289
|
+
|
290
|
+
Fluent::VariableStore.try_to_reset do
|
291
|
+
Fluent::Engine.reload_config(conf, supervisor: true)
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
t.report_on_exception = false # Error is handled by myself
|
296
|
+
t.join
|
297
|
+
|
298
|
+
reopen_log
|
299
|
+
send_signal_to_workers(:USR2)
|
300
|
+
@fluentd_conf = conf.to_s
|
301
|
+
rescue => e
|
302
|
+
$log.error "Failed to reload config file: #{e}"
|
303
|
+
end
|
304
|
+
|
305
|
+
def kill_worker
|
306
|
+
if config[:worker_pid]
|
307
|
+
pids = config[:worker_pid].clone
|
308
|
+
config[:worker_pid].clear
|
309
|
+
pids.each_value do |pid|
|
310
|
+
if Fluent.windows?
|
311
|
+
Process.kill :KILL, pid
|
312
|
+
else
|
313
|
+
Process.kill :TERM, pid
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
|
319
|
+
def supervisor_dump_config_handler
|
320
|
+
$log.info @fluentd_conf
|
321
|
+
end
|
322
|
+
|
323
|
+
def supervisor_get_dump_config_handler
|
324
|
+
{ conf: @fluentd_conf }
|
325
|
+
end
|
326
|
+
|
327
|
+
private
|
328
|
+
|
329
|
+
def reopen_log
|
330
|
+
if (log = config[:logger_initializer])
|
331
|
+
# Creating new thread due to mutex can't lock
|
332
|
+
# in main thread during trap context
|
333
|
+
Thread.new do
|
334
|
+
log.reopen!
|
335
|
+
end
|
336
|
+
end
|
337
|
+
end
|
338
|
+
|
339
|
+
def send_signal_to_workers(signal)
|
340
|
+
return unless config[:worker_pid]
|
341
|
+
|
342
|
+
if Fluent.windows?
|
343
|
+
send_command_to_workers(signal)
|
344
|
+
else
|
345
|
+
config[:worker_pid].each_value do |pid|
|
346
|
+
# don't rescue Errno::ESRCH here (invalid status)
|
347
|
+
Process.kill(signal, pid)
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
def send_command_to_workers(signal)
|
353
|
+
# Use SeverEngine's CommandSender on Windows
|
354
|
+
case signal
|
355
|
+
when :HUP
|
356
|
+
restart(false)
|
357
|
+
when :USR1
|
358
|
+
restart(true)
|
359
|
+
when :USR2
|
360
|
+
reload
|
361
|
+
end
|
362
|
+
end
|
363
|
+
end
|
364
|
+
|
365
|
+
module WorkerModule
|
366
|
+
def spawn(process_manager)
|
367
|
+
main_cmd = config[:main_cmd]
|
368
|
+
env = {
|
369
|
+
'SERVERENGINE_WORKER_ID' => @worker_id.to_i.to_s,
|
370
|
+
}
|
371
|
+
@pm = process_manager.spawn(env, *main_cmd)
|
372
|
+
end
|
373
|
+
|
374
|
+
def after_start
|
375
|
+
(config[:worker_pid] ||= {})[@worker_id] = @pm.pid
|
376
|
+
end
|
377
|
+
end
|
378
|
+
|
379
|
+
class Supervisor
|
380
|
+
def self.load_config(path, params = {})
|
381
|
+
pre_loadtime = 0
|
382
|
+
pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
|
383
|
+
pre_config_mtime = nil
|
384
|
+
pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
|
385
|
+
config_mtime = File.mtime(path)
|
386
|
+
|
387
|
+
# reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
|
388
|
+
if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
|
389
|
+
return params['pre_conf']
|
390
|
+
end
|
391
|
+
|
392
|
+
log_level = params['log_level']
|
393
|
+
suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
|
394
|
+
ignore_repeated_log_interval = params['ignore_repeated_log_interval']
|
395
|
+
ignore_same_log_interval = params['ignore_same_log_interval']
|
396
|
+
|
397
|
+
log_path = params['log_path']
|
398
|
+
chuser = params['chuser']
|
399
|
+
chgroup = params['chgroup']
|
400
|
+
log_rotate_age = params['log_rotate_age']
|
401
|
+
log_rotate_size = params['log_rotate_size']
|
402
|
+
|
403
|
+
log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
|
404
|
+
ignore_same_log_interval: ignore_same_log_interval}
|
405
|
+
logger_initializer = Supervisor::LoggerInitializer.new(
|
406
|
+
log_path, log_level, chuser, chgroup, log_opts,
|
407
|
+
log_rotate_age: log_rotate_age,
|
408
|
+
log_rotate_size: log_rotate_size
|
409
|
+
)
|
410
|
+
# this #init sets initialized logger to $log
|
411
|
+
logger_initializer.init(:supervisor, 0)
|
412
|
+
logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
|
413
|
+
logger = $log
|
414
|
+
|
415
|
+
command_sender = Fluent.windows? ? "pipe" : "signal"
|
416
|
+
|
417
|
+
# ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
|
418
|
+
pid_path = params['daemonize']
|
419
|
+
daemonize = !!params['daemonize']
|
420
|
+
|
421
|
+
se_config = {
|
422
|
+
worker_type: 'spawn',
|
423
|
+
workers: params['workers'],
|
424
|
+
log_stdin: false,
|
425
|
+
log_stdout: false,
|
426
|
+
log_stderr: false,
|
427
|
+
enable_heartbeat: true,
|
428
|
+
auto_heartbeat: false,
|
429
|
+
unrecoverable_exit_codes: [2],
|
430
|
+
stop_immediately_at_unrecoverable_exit: true,
|
431
|
+
root_dir: params['root_dir'],
|
432
|
+
logger: logger,
|
433
|
+
log: logger.out,
|
434
|
+
log_path: log_path,
|
435
|
+
log_level: log_level,
|
436
|
+
logger_initializer: logger_initializer,
|
437
|
+
chuser: chuser,
|
438
|
+
chgroup: chgroup,
|
439
|
+
chumask: 0,
|
440
|
+
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
441
|
+
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
442
|
+
ignore_same_log_interval: ignore_same_log_interval,
|
443
|
+
daemonize: daemonize,
|
444
|
+
rpc_endpoint: params['rpc_endpoint'],
|
445
|
+
counter_server: params['counter_server'],
|
446
|
+
enable_get_dump: params['enable_get_dump'],
|
447
|
+
windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
|
448
|
+
File.join(File.dirname(__FILE__), 'daemon.rb'),
|
449
|
+
ServerModule.name,
|
450
|
+
WorkerModule.name,
|
451
|
+
path,
|
452
|
+
JSON.dump(params)],
|
453
|
+
command_sender: command_sender,
|
454
|
+
fluentd_conf: params['fluentd_conf'],
|
455
|
+
conf_encoding: params['conf_encoding'],
|
456
|
+
inline_config: params['inline_config'],
|
457
|
+
config_path: path,
|
458
|
+
main_cmd: params['main_cmd'],
|
459
|
+
signame: params['signame'],
|
460
|
+
disable_shared_socket: params['disable_shared_socket']
|
461
|
+
}
|
462
|
+
if daemonize
|
463
|
+
se_config[:pid_path] = pid_path
|
464
|
+
end
|
465
|
+
pre_params = params.dup
|
466
|
+
params['pre_loadtime'] = Time.now.to_i
|
467
|
+
params['pre_config_mtime'] = config_mtime
|
468
|
+
params['pre_conf'] = se_config
|
469
|
+
# prevent pre_conf from being too big by reloading many times.
|
470
|
+
pre_params['pre_conf'] = nil
|
471
|
+
params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
|
472
|
+
|
473
|
+
se_config
|
474
|
+
end
|
475
|
+
|
476
|
+
class LoggerInitializer
|
477
|
+
def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
|
478
|
+
@path = path
|
479
|
+
@level = level
|
480
|
+
@chuser = chuser
|
481
|
+
@chgroup = chgroup
|
482
|
+
@opts = opts
|
483
|
+
@log_rotate_age = log_rotate_age
|
484
|
+
@log_rotate_size = log_rotate_size
|
485
|
+
end
|
486
|
+
|
487
|
+
def worker_id_suffixed_path(worker_id, path)
|
488
|
+
require 'pathname'
|
489
|
+
|
490
|
+
Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
|
491
|
+
end
|
492
|
+
|
493
|
+
def init(process_type, worker_id)
|
494
|
+
@opts[:process_type] = process_type
|
495
|
+
@opts[:worker_id] = worker_id
|
496
|
+
|
497
|
+
if @path && @path != "-"
|
498
|
+
unless File.exist?(@path)
|
499
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
500
|
+
end
|
501
|
+
|
502
|
+
@logdev = if @log_rotate_age || @log_rotate_size
|
503
|
+
Fluent::LogDeviceIO.new(Fluent.windows? ?
|
504
|
+
worker_id_suffixed_path(worker_id, @path) : @path,
|
505
|
+
shift_age: @log_rotate_age, shift_size: @log_rotate_size)
|
506
|
+
else
|
507
|
+
File.open(@path, "a")
|
508
|
+
end
|
509
|
+
if @chuser || @chgroup
|
510
|
+
chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
|
511
|
+
chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
|
512
|
+
File.chown(chuid, chgid, @path)
|
513
|
+
end
|
514
|
+
else
|
515
|
+
@logdev = STDOUT
|
516
|
+
end
|
517
|
+
|
518
|
+
dl_opts = {}
|
519
|
+
# subtract 1 to match serverengine daemon logger side logging severity.
|
520
|
+
dl_opts[:log_level] = @level - 1
|
521
|
+
dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
|
522
|
+
dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
|
523
|
+
logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
|
524
|
+
$log = Fluent::Log.new(logger, @opts)
|
525
|
+
$log.enable_color(false) if @path
|
526
|
+
$log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
|
527
|
+
end
|
528
|
+
|
529
|
+
def stdout?
|
530
|
+
@logdev == STDOUT
|
531
|
+
end
|
532
|
+
|
533
|
+
def reopen!
|
534
|
+
if @path && @path != "-"
|
535
|
+
@logdev.reopen(@path, "a")
|
536
|
+
end
|
537
|
+
self
|
538
|
+
end
|
539
|
+
|
540
|
+
def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
|
541
|
+
$log.format = format if format
|
542
|
+
$log.time_format = time_format if time_format
|
543
|
+
$log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
|
544
|
+
$log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
|
545
|
+
|
546
|
+
if @path && log_dir_perm
|
547
|
+
File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
|
548
|
+
end
|
549
|
+
end
|
550
|
+
|
551
|
+
def level=(level)
|
552
|
+
@level = level
|
553
|
+
$log.level = level
|
554
|
+
end
|
555
|
+
end
|
556
|
+
|
557
|
+
def self.default_options
|
558
|
+
{
|
559
|
+
config_path: Fluent::DEFAULT_CONFIG_PATH,
|
560
|
+
plugin_dirs: [Fluent::DEFAULT_PLUGIN_DIR],
|
561
|
+
log_level: Fluent::Log::LEVEL_INFO,
|
562
|
+
log_path: nil,
|
563
|
+
daemonize: nil,
|
564
|
+
libs: [],
|
565
|
+
setup_path: nil,
|
566
|
+
chuser: nil,
|
567
|
+
chgroup: nil,
|
568
|
+
root_dir: nil,
|
569
|
+
suppress_interval: 0,
|
570
|
+
suppress_repeated_stacktrace: true,
|
571
|
+
ignore_repeated_log_interval: nil,
|
572
|
+
without_source: nil,
|
573
|
+
enable_input_metrics: nil,
|
574
|
+
enable_size_metrics: nil,
|
575
|
+
use_v1_config: true,
|
576
|
+
strict_config_value: nil,
|
577
|
+
supervise: true,
|
578
|
+
standalone_worker: false,
|
579
|
+
signame: nil,
|
580
|
+
conf_encoding: 'utf-8',
|
581
|
+
disable_shared_socket: nil
|
582
|
+
}
|
583
|
+
end
|
584
|
+
|
585
|
+
def self.cleanup_resources
|
586
|
+
unless Fluent.windows?
|
587
|
+
if ENV.has_key?('SERVERENGINE_SOCKETMANAGER_PATH')
|
588
|
+
FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
|
589
|
+
end
|
590
|
+
end
|
591
|
+
end
|
592
|
+
|
593
|
+
def initialize(opt)
|
594
|
+
@daemonize = opt[:daemonize]
|
595
|
+
@standalone_worker= opt[:standalone_worker]
|
596
|
+
@config_path = opt[:config_path]
|
597
|
+
@inline_config = opt[:inline_config]
|
598
|
+
@use_v1_config = opt[:use_v1_config]
|
599
|
+
@conf_encoding = opt[:conf_encoding]
|
600
|
+
@log_path = opt[:log_path]
|
601
|
+
@show_plugin_config = opt[:show_plugin_config]
|
602
|
+
@libs = opt[:libs]
|
603
|
+
@plugin_dirs = opt[:plugin_dirs]
|
604
|
+
@chgroup = opt[:chgroup]
|
605
|
+
@chuser = opt[:chuser]
|
606
|
+
|
607
|
+
@log_rotate_age = opt[:log_rotate_age]
|
608
|
+
@log_rotate_size = opt[:log_rotate_size]
|
609
|
+
@signame = opt[:signame]
|
610
|
+
|
611
|
+
@cl_opt = opt
|
612
|
+
@conf = nil
|
613
|
+
# parse configuration immediately to initialize logger in early stage
|
614
|
+
if @config_path and File.exist?(@config_path)
|
615
|
+
@conf = Fluent::Config.build(config_path: @config_path,
|
616
|
+
encoding: @conf_encoding ? @conf_encoding : 'utf-8',
|
617
|
+
additional_config: @inline_config ? @inline_config : nil,
|
618
|
+
use_v1_config: !!@use_v1_config)
|
619
|
+
@system_config = build_system_config(@conf)
|
620
|
+
if @system_config.log
|
621
|
+
@log_rotate_age ||= @system_config.log.rotate_age
|
622
|
+
@log_rotate_size ||= @system_config.log.rotate_size
|
623
|
+
end
|
624
|
+
@conf = nil
|
625
|
+
end
|
626
|
+
|
627
|
+
log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
|
628
|
+
ignore_same_log_interval: opt[:ignore_same_log_interval]}
|
629
|
+
@log = LoggerInitializer.new(
|
630
|
+
@log_path, opt[:log_level], @chuser, @chgroup, log_opts,
|
631
|
+
log_rotate_age: @log_rotate_age,
|
632
|
+
log_rotate_size: @log_rotate_size
|
633
|
+
)
|
634
|
+
@finished = false
|
635
|
+
end
|
636
|
+
|
637
|
+
def run_supervisor(dry_run: false)
|
638
|
+
if dry_run
|
639
|
+
$log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
|
640
|
+
end
|
641
|
+
|
642
|
+
if @system_config.workers < 1
|
643
|
+
raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@system_config.workers}"
|
644
|
+
end
|
645
|
+
|
646
|
+
root_dir = @system_config.root_dir
|
647
|
+
if root_dir
|
648
|
+
if File.exist?(root_dir)
|
649
|
+
unless Dir.exist?(root_dir)
|
650
|
+
raise Fluent::InvalidRootDirectory, "non directory entry exists:#{root_dir}"
|
651
|
+
end
|
652
|
+
else
|
653
|
+
begin
|
654
|
+
FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
|
655
|
+
rescue => e
|
656
|
+
raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
|
657
|
+
end
|
658
|
+
end
|
659
|
+
end
|
660
|
+
|
661
|
+
begin
|
662
|
+
ServerEngine::Privilege.change(@chuser, @chgroup)
|
663
|
+
MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
|
664
|
+
Fluent::Engine.init(@system_config, supervisor_mode: true)
|
665
|
+
Fluent::Engine.run_configure(@conf, dry_run: dry_run)
|
666
|
+
rescue Fluent::ConfigError => e
|
667
|
+
$log.error 'config error', file: @config_path, error: e
|
668
|
+
$log.debug_backtrace
|
669
|
+
exit!(1)
|
670
|
+
end
|
671
|
+
|
672
|
+
if dry_run
|
673
|
+
$log.info 'finished dry run mode'
|
674
|
+
exit 0
|
675
|
+
else
|
676
|
+
supervise
|
677
|
+
end
|
678
|
+
end
|
679
|
+
|
680
|
+
def options
|
681
|
+
{
|
682
|
+
'config_path' => @config_path,
|
683
|
+
'pid_file' => @daemonize,
|
684
|
+
'plugin_dirs' => @plugin_dirs,
|
685
|
+
'log_path' => @log_path,
|
686
|
+
'root_dir' => @system_config.root_dir,
|
687
|
+
}
|
688
|
+
end
|
689
|
+
|
690
|
+
def run_worker
|
691
|
+
begin
|
692
|
+
require 'sigdump/setup'
|
693
|
+
rescue Exception
|
694
|
+
# ignore LoadError and others (related with signals): it may raise these errors in Windows
|
695
|
+
end
|
696
|
+
|
697
|
+
Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
|
698
|
+
|
699
|
+
if @standalone_worker && @system_config.workers != 1
|
700
|
+
raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}"
|
701
|
+
end
|
702
|
+
|
703
|
+
install_main_process_signal_handlers
|
704
|
+
|
705
|
+
# This is the only log messsage for @standalone_worker
|
706
|
+
$log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker
|
707
|
+
|
708
|
+
main_process do
|
709
|
+
create_socket_manager if @standalone_worker
|
710
|
+
if @standalone_worker
|
711
|
+
ServerEngine::Privilege.change(@chuser, @chgroup)
|
712
|
+
File.umask(0)
|
713
|
+
end
|
714
|
+
MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
|
715
|
+
Fluent::Engine.init(@system_config)
|
716
|
+
Fluent::Engine.run_configure(@conf)
|
717
|
+
Fluent::Engine.run
|
718
|
+
self.class.cleanup_resources if @standalone_worker
|
719
|
+
exit 0
|
720
|
+
end
|
721
|
+
end
|
722
|
+
|
723
|
+
def configure(supervisor: false)
|
724
|
+
if supervisor
|
725
|
+
@log.init(:supervisor, 0)
|
726
|
+
else
|
727
|
+
worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
|
728
|
+
process_type = case
|
729
|
+
when @standalone_worker then :standalone
|
730
|
+
when worker_id == 0 then :worker0
|
731
|
+
else :workers
|
732
|
+
end
|
733
|
+
@log.init(process_type, worker_id)
|
734
|
+
end
|
735
|
+
|
736
|
+
if @show_plugin_config
|
737
|
+
show_plugin_config
|
738
|
+
end
|
739
|
+
|
740
|
+
if @inline_config == '-'
|
741
|
+
$log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
|
742
|
+
@inline_config = STDIN.read
|
743
|
+
end
|
744
|
+
@conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
|
745
|
+
@system_config = build_system_config(@conf)
|
746
|
+
|
747
|
+
@log.level = @system_config.log_level
|
748
|
+
@log.apply_options(
|
749
|
+
format: @system_config.log.format,
|
750
|
+
time_format: @system_config.log.time_format,
|
751
|
+
log_dir_perm: @system_config.dir_permission,
|
752
|
+
ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
|
753
|
+
ignore_same_log_interval: @system_config.ignore_same_log_interval
|
754
|
+
)
|
755
|
+
|
756
|
+
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
|
757
|
+
|
758
|
+
@libs.each do |lib|
|
759
|
+
require lib
|
760
|
+
end
|
761
|
+
|
762
|
+
@plugin_dirs.each do |dir|
|
763
|
+
if Dir.exist?(dir)
|
764
|
+
dir = File.expand_path(dir)
|
765
|
+
Fluent::Plugin.add_plugin_dir(dir)
|
766
|
+
end
|
767
|
+
end
|
768
|
+
|
769
|
+
if supervisor
|
770
|
+
# plugins / configuration dumps
|
771
|
+
Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|
|
772
|
+
$log.info("gem '#{spec.name}' version '#{spec.version}'")
|
773
|
+
end
|
774
|
+
end
|
775
|
+
end
|
776
|
+
|
777
|
+
private
|
778
|
+
|
779
|
+
def create_socket_manager
|
780
|
+
socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
781
|
+
ServerEngine::SocketManager::Server.open(socket_manager_path)
|
782
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
783
|
+
end
|
784
|
+
|
785
|
+
def show_plugin_config
|
786
|
+
name, type = @show_plugin_config.split(":") # input:tail
|
787
|
+
$log.info "show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}"
|
788
|
+
exit 0
|
789
|
+
end
|
790
|
+
|
791
|
+
def supervise
|
792
|
+
Process.setproctitle("supervisor:#{@system_config.process_name}") if @system_config.process_name
|
793
|
+
$log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
|
794
|
+
|
795
|
+
fluentd_spawn_cmd = build_spawn_command
|
796
|
+
$log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
|
797
|
+
|
798
|
+
params = {
|
799
|
+
'main_cmd' => fluentd_spawn_cmd,
|
800
|
+
'daemonize' => @daemonize,
|
801
|
+
'inline_config' => @inline_config,
|
802
|
+
'log_path' => @log_path,
|
803
|
+
'log_rotate_age' => @log_rotate_age,
|
804
|
+
'log_rotate_size' => @log_rotate_size,
|
805
|
+
'chuser' => @chuser,
|
806
|
+
'chgroup' => @chgroup,
|
807
|
+
'use_v1_config' => @use_v1_config,
|
808
|
+
'conf_encoding' => @conf_encoding,
|
809
|
+
'signame' => @signame,
|
810
|
+
'fluentd_conf' => @conf.to_s,
|
811
|
+
|
812
|
+
'workers' => @system_config.workers,
|
813
|
+
'root_dir' => @system_config.root_dir,
|
814
|
+
'log_level' => @system_config.log_level,
|
815
|
+
'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
|
816
|
+
'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
|
817
|
+
'rpc_endpoint' => @system_config.rpc_endpoint,
|
818
|
+
'enable_get_dump' => @system_config.enable_get_dump,
|
819
|
+
'counter_server' => @system_config.counter_server,
|
820
|
+
'log_format' => @system_config.log.format,
|
821
|
+
'log_time_format' => @system_config.log.time_format,
|
822
|
+
'disable_shared_socket' => @system_config.disable_shared_socket
|
823
|
+
}
|
824
|
+
|
825
|
+
se = ServerEngine.create(ServerModule, WorkerModule){
|
826
|
+
Fluent::Supervisor.load_config(@config_path, params)
|
827
|
+
}
|
828
|
+
se.run
|
829
|
+
end
|
830
|
+
|
831
|
+
def install_main_process_signal_handlers
|
832
|
+
# Fluentd worker process (worker of ServerEngine) don't use code in serverengine to set signal handlers,
|
833
|
+
# because it does almost nothing.
|
834
|
+
# This method is the only method to set signal handlers in Fluentd worker process.
|
835
|
+
|
836
|
+
# When user use Ctrl + C not SIGINT, SIGINT is sent to all process in same process group.
|
837
|
+
# ServerEngine server process will send SIGTERM to child(spawned) processes by that SIGINT, so
|
838
|
+
# worker process SHOULD NOT do anything with SIGINT, SHOULD just ignore.
|
839
|
+
trap :INT do
|
840
|
+
$log.debug "fluentd main process get SIGINT"
|
841
|
+
|
842
|
+
# When Fluentd is launched without supervisor, worker should handle ctrl-c by itself
|
843
|
+
if @standalone_worker
|
844
|
+
@finished = true
|
845
|
+
$log.debug "getting start to shutdown main process"
|
846
|
+
Fluent::Engine.stop
|
847
|
+
end
|
848
|
+
end
|
849
|
+
|
850
|
+
trap :TERM do
|
851
|
+
$log.debug "fluentd main process get SIGTERM"
|
852
|
+
unless @finished
|
853
|
+
@finished = true
|
854
|
+
$log.debug "getting start to shutdown main process"
|
855
|
+
Fluent::Engine.stop
|
856
|
+
end
|
857
|
+
end
|
858
|
+
|
859
|
+
if Fluent.windows?
|
860
|
+
install_main_process_command_handlers
|
861
|
+
else
|
862
|
+
trap :USR1 do
|
863
|
+
flush_buffer
|
864
|
+
end
|
865
|
+
|
866
|
+
trap :USR2 do
|
867
|
+
reload_config
|
868
|
+
end
|
869
|
+
end
|
870
|
+
end
|
871
|
+
|
872
|
+
def install_main_process_command_handlers
|
873
|
+
command_pipe = $stdin.dup
|
874
|
+
$stdin.reopen(File::NULL, "rb")
|
875
|
+
command_pipe.binmode
|
876
|
+
command_pipe.sync = true
|
877
|
+
|
878
|
+
Thread.new do
|
879
|
+
loop do
|
880
|
+
cmd = command_pipe.gets
|
881
|
+
break unless cmd
|
882
|
+
|
883
|
+
case cmd.chomp!
|
884
|
+
when "GRACEFUL_STOP", "IMMEDIATE_STOP"
|
885
|
+
$log.debug "fluentd main process get #{cmd} command"
|
886
|
+
@finished = true
|
887
|
+
$log.debug "getting start to shutdown main process"
|
888
|
+
Fluent::Engine.stop
|
889
|
+
break
|
890
|
+
when "GRACEFUL_RESTART"
|
891
|
+
$log.debug "fluentd main process get #{cmd} command"
|
892
|
+
flush_buffer
|
893
|
+
when "RELOAD"
|
894
|
+
$log.debug "fluentd main process get #{cmd} command"
|
895
|
+
reload_config
|
896
|
+
else
|
897
|
+
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
898
|
+
end
|
899
|
+
end
|
900
|
+
end
|
901
|
+
end
|
902
|
+
|
903
|
+
def flush_buffer
|
904
|
+
# Creating new thread due to mutex can't lock
|
905
|
+
# in main thread during trap context
|
906
|
+
Thread.new do
|
907
|
+
begin
|
908
|
+
$log.debug "fluentd main process get SIGUSR1"
|
909
|
+
$log.info "force flushing buffered events"
|
910
|
+
@log.reopen!
|
911
|
+
Fluent::Engine.flush!
|
912
|
+
$log.debug "flushing thread: flushed"
|
913
|
+
rescue Exception => e
|
914
|
+
$log.warn "flushing thread error: #{e}"
|
915
|
+
end
|
916
|
+
end
|
917
|
+
end
|
918
|
+
|
919
|
+
def reload_config
|
920
|
+
Thread.new do
|
921
|
+
$log.debug('worker got SIGUSR2')
|
922
|
+
|
923
|
+
begin
|
924
|
+
conf = Fluent::Config.build(
|
925
|
+
config_path: @config_path,
|
926
|
+
encoding: @conf_encoding,
|
927
|
+
additional_config: @inline_config,
|
928
|
+
use_v1_config: @use_v1_config,
|
929
|
+
)
|
930
|
+
|
931
|
+
Fluent::VariableStore.try_to_reset do
|
932
|
+
Fluent::Engine.reload_config(conf)
|
933
|
+
end
|
934
|
+
rescue => e
|
935
|
+
# it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
|
936
|
+
# So need this rescue code
|
937
|
+
$log.error("failed to reload config: #{e}")
|
938
|
+
next
|
939
|
+
end
|
940
|
+
|
941
|
+
@conf = conf
|
942
|
+
end
|
943
|
+
end
|
944
|
+
|
945
|
+
def logging_with_console_output
|
946
|
+
yield $log
|
947
|
+
unless @log.stdout?
|
948
|
+
logger = ServerEngine::DaemonLogger.new(STDOUT)
|
949
|
+
log = Fluent::Log.new(logger)
|
950
|
+
log.level = @system_config.log_level
|
951
|
+
console = log.enable_debug
|
952
|
+
yield console
|
953
|
+
end
|
954
|
+
end
|
955
|
+
|
956
|
+
def main_process(&block)
|
957
|
+
if @system_config.process_name
|
958
|
+
if @system_config.workers > 1
|
959
|
+
Process.setproctitle("worker:#{@system_config.process_name}#{ENV['SERVERENGINE_WORKER_ID']}")
|
960
|
+
else
|
961
|
+
Process.setproctitle("worker:#{@system_config.process_name}")
|
962
|
+
end
|
963
|
+
end
|
964
|
+
|
965
|
+
unrecoverable_error = false
|
966
|
+
|
967
|
+
begin
|
968
|
+
block.call
|
969
|
+
rescue Fluent::ConfigError => e
|
970
|
+
logging_with_console_output do |log|
|
971
|
+
log.error "config error", file: @config_path, error: e
|
972
|
+
log.debug_backtrace
|
973
|
+
end
|
974
|
+
unrecoverable_error = true
|
975
|
+
rescue Fluent::UnrecoverableError => e
|
976
|
+
logging_with_console_output do |log|
|
977
|
+
log.error e.message, error: e
|
978
|
+
log.error_backtrace
|
979
|
+
end
|
980
|
+
unrecoverable_error = true
|
981
|
+
rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError
|
982
|
+
logging_with_console_output do |log|
|
983
|
+
if e.respond_to?(:path)
|
984
|
+
log.error e.message, path: e.path, error: e
|
985
|
+
else
|
986
|
+
log.error e.message, error: e
|
987
|
+
end
|
988
|
+
log.error_backtrace
|
989
|
+
end
|
990
|
+
unrecoverable_error = true
|
991
|
+
rescue => e
|
992
|
+
logging_with_console_output do |log|
|
993
|
+
log.error "unexpected error", error: e
|
994
|
+
log.error_backtrace
|
995
|
+
end
|
996
|
+
end
|
997
|
+
|
998
|
+
exit!(unrecoverable_error ? 2 : 1)
|
999
|
+
end
|
1000
|
+
|
1001
|
+
def build_system_config(conf)
|
1002
|
+
system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
|
1003
|
+
opt = {}
|
1004
|
+
Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
|
1005
|
+
if @cl_opt.key?(param) && !@cl_opt[param].nil?
|
1006
|
+
if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
|
1007
|
+
# info level can't be specified via command line option.
|
1008
|
+
# log_level is info here, it is default value and <system>'s log_level should be applied if exists.
|
1009
|
+
next
|
1010
|
+
end
|
1011
|
+
|
1012
|
+
opt[param] = @cl_opt[param]
|
1013
|
+
end
|
1014
|
+
end
|
1015
|
+
system_config.overwrite_variables(**opt)
|
1016
|
+
system_config
|
1017
|
+
end
|
1018
|
+
|
1019
|
+
RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
|
1020
|
+
|
1021
|
+
def build_spawn_command
|
1022
|
+
if ENV['TEST_RUBY_PATH']
|
1023
|
+
fluentd_spawn_cmd = [ENV['TEST_RUBY_PATH']]
|
1024
|
+
else
|
1025
|
+
fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
|
1026
|
+
end
|
1027
|
+
|
1028
|
+
rubyopt = ENV['RUBYOPT']
|
1029
|
+
if rubyopt
|
1030
|
+
encodes, others = rubyopt.split(' ').partition { |e| e.match?(RUBY_ENCODING_OPTIONS_REGEX) }
|
1031
|
+
fluentd_spawn_cmd.concat(others)
|
1032
|
+
|
1033
|
+
adopted_encodes = encodes.empty? ? ['-Eascii-8bit:ascii-8bit'] : encodes
|
1034
|
+
fluentd_spawn_cmd.concat(adopted_encodes)
|
1035
|
+
else
|
1036
|
+
fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
|
1037
|
+
end
|
1038
|
+
|
1039
|
+
# Adding `-h` so that it can avoid ruby's command blocking
|
1040
|
+
# e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
|
1041
|
+
_, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
|
1042
|
+
if s.exitstatus != 0
|
1043
|
+
$log.error('Invalid option is passed to RUBYOPT', command: fluentd_spawn_cmd, error: e)
|
1044
|
+
exit s.exitstatus
|
1045
|
+
end
|
1046
|
+
|
1047
|
+
fluentd_spawn_cmd << $0
|
1048
|
+
fluentd_spawn_cmd += $fluentdargv
|
1049
|
+
fluentd_spawn_cmd << '--under-supervisor'
|
1050
|
+
|
1051
|
+
fluentd_spawn_cmd
|
1052
|
+
end
|
1053
|
+
end
|
1054
|
+
end
|