fluentd 0.12.40 → 1.6.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
- data/.gitignore +5 -0
- data/.gitlab/cicd-template.yaml +10 -0
- data/.gitlab-ci.yml +147 -0
- data/.travis.yml +56 -20
- data/ADOPTERS.md +5 -0
- data/CHANGELOG.md +1369 -0
- data/CONTRIBUTING.md +16 -5
- data/GOVERNANCE.md +55 -0
- data/Gemfile +5 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +7 -0
- data/README.md +23 -11
- data/Rakefile +48 -2
- data/Vagrantfile +17 -0
- data/appveyor.yml +37 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +3 -0
- data/code-of-conduct.md +3 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_dummy_with_compression.conf +23 -0
- data/example/in_forward.conf +7 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +3 -1
- data/example/in_out_forward.conf +17 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_forward.conf +13 -13
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +29 -0
- data/fluentd.gemspec +21 -11
- data/lib/fluent/agent.rb +67 -90
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/ca_generate.rb +181 -0
- data/lib/fluent/command/cat.rb +42 -18
- data/lib/fluent/command/debug.rb +12 -10
- data/lib/fluent/command/fluentd.rb +153 -5
- data/lib/fluent/command/plugin_config_formatter.rb +292 -0
- data/lib/fluent/command/plugin_generator.rb +324 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +718 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/configure_proxy.rb +210 -62
- data/lib/fluent/config/dsl.rb +12 -5
- data/lib/fluent/config/element.rb +107 -9
- data/lib/fluent/config/literal_parser.rb +9 -3
- data/lib/fluent/config/parser.rb +4 -4
- data/lib/fluent/config/section.rb +51 -14
- data/lib/fluent/config/types.rb +28 -13
- data/lib/fluent/config/v1_parser.rb +3 -5
- data/lib/fluent/config.rb +23 -20
- data/lib/fluent/configurable.rb +79 -21
- data/lib/fluent/counter/base_socket.rb +46 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +102 -65
- data/lib/fluent/env.rb +7 -3
- data/lib/fluent/error.rb +30 -0
- data/lib/fluent/event.rb +197 -21
- data/lib/fluent/event_router.rb +93 -10
- data/lib/fluent/filter.rb +2 -50
- data/lib/fluent/formatter.rb +4 -293
- data/lib/fluent/input.rb +2 -32
- data/lib/fluent/label.rb +10 -2
- data/lib/fluent/load.rb +3 -3
- data/lib/fluent/log.rb +348 -81
- data/lib/fluent/match.rb +37 -36
- data/lib/fluent/mixin.rb +12 -176
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +10 -612
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +4 -800
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +192 -0
- data/lib/fluent/plugin/buf_file.rb +128 -174
- data/lib/fluent/plugin/buf_memory.rb +9 -92
- data/lib/fluent/plugin/buffer/chunk.rb +221 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/buffer.rb +779 -0
- data/lib/fluent/plugin/compressable.rb +92 -0
- data/lib/fluent/plugin/exec_util.rb +3 -108
- data/lib/fluent/plugin/file_util.rb +4 -34
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +117 -34
- data/lib/fluent/plugin/filter_parser.rb +85 -62
- data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
- data/lib/fluent/plugin/filter_stdout.rb +15 -12
- data/lib/fluent/plugin/formatter.rb +50 -0
- data/lib/fluent/plugin/formatter_csv.rb +52 -0
- data/lib/fluent/plugin/formatter_hash.rb +33 -0
- data/lib/fluent/plugin/formatter_json.rb +55 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +51 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +38 -0
- data/lib/fluent/plugin/in_debug_agent.rb +17 -6
- data/lib/fluent/plugin/in_dummy.rb +47 -20
- data/lib/fluent/plugin/in_exec.rb +55 -123
- data/lib/fluent/plugin/in_forward.rb +299 -216
- data/lib/fluent/plugin/in_gc_stat.rb +14 -36
- data/lib/fluent/plugin/in_http.rb +204 -91
- data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
- data/lib/fluent/plugin/in_object_space.rb +13 -41
- data/lib/fluent/plugin/in_syslog.rb +112 -134
- data/lib/fluent/plugin/in_tail.rb +408 -745
- data/lib/fluent/plugin/in_tcp.rb +66 -9
- data/lib/fluent/plugin/in_udp.rb +60 -11
- data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
- data/lib/fluent/plugin/input.rb +37 -0
- data/lib/fluent/plugin/multi_output.rb +158 -0
- data/lib/fluent/plugin/out_copy.rb +23 -35
- data/lib/fluent/plugin/out_exec.rb +67 -70
- data/lib/fluent/plugin/out_exec_filter.rb +204 -271
- data/lib/fluent/plugin/out_file.rb +267 -73
- data/lib/fluent/plugin/out_forward.rb +854 -325
- data/lib/fluent/plugin/out_null.rb +42 -9
- data/lib/fluent/plugin/out_relabel.rb +9 -5
- data/lib/fluent/plugin/out_roundrobin.rb +18 -37
- data/lib/fluent/plugin/out_secondary_file.rb +133 -0
- data/lib/fluent/plugin/out_stdout.rb +43 -10
- data/lib/fluent/plugin/out_stream.rb +7 -2
- data/lib/fluent/plugin/output.rb +1498 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +191 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +39 -0
- data/lib/fluent/plugin/parser_json.rb +94 -0
- data/lib/fluent/plugin/parser_ltsv.rb +49 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +106 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +142 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/socket_util.rb +3 -143
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +164 -0
- data/lib/fluent/plugin/string_util.rb +3 -15
- data/lib/fluent/plugin.rb +122 -121
- data/lib/fluent/plugin_helper/cert_option.rb +178 -0
- data/lib/fluent/plugin_helper/child_process.rb +364 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
- data/lib/fluent/plugin_helper/http_server.rb +76 -0
- data/lib/fluent/plugin_helper/inject.rb +151 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
- data/lib/fluent/plugin_helper/retry_state.rb +205 -0
- data/lib/fluent/plugin_helper/server.rb +807 -0
- data/lib/fluent/plugin_helper/socket.rb +250 -0
- data/lib/fluent/plugin_helper/socket_option.rb +80 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +179 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +73 -0
- data/lib/fluent/plugin_id.rb +80 -0
- data/lib/fluent/process.rb +3 -489
- data/lib/fluent/registry.rb +52 -10
- data/lib/fluent/root_agent.rb +204 -42
- data/lib/fluent/supervisor.rb +597 -359
- data/lib/fluent/system_config.rb +131 -42
- data/lib/fluent/test/base.rb +6 -54
- data/lib/fluent/test/driver/base.rb +224 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +0 -1
- data/lib/fluent/test/formatter_test.rb +4 -1
- data/lib/fluent/test/helpers.rb +58 -10
- data/lib/fluent/test/input_test.rb +27 -19
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +28 -39
- data/lib/fluent/test/parser_test.rb +3 -1
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +33 -1
- data/lib/fluent/time.rb +450 -1
- data/lib/fluent/timezone.rb +27 -3
- data/lib/fluent/{status.rb → unique_id.rb} +15 -24
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +85 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +346 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_fluentd.rb +901 -0
- data/test/command/test_plugin_config_formatter.rb +276 -0
- data/test/command/test_plugin_generator.rb +92 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/test_config_parser.rb +126 -2
- data/test/config/test_configurable.rb +946 -187
- data/test/config/test_configure_proxy.rb +424 -74
- data/test/config/test_dsl.rb +11 -11
- data/test/config/test_element.rb +500 -0
- data/test/config/test_literal_parser.rb +8 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +79 -7
- data/test/config/test_system_config.rb +122 -35
- data/test/config/test_types.rb +38 -0
- data/test/counter/test_client.rb +559 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +89 -6
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +823 -460
- data/test/plugin/test_buf_memory.rb +32 -194
- data/test/plugin/test_buffer.rb +1233 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +844 -0
- data/test/plugin/test_buffer_memory_chunk.rb +338 -0
- data/test/plugin/test_compressable.rb +84 -0
- data/test/plugin/test_filter.rb +357 -0
- data/test/plugin/test_filter_grep.rb +540 -29
- data/test/plugin/test_filter_parser.rb +439 -452
- data/test/plugin/test_filter_record_transformer.rb +123 -166
- data/test/plugin/test_filter_stdout.rb +160 -72
- data/test/plugin/test_formatter_csv.rb +111 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +51 -0
- data/test/plugin/test_formatter_ltsv.rb +62 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_formatter_tsv.rb +68 -0
- data/test/plugin/test_in_debug_agent.rb +24 -1
- data/test/plugin/test_in_dummy.rb +111 -18
- data/test/plugin/test_in_exec.rb +200 -113
- data/test/plugin/test_in_forward.rb +990 -387
- data/test/plugin/test_in_gc_stat.rb +10 -8
- data/test/plugin/test_in_http.rb +600 -224
- data/test/plugin/test_in_monitor_agent.rb +690 -0
- data/test/plugin/test_in_object_space.rb +24 -8
- data/test/plugin/test_in_syslog.rb +154 -215
- data/test/plugin/test_in_tail.rb +1006 -707
- data/test/plugin/test_in_tcp.rb +125 -48
- data/test/plugin/test_in_udp.rb +204 -63
- data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
- data/test/plugin/test_input.rb +126 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_copy.rb +117 -112
- data/test/plugin/test_out_exec.rb +258 -53
- data/test/plugin/test_out_exec_filter.rb +538 -115
- data/test/plugin/test_out_file.rb +865 -178
- data/test/plugin/test_out_forward.rb +998 -210
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +36 -29
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +135 -37
- data/test/plugin/test_out_stream.rb +18 -0
- data/test/plugin/test_output.rb +984 -0
- data/test/plugin/test_output_as_buffered.rb +2021 -0
- data/test/plugin/test_output_as_buffered_backup.rb +312 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +911 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +103 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +145 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +441 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/test_cert_option.rb +16 -0
- data/test/plugin_helper/test_child_process.rb +794 -0
- data/test/plugin_helper/test_compat_parameters.rb +353 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +205 -0
- data/test/plugin_helper/test_inject.rb +519 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +197 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1714 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +132 -0
- data/test/scripts/exec_script.rb +0 -6
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/out_test.rb +23 -15
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +16 -7
- data/test/test_configdsl.rb +2 -2
- data/test/test_event.rb +360 -13
- data/test/test_event_router.rb +108 -11
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +48 -6
- data/test/test_formatter.rb +11 -391
- data/test/test_input.rb +1 -1
- data/test/test_log.rb +591 -31
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +121 -185
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +177 -10
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +101 -0
- data/test/test_process.rb +8 -42
- data/test/test_root_agent.rb +766 -21
- data/test/test_supervisor.rb +481 -0
- data/test/test_test_drivers.rb +135 -0
- data/test/test_time_formatter.rb +282 -0
- data/test/test_time_parser.rb +231 -0
- data/test/test_unique_id.rb +47 -0
- metadata +454 -60
- data/COPYING +0 -14
- data/ChangeLog +0 -666
- data/lib/fluent/buffer.rb +0 -365
- data/lib/fluent/plugin/in_status.rb +0 -76
- data/test/plugin/test_in_status.rb +0 -38
- data/test/test_buffer.rb +0 -624
- data/test/test_parser.rb +0 -1305
@@ -14,32 +14,41 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'fcntl'
|
18
17
|
|
19
|
-
require '
|
18
|
+
require 'fluent/plugin/input'
|
19
|
+
require 'fluent/msgpack_factory'
|
20
20
|
require 'yajl'
|
21
|
+
require 'digest'
|
22
|
+
require 'securerandom'
|
21
23
|
|
22
|
-
|
23
|
-
|
24
|
-
module Fluent
|
24
|
+
module Fluent::Plugin
|
25
25
|
class ForwardInput < Input
|
26
|
-
Plugin.register_input('forward', self)
|
26
|
+
Fluent::Plugin.register_input('forward', self)
|
27
27
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
# See the wiki page below for protocol specification
|
29
|
+
# https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
|
30
|
+
|
31
|
+
helpers :server
|
32
|
+
|
33
|
+
LISTEN_PORT = 24224
|
32
34
|
|
33
35
|
desc 'The port to listen to.'
|
34
|
-
config_param :port, :integer, default:
|
36
|
+
config_param :port, :integer, default: LISTEN_PORT
|
35
37
|
desc 'The bind address to listen to.'
|
36
38
|
config_param :bind, :string, default: '0.0.0.0'
|
39
|
+
|
37
40
|
config_param :backlog, :integer, default: nil
|
38
41
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
|
39
42
|
desc 'The timeout time used to set linger option.'
|
40
43
|
config_param :linger_timeout, :integer, default: 0
|
41
44
|
# This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
|
42
45
|
config_param :blocking_timeout, :time, default: 0.5
|
46
|
+
desc 'Try to resolve hostname from IP addresses or not.'
|
47
|
+
config_param :resolve_hostname, :bool, default: nil
|
48
|
+
desc 'Connections will be disconnected right after receiving first message if this value is true.'
|
49
|
+
config_param :deny_keepalive, :bool, default: false
|
50
|
+
desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
|
51
|
+
config_param :send_keepalive_packet, :bool, default: false
|
43
52
|
|
44
53
|
desc 'Log warning if received chunk size is larger than this value.'
|
45
54
|
config_param :chunk_size_warn_limit, :size, default: nil
|
@@ -47,108 +56,230 @@ module Fluent
|
|
47
56
|
config_param :chunk_size_limit, :size, default: nil
|
48
57
|
desc 'Skip an event if incoming event is invalid.'
|
49
58
|
config_param :skip_invalid_event, :bool, default: false
|
50
|
-
|
51
|
-
config_param :resolve_hostname, :bool, default: nil
|
59
|
+
|
52
60
|
desc "The field name of the client's source address."
|
53
61
|
config_param :source_address_key, :string, default: nil
|
54
62
|
desc "The field name of the client's hostname."
|
55
63
|
config_param :source_hostname_key, :string, default: nil
|
56
64
|
|
65
|
+
desc "New tag instead of incoming tag"
|
66
|
+
config_param :tag, :string, default: nil
|
67
|
+
desc "Add prefix to incoming tag"
|
68
|
+
config_param :add_tag_prefix, :string, default: nil
|
69
|
+
|
70
|
+
config_section :security, required: false, multi: false do
|
71
|
+
desc 'The hostname'
|
72
|
+
config_param :self_hostname, :string
|
73
|
+
desc 'Shared key for authentication'
|
74
|
+
config_param :shared_key, :string, secret: true
|
75
|
+
desc 'If true, use user based authentication'
|
76
|
+
config_param :user_auth, :bool, default: false
|
77
|
+
desc 'Allow anonymous source. <client> sections required if disabled.'
|
78
|
+
config_param :allow_anonymous_source, :bool, default: true
|
79
|
+
|
80
|
+
### User based authentication
|
81
|
+
config_section :user, param_name: :users, required: false, multi: true do
|
82
|
+
desc 'The username for authentication'
|
83
|
+
config_param :username, :string
|
84
|
+
desc 'The password for authentication'
|
85
|
+
config_param :password, :string, secret: true
|
86
|
+
end
|
87
|
+
|
88
|
+
### Client ip/network authentication & per_host shared key
|
89
|
+
config_section :client, param_name: :clients, required: false, multi: true do
|
90
|
+
desc 'The IP address or host name of the client'
|
91
|
+
config_param :host, :string, default: nil
|
92
|
+
desc 'Network address specification'
|
93
|
+
config_param :network, :string, default: nil
|
94
|
+
desc 'Shared key per client'
|
95
|
+
config_param :shared_key, :string, default: nil, secret: true
|
96
|
+
desc 'Array of username.'
|
97
|
+
config_param :users, :array, default: []
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
57
101
|
def configure(conf)
|
58
102
|
super
|
59
103
|
|
60
104
|
if @source_hostname_key
|
105
|
+
# TODO: add test
|
61
106
|
if @resolve_hostname.nil?
|
62
107
|
@resolve_hostname = true
|
63
|
-
elsif !@resolve_hostname # user specifies "false" in
|
108
|
+
elsif !@resolve_hostname # user specifies "false" in config
|
64
109
|
raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
|
65
110
|
end
|
66
111
|
end
|
67
112
|
@enable_field_injection = @source_address_key || @source_hostname_key
|
113
|
+
|
114
|
+
raise Fluent::ConfigError, "'tag' parameter must not be empty" if @tag && @tag.empty?
|
115
|
+
raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
|
116
|
+
|
117
|
+
if @security
|
118
|
+
if @security.user_auth && @security.users.empty?
|
119
|
+
raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
|
120
|
+
end
|
121
|
+
if !@security.allow_anonymous_source && @security.clients.empty?
|
122
|
+
raise Fluent::ConfigError, "<client> sections required if allow_anonymous_source disabled"
|
123
|
+
end
|
124
|
+
|
125
|
+
@nodes = []
|
126
|
+
|
127
|
+
@security.clients.each do |client|
|
128
|
+
if client.host && client.network
|
129
|
+
raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
|
130
|
+
end
|
131
|
+
if !client.host && !client.network
|
132
|
+
raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
|
133
|
+
end
|
134
|
+
source = nil
|
135
|
+
if client.host
|
136
|
+
begin
|
137
|
+
source = IPSocket.getaddress(client.host)
|
138
|
+
rescue SocketError
|
139
|
+
raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
source_addr = begin
|
143
|
+
IPAddr.new(source || client.network)
|
144
|
+
rescue ArgumentError
|
145
|
+
raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
|
146
|
+
end
|
147
|
+
@nodes.push({
|
148
|
+
address: source_addr,
|
149
|
+
shared_key: (client.shared_key || @security.shared_key),
|
150
|
+
users: client.users
|
151
|
+
})
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
if @send_keepalive_packet && @deny_keepalive
|
156
|
+
raise Fluent::ConfigError, "both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true"
|
157
|
+
end
|
68
158
|
end
|
69
159
|
|
70
|
-
def
|
71
|
-
|
160
|
+
def multi_workers_ready?
|
161
|
+
true
|
162
|
+
end
|
72
163
|
|
73
|
-
|
74
|
-
@loop.attach(@lsock)
|
164
|
+
HEARTBEAT_UDP_PAYLOAD = "\0"
|
75
165
|
|
76
|
-
|
77
|
-
|
78
|
-
@usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
|
79
|
-
@hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
|
80
|
-
@loop.attach(@hbr)
|
166
|
+
def start
|
167
|
+
super
|
81
168
|
|
82
|
-
|
169
|
+
shared_socket = system_config.workers > 1
|
170
|
+
|
171
|
+
log.info "listening port", port: @port, bind: @bind
|
172
|
+
server_create_connection(
|
173
|
+
:in_forward_server, @port,
|
174
|
+
bind: @bind,
|
175
|
+
shared: shared_socket,
|
176
|
+
resolve_name: @resolve_hostname,
|
177
|
+
linger_timeout: @linger_timeout,
|
178
|
+
send_keepalive_packet: @send_keepalive_packet,
|
179
|
+
backlog: @backlog,
|
180
|
+
&method(:handle_connection)
|
181
|
+
)
|
182
|
+
|
183
|
+
server_create(:in_forward_server_udp_heartbeat, @port, shared: shared_socket, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
|
184
|
+
log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
|
185
|
+
begin
|
186
|
+
sock.write HEARTBEAT_UDP_PAYLOAD
|
187
|
+
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
188
|
+
log.trace "error while heartbeat response", host: sock.remote_host, error: e
|
189
|
+
end
|
190
|
+
end
|
83
191
|
end
|
84
192
|
|
85
|
-
def
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
193
|
+
def handle_connection(conn)
|
194
|
+
send_data = ->(serializer, data){ conn.write serializer.call(data) }
|
195
|
+
|
196
|
+
log.trace "connected fluent socket", addr: conn.remote_addr, port: conn.remote_port
|
197
|
+
state = :established
|
198
|
+
nonce = nil
|
199
|
+
user_auth_salt = nil
|
200
|
+
|
201
|
+
if @security
|
202
|
+
# security enabled session MUST use MessagePack as serialization format
|
203
|
+
state = :helo
|
204
|
+
nonce = generate_salt
|
205
|
+
user_auth_salt = generate_salt
|
206
|
+
send_data.call(:to_msgpack.to_proc, generate_helo(nonce, user_auth_salt))
|
207
|
+
state = :pingpong
|
208
|
+
end
|
209
|
+
|
210
|
+
log.trace "accepted fluent socket", addr: conn.remote_addr, port: conn.remote_port
|
211
|
+
|
212
|
+
read_messages(conn) do |msg, chunk_size, serializer|
|
213
|
+
case state
|
214
|
+
when :pingpong
|
215
|
+
success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
|
216
|
+
unless success
|
217
|
+
conn.on(:write_complete) { |c| c.close_after_write_complete }
|
218
|
+
send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
|
219
|
+
next
|
220
|
+
end
|
221
|
+
send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))
|
222
|
+
|
223
|
+
log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
|
224
|
+
state = :established
|
225
|
+
when :established
|
226
|
+
options = on_message(msg, chunk_size, conn)
|
227
|
+
if options && r = response(options)
|
228
|
+
log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
|
229
|
+
conn.on(:write_complete) { |c| c.close } if @deny_keepalive
|
230
|
+
send_data.call(serializer, r)
|
231
|
+
else
|
232
|
+
if @deny_keepalive
|
233
|
+
conn.close
|
234
|
+
end
|
235
|
+
end
|
236
|
+
else
|
237
|
+
raise "BUG: unknown session state: #{state}"
|
238
|
+
end
|
239
|
+
end
|
100
240
|
end
|
101
241
|
|
102
|
-
def
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
242
|
+
def read_messages(conn, &block)
|
243
|
+
feeder = nil
|
244
|
+
serializer = nil
|
245
|
+
bytes = 0
|
246
|
+
conn.data do |data|
|
247
|
+
# only for first call of callback
|
248
|
+
unless feeder
|
249
|
+
first = data[0]
|
250
|
+
if first == '{' || first == '[' # json
|
251
|
+
parser = Yajl::Parser.new
|
252
|
+
parser.on_parse_complete = ->(obj){
|
253
|
+
block.call(obj, bytes, serializer)
|
254
|
+
bytes = 0
|
255
|
+
}
|
256
|
+
serializer = :to_json.to_proc
|
257
|
+
feeder = ->(d){ parser << d }
|
258
|
+
else # msgpack
|
259
|
+
parser = Fluent::Engine.msgpack_factory.unpacker
|
260
|
+
serializer = :to_msgpack.to_proc
|
261
|
+
feeder = ->(d){
|
262
|
+
parser.feed_each(d){|obj|
|
263
|
+
block.call(obj, bytes, serializer)
|
264
|
+
bytes = 0
|
265
|
+
}
|
266
|
+
}
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
bytes += data.bytesize
|
271
|
+
feeder.call(data)
|
272
|
+
end
|
107
273
|
end
|
108
274
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
# FileUtils.mkdir_p File.dirname(@path)
|
115
|
-
# log.debug "listening fluent socket on #{@path}"
|
116
|
-
# Coolio::UNIXServer.new(@path, Handler, method(:on_message))
|
117
|
-
#end
|
118
|
-
|
119
|
-
def run
|
120
|
-
@loop.run(@blocking_timeout)
|
121
|
-
rescue => e
|
122
|
-
log.error "unexpected error", error: e, error_class: e.class
|
123
|
-
log.error_backtrace
|
275
|
+
def response(option)
|
276
|
+
if option && option['chunk']
|
277
|
+
return { 'ack' => option['chunk'] }
|
278
|
+
end
|
279
|
+
nil
|
124
280
|
end
|
125
281
|
|
126
|
-
|
127
|
-
|
128
|
-
# message Entry {
|
129
|
-
# 1: long time
|
130
|
-
# 2: object record
|
131
|
-
# }
|
132
|
-
#
|
133
|
-
# message Forward {
|
134
|
-
# 1: string tag
|
135
|
-
# 2: list<Entry> entries
|
136
|
-
# 3: object option (optional)
|
137
|
-
# }
|
138
|
-
#
|
139
|
-
# message PackedForward {
|
140
|
-
# 1: string tag
|
141
|
-
# 2: raw entries # msgpack stream of Entry
|
142
|
-
# 3: object option (optional)
|
143
|
-
# }
|
144
|
-
#
|
145
|
-
# message Message {
|
146
|
-
# 1: string tag
|
147
|
-
# 2: long? time
|
148
|
-
# 3: object record
|
149
|
-
# 4: object option (optional)
|
150
|
-
# }
|
151
|
-
def on_message(msg, chunk_size, peeraddr)
|
282
|
+
def on_message(msg, chunk_size, conn)
|
152
283
|
if msg.nil?
|
153
284
|
# for future TCP heartbeat_request
|
154
285
|
return
|
@@ -156,7 +287,7 @@ module Fluent
|
|
156
287
|
|
157
288
|
# TODO: raise an exception if broken chunk is generated by recoverable situation
|
158
289
|
unless msg.is_a?(Array)
|
159
|
-
log.warn "incoming chunk is broken:",
|
290
|
+
log.warn "incoming chunk is broken:", host: conn.remote_host, msg: msg
|
160
291
|
return
|
161
292
|
end
|
162
293
|
|
@@ -164,36 +295,46 @@ module Fluent
|
|
164
295
|
entries = msg[1]
|
165
296
|
|
166
297
|
if @chunk_size_limit && (chunk_size > @chunk_size_limit)
|
167
|
-
log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag,
|
298
|
+
log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size
|
168
299
|
return
|
169
300
|
elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
|
170
|
-
log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag,
|
301
|
+
log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
|
171
302
|
end
|
172
303
|
|
173
|
-
|
304
|
+
tag = @tag.dup if @tag
|
305
|
+
tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
|
306
|
+
|
307
|
+
case entries
|
308
|
+
when String
|
174
309
|
# PackedForward
|
175
|
-
es = MessagePackEventStream.new(entries)
|
176
|
-
es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
|
177
|
-
es = add_source_host(es, peeraddr) if @enable_field_injection
|
178
|
-
router.emit_stream(tag, es)
|
179
310
|
option = msg[2]
|
311
|
+
size = (option && option['size']) || 0
|
312
|
+
es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
|
313
|
+
es = es_class.new(entries, nil, size.to_i)
|
314
|
+
es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
|
315
|
+
if @enable_field_injection
|
316
|
+
es = add_source_info(es, conn)
|
317
|
+
end
|
318
|
+
router.emit_stream(tag, es)
|
180
319
|
|
181
|
-
|
320
|
+
when Array
|
182
321
|
# Forward
|
183
322
|
es = if @skip_invalid_event
|
184
|
-
check_and_skip_invalid_event(tag, entries,
|
323
|
+
check_and_skip_invalid_event(tag, entries, conn.remote_host)
|
185
324
|
else
|
186
|
-
es = MultiEventStream.new
|
325
|
+
es = Fluent::MultiEventStream.new
|
187
326
|
entries.each { |e|
|
188
327
|
record = e[1]
|
189
328
|
next if record.nil?
|
190
329
|
time = e[0]
|
191
|
-
time =
|
330
|
+
time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
|
192
331
|
es.add(time, record)
|
193
332
|
}
|
194
333
|
es
|
195
334
|
end
|
196
|
-
|
335
|
+
if @enable_field_injection
|
336
|
+
es = add_source_info(es, conn)
|
337
|
+
end
|
197
338
|
router.emit_stream(tag, es)
|
198
339
|
option = msg[2]
|
199
340
|
|
@@ -202,14 +343,14 @@ module Fluent
|
|
202
343
|
time = msg[1]
|
203
344
|
record = msg[2]
|
204
345
|
if @skip_invalid_event && invalid_event?(tag, time, record)
|
205
|
-
log.warn "got invalid event and drop it:",
|
346
|
+
log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
|
206
347
|
return msg[3] # retry never succeeded so return ack and drop incoming event.
|
207
348
|
end
|
208
349
|
return if record.nil?
|
209
|
-
time = Engine.now if time == 0
|
350
|
+
time = Fluent::Engine.now if time.to_i == 0
|
210
351
|
if @enable_field_injection
|
211
|
-
record[@
|
212
|
-
record[@
|
352
|
+
record[@source_address_key] = conn.remote_addr if @source_address_key
|
353
|
+
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
|
213
354
|
end
|
214
355
|
router.emit(tag, time, record)
|
215
356
|
option = msg[3]
|
@@ -220,14 +361,14 @@ module Fluent
|
|
220
361
|
end
|
221
362
|
|
222
363
|
def invalid_event?(tag, time, record)
|
223
|
-
!(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
|
364
|
+
!((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
|
224
365
|
end
|
225
366
|
|
226
|
-
def check_and_skip_invalid_event(tag, es,
|
227
|
-
new_es = MultiEventStream.new
|
367
|
+
def check_and_skip_invalid_event(tag, es, remote_host)
|
368
|
+
new_es = Fluent::MultiEventStream.new
|
228
369
|
es.each { |time, record|
|
229
370
|
if invalid_event?(tag, time, record)
|
230
|
-
log.warn "skip invalid event:",
|
371
|
+
log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
|
231
372
|
next
|
232
373
|
end
|
233
374
|
new_es.add(time, record)
|
@@ -235,24 +376,24 @@ module Fluent
|
|
235
376
|
new_es
|
236
377
|
end
|
237
378
|
|
238
|
-
def
|
239
|
-
new_es = MultiEventStream.new
|
379
|
+
def add_source_info(es, conn)
|
380
|
+
new_es = Fluent::MultiEventStream.new
|
240
381
|
if @source_address_key && @source_hostname_key
|
241
|
-
address =
|
242
|
-
hostname =
|
382
|
+
address = conn.remote_addr
|
383
|
+
hostname = conn.remote_host
|
243
384
|
es.each { |time, record|
|
244
385
|
record[@source_address_key] = address
|
245
386
|
record[@source_hostname_key] = hostname
|
246
387
|
new_es.add(time, record)
|
247
388
|
}
|
248
389
|
elsif @source_address_key
|
249
|
-
address =
|
390
|
+
address = conn.remote_addr
|
250
391
|
es.each { |time, record|
|
251
392
|
record[@source_address_key] = address
|
252
393
|
new_es.add(time, record)
|
253
394
|
}
|
254
395
|
elsif @source_hostname_key
|
255
|
-
hostname =
|
396
|
+
hostname = conn.remote_host
|
256
397
|
es.each { |time, record|
|
257
398
|
record[@source_hostname_key] = hostname
|
258
399
|
new_es.add(time, record)
|
@@ -263,128 +404,70 @@ module Fluent
|
|
263
404
|
new_es
|
264
405
|
end
|
265
406
|
|
266
|
-
def
|
267
|
-
|
268
|
-
|
407
|
+
def select_authenticate_users(node, username)
|
408
|
+
if node.nil? || node[:users].empty?
|
409
|
+
@security.users.select{|u| u.username == username}
|
410
|
+
else
|
411
|
+
@security.users.select{|u| node[:users].include?(u.username) && u.username == username}
|
412
|
+
end
|
269
413
|
end
|
270
414
|
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
def initialize(io, linger_timeout, log, resolve_hostname, on_message)
|
275
|
-
super(io)
|
276
|
-
|
277
|
-
@peeraddr = nil
|
278
|
-
if io.is_a?(TCPSocket) # for unix domain socket support in the future
|
279
|
-
io.do_not_reverse_lookup = !resolve_hostname unless resolve_hostname.nil?
|
280
|
-
|
281
|
-
@peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
|
282
|
-
opt = [1, linger_timeout].pack('I!I!') # { int l_onoff; int l_linger; }
|
283
|
-
io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
284
|
-
end
|
415
|
+
def generate_salt
|
416
|
+
::SecureRandom.random_bytes(16)
|
417
|
+
end
|
285
418
|
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
|
292
|
-
rescue => e
|
293
|
-
remote_port = nil
|
294
|
-
remote_addr = nil
|
295
|
-
end
|
296
|
-
"accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
|
297
|
-
}
|
298
|
-
end
|
419
|
+
def generate_helo(nonce, user_auth_salt)
|
420
|
+
log.debug "generating helo"
|
421
|
+
# ['HELO', options(hash)]
|
422
|
+
['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
|
423
|
+
end
|
299
424
|
|
300
|
-
|
425
|
+
def check_ping(message, remote_addr, user_auth_salt, nonce)
|
426
|
+
log.debug "checking ping"
|
427
|
+
# ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']
|
428
|
+
unless message.size == 6 && message[0] == 'PING'
|
429
|
+
return false, 'invalid ping message'
|
301
430
|
end
|
431
|
+
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
302
432
|
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
@serializer = :to_json.to_proc
|
308
|
-
@y = Yajl::Parser.new
|
309
|
-
@y.on_parse_complete = lambda { |obj|
|
310
|
-
option = @on_message.call(obj, @chunk_counter, @peeraddr)
|
311
|
-
respond option if option
|
312
|
-
@chunk_counter = 0
|
313
|
-
}
|
314
|
-
else
|
315
|
-
m = method(:on_read_msgpack)
|
316
|
-
@serializer = :to_msgpack.to_proc
|
317
|
-
@u = Fluent::Engine.msgpack_factory.unpacker
|
318
|
-
end
|
319
|
-
|
320
|
-
(class << self; self; end).module_eval do
|
321
|
-
define_method(:on_read, m)
|
322
|
-
end
|
323
|
-
m.call(data)
|
433
|
+
node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
|
434
|
+
if !node && !@security.allow_anonymous_source
|
435
|
+
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
436
|
+
return false, "anonymous source host '#{remote_addr}' denied", nil
|
324
437
|
end
|
325
438
|
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
@log.error_backtrace
|
332
|
-
close
|
439
|
+
shared_key = node ? node[:shared_key] : @security.shared_key
|
440
|
+
serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(nonce).update(shared_key).hexdigest
|
441
|
+
if shared_key_hexdigest != serverside
|
442
|
+
log.warn "Shared key mismatch", address: remote_addr, hostname: hostname
|
443
|
+
return false, 'shared_key mismatch', nil
|
333
444
|
end
|
334
445
|
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
446
|
+
if @security.user_auth
|
447
|
+
users = select_authenticate_users(node, username)
|
448
|
+
success = false
|
449
|
+
users.each do |user|
|
450
|
+
passhash = Digest::SHA512.new.update(user_auth_salt).update(username).update(user[:password]).hexdigest
|
451
|
+
success ||= (passhash == password_digest)
|
341
452
|
end
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
close
|
346
|
-
end
|
347
|
-
|
348
|
-
def respond(option)
|
349
|
-
if option && option['chunk']
|
350
|
-
res = { 'ack' => option['chunk'] }
|
351
|
-
write @serializer.call(res)
|
352
|
-
@log.trace { "sent response to fluent socket" }
|
453
|
+
unless success
|
454
|
+
log.warn "Authentication failed", address: remote_addr, hostname: hostname, username: username
|
455
|
+
return false, 'username/password mismatch', nil
|
353
456
|
end
|
354
457
|
end
|
355
458
|
|
356
|
-
|
357
|
-
@log.trace { "closed socket" }
|
358
|
-
end
|
459
|
+
return true, shared_key_salt, shared_key
|
359
460
|
end
|
360
461
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
end
|
367
|
-
|
368
|
-
def on_readable
|
369
|
-
begin
|
370
|
-
msg, addr = @io.recvfrom(1024)
|
371
|
-
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
372
|
-
return
|
373
|
-
end
|
374
|
-
host = addr[3]
|
375
|
-
port = addr[1]
|
376
|
-
@callback.call(host, port, msg)
|
377
|
-
rescue
|
378
|
-
# TODO log?
|
462
|
+
def generate_pong(auth_result, reason_or_salt, nonce, shared_key)
|
463
|
+
log.debug "generating pong"
|
464
|
+
# ['PONG', bool(authentication result), 'reason if authentication failed', self_hostname, sha512_hex(salt + self_hostname + nonce + sharedkey)]
|
465
|
+
unless auth_result
|
466
|
+
return ['PONG', false, reason_or_salt, '', '']
|
379
467
|
end
|
380
|
-
end
|
381
468
|
|
382
|
-
|
383
|
-
|
384
|
-
begin
|
385
|
-
@usock.send "\0", 0, host, port
|
386
|
-
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
387
|
-
end
|
469
|
+
shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
|
470
|
+
['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
|
388
471
|
end
|
389
472
|
end
|
390
473
|
end
|