fluentd 0.12.40 → 1.6.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
- data/.gitignore +5 -0
- data/.gitlab/cicd-template.yaml +10 -0
- data/.gitlab-ci.yml +147 -0
- data/.travis.yml +56 -20
- data/ADOPTERS.md +5 -0
- data/CHANGELOG.md +1369 -0
- data/CONTRIBUTING.md +16 -5
- data/GOVERNANCE.md +55 -0
- data/Gemfile +5 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +7 -0
- data/README.md +23 -11
- data/Rakefile +48 -2
- data/Vagrantfile +17 -0
- data/appveyor.yml +37 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +3 -0
- data/code-of-conduct.md +3 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_dummy_with_compression.conf +23 -0
- data/example/in_forward.conf +7 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +3 -1
- data/example/in_out_forward.conf +17 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_forward.conf +13 -13
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +29 -0
- data/fluentd.gemspec +21 -11
- data/lib/fluent/agent.rb +67 -90
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/ca_generate.rb +181 -0
- data/lib/fluent/command/cat.rb +42 -18
- data/lib/fluent/command/debug.rb +12 -10
- data/lib/fluent/command/fluentd.rb +153 -5
- data/lib/fluent/command/plugin_config_formatter.rb +292 -0
- data/lib/fluent/command/plugin_generator.rb +324 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +718 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/configure_proxy.rb +210 -62
- data/lib/fluent/config/dsl.rb +12 -5
- data/lib/fluent/config/element.rb +107 -9
- data/lib/fluent/config/literal_parser.rb +9 -3
- data/lib/fluent/config/parser.rb +4 -4
- data/lib/fluent/config/section.rb +51 -14
- data/lib/fluent/config/types.rb +28 -13
- data/lib/fluent/config/v1_parser.rb +3 -5
- data/lib/fluent/config.rb +23 -20
- data/lib/fluent/configurable.rb +79 -21
- data/lib/fluent/counter/base_socket.rb +46 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +102 -65
- data/lib/fluent/env.rb +7 -3
- data/lib/fluent/error.rb +30 -0
- data/lib/fluent/event.rb +197 -21
- data/lib/fluent/event_router.rb +93 -10
- data/lib/fluent/filter.rb +2 -50
- data/lib/fluent/formatter.rb +4 -293
- data/lib/fluent/input.rb +2 -32
- data/lib/fluent/label.rb +10 -2
- data/lib/fluent/load.rb +3 -3
- data/lib/fluent/log.rb +348 -81
- data/lib/fluent/match.rb +37 -36
- data/lib/fluent/mixin.rb +12 -176
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +10 -612
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +4 -800
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +192 -0
- data/lib/fluent/plugin/buf_file.rb +128 -174
- data/lib/fluent/plugin/buf_memory.rb +9 -92
- data/lib/fluent/plugin/buffer/chunk.rb +221 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/buffer.rb +779 -0
- data/lib/fluent/plugin/compressable.rb +92 -0
- data/lib/fluent/plugin/exec_util.rb +3 -108
- data/lib/fluent/plugin/file_util.rb +4 -34
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +117 -34
- data/lib/fluent/plugin/filter_parser.rb +85 -62
- data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
- data/lib/fluent/plugin/filter_stdout.rb +15 -12
- data/lib/fluent/plugin/formatter.rb +50 -0
- data/lib/fluent/plugin/formatter_csv.rb +52 -0
- data/lib/fluent/plugin/formatter_hash.rb +33 -0
- data/lib/fluent/plugin/formatter_json.rb +55 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +51 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +38 -0
- data/lib/fluent/plugin/in_debug_agent.rb +17 -6
- data/lib/fluent/plugin/in_dummy.rb +47 -20
- data/lib/fluent/plugin/in_exec.rb +55 -123
- data/lib/fluent/plugin/in_forward.rb +299 -216
- data/lib/fluent/plugin/in_gc_stat.rb +14 -36
- data/lib/fluent/plugin/in_http.rb +204 -91
- data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
- data/lib/fluent/plugin/in_object_space.rb +13 -41
- data/lib/fluent/plugin/in_syslog.rb +112 -134
- data/lib/fluent/plugin/in_tail.rb +408 -745
- data/lib/fluent/plugin/in_tcp.rb +66 -9
- data/lib/fluent/plugin/in_udp.rb +60 -11
- data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
- data/lib/fluent/plugin/input.rb +37 -0
- data/lib/fluent/plugin/multi_output.rb +158 -0
- data/lib/fluent/plugin/out_copy.rb +23 -35
- data/lib/fluent/plugin/out_exec.rb +67 -70
- data/lib/fluent/plugin/out_exec_filter.rb +204 -271
- data/lib/fluent/plugin/out_file.rb +267 -73
- data/lib/fluent/plugin/out_forward.rb +854 -325
- data/lib/fluent/plugin/out_null.rb +42 -9
- data/lib/fluent/plugin/out_relabel.rb +9 -5
- data/lib/fluent/plugin/out_roundrobin.rb +18 -37
- data/lib/fluent/plugin/out_secondary_file.rb +133 -0
- data/lib/fluent/plugin/out_stdout.rb +43 -10
- data/lib/fluent/plugin/out_stream.rb +7 -2
- data/lib/fluent/plugin/output.rb +1498 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +191 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +39 -0
- data/lib/fluent/plugin/parser_json.rb +94 -0
- data/lib/fluent/plugin/parser_ltsv.rb +49 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +106 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +142 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/socket_util.rb +3 -143
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +164 -0
- data/lib/fluent/plugin/string_util.rb +3 -15
- data/lib/fluent/plugin.rb +122 -121
- data/lib/fluent/plugin_helper/cert_option.rb +178 -0
- data/lib/fluent/plugin_helper/child_process.rb +364 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
- data/lib/fluent/plugin_helper/http_server.rb +76 -0
- data/lib/fluent/plugin_helper/inject.rb +151 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
- data/lib/fluent/plugin_helper/retry_state.rb +205 -0
- data/lib/fluent/plugin_helper/server.rb +807 -0
- data/lib/fluent/plugin_helper/socket.rb +250 -0
- data/lib/fluent/plugin_helper/socket_option.rb +80 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +179 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +73 -0
- data/lib/fluent/plugin_id.rb +80 -0
- data/lib/fluent/process.rb +3 -489
- data/lib/fluent/registry.rb +52 -10
- data/lib/fluent/root_agent.rb +204 -42
- data/lib/fluent/supervisor.rb +597 -359
- data/lib/fluent/system_config.rb +131 -42
- data/lib/fluent/test/base.rb +6 -54
- data/lib/fluent/test/driver/base.rb +224 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +0 -1
- data/lib/fluent/test/formatter_test.rb +4 -1
- data/lib/fluent/test/helpers.rb +58 -10
- data/lib/fluent/test/input_test.rb +27 -19
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +28 -39
- data/lib/fluent/test/parser_test.rb +3 -1
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +33 -1
- data/lib/fluent/time.rb +450 -1
- data/lib/fluent/timezone.rb +27 -3
- data/lib/fluent/{status.rb → unique_id.rb} +15 -24
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +85 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +346 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_fluentd.rb +901 -0
- data/test/command/test_plugin_config_formatter.rb +276 -0
- data/test/command/test_plugin_generator.rb +92 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/test_config_parser.rb +126 -2
- data/test/config/test_configurable.rb +946 -187
- data/test/config/test_configure_proxy.rb +424 -74
- data/test/config/test_dsl.rb +11 -11
- data/test/config/test_element.rb +500 -0
- data/test/config/test_literal_parser.rb +8 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +79 -7
- data/test/config/test_system_config.rb +122 -35
- data/test/config/test_types.rb +38 -0
- data/test/counter/test_client.rb +559 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +89 -6
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +823 -460
- data/test/plugin/test_buf_memory.rb +32 -194
- data/test/plugin/test_buffer.rb +1233 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +844 -0
- data/test/plugin/test_buffer_memory_chunk.rb +338 -0
- data/test/plugin/test_compressable.rb +84 -0
- data/test/plugin/test_filter.rb +357 -0
- data/test/plugin/test_filter_grep.rb +540 -29
- data/test/plugin/test_filter_parser.rb +439 -452
- data/test/plugin/test_filter_record_transformer.rb +123 -166
- data/test/plugin/test_filter_stdout.rb +160 -72
- data/test/plugin/test_formatter_csv.rb +111 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +51 -0
- data/test/plugin/test_formatter_ltsv.rb +62 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_formatter_tsv.rb +68 -0
- data/test/plugin/test_in_debug_agent.rb +24 -1
- data/test/plugin/test_in_dummy.rb +111 -18
- data/test/plugin/test_in_exec.rb +200 -113
- data/test/plugin/test_in_forward.rb +990 -387
- data/test/plugin/test_in_gc_stat.rb +10 -8
- data/test/plugin/test_in_http.rb +600 -224
- data/test/plugin/test_in_monitor_agent.rb +690 -0
- data/test/plugin/test_in_object_space.rb +24 -8
- data/test/plugin/test_in_syslog.rb +154 -215
- data/test/plugin/test_in_tail.rb +1006 -707
- data/test/plugin/test_in_tcp.rb +125 -48
- data/test/plugin/test_in_udp.rb +204 -63
- data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
- data/test/plugin/test_input.rb +126 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_copy.rb +117 -112
- data/test/plugin/test_out_exec.rb +258 -53
- data/test/plugin/test_out_exec_filter.rb +538 -115
- data/test/plugin/test_out_file.rb +865 -178
- data/test/plugin/test_out_forward.rb +998 -210
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +36 -29
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +135 -37
- data/test/plugin/test_out_stream.rb +18 -0
- data/test/plugin/test_output.rb +984 -0
- data/test/plugin/test_output_as_buffered.rb +2021 -0
- data/test/plugin/test_output_as_buffered_backup.rb +312 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +911 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +103 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +145 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +441 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/test_cert_option.rb +16 -0
- data/test/plugin_helper/test_child_process.rb +794 -0
- data/test/plugin_helper/test_compat_parameters.rb +353 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +205 -0
- data/test/plugin_helper/test_inject.rb +519 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +197 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1714 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +132 -0
- data/test/scripts/exec_script.rb +0 -6
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/out_test.rb +23 -15
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +16 -7
- data/test/test_configdsl.rb +2 -2
- data/test/test_event.rb +360 -13
- data/test/test_event_router.rb +108 -11
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +48 -6
- data/test/test_formatter.rb +11 -391
- data/test/test_input.rb +1 -1
- data/test/test_log.rb +591 -31
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +121 -185
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +177 -10
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +101 -0
- data/test/test_process.rb +8 -42
- data/test/test_root_agent.rb +766 -21
- data/test/test_supervisor.rb +481 -0
- data/test/test_test_drivers.rb +135 -0
- data/test/test_time_formatter.rb +282 -0
- data/test/test_time_parser.rb +231 -0
- data/test/test_unique_id.rb +47 -0
- metadata +454 -60
- data/COPYING +0 -14
- data/ChangeLog +0 -666
- data/lib/fluent/buffer.rb +0 -365
- data/lib/fluent/plugin/in_status.rb +0 -76
- data/test/plugin/test_in_status.rb +0 -38
- data/test/test_buffer.rb +0 -624
- data/test/test_parser.rb +0 -1305
@@ -0,0 +1,807 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin_helper/event_loop'
|
18
|
+
|
19
|
+
require 'serverengine'
|
20
|
+
require 'cool.io'
|
21
|
+
require 'socket'
|
22
|
+
require 'ipaddr'
|
23
|
+
require 'fcntl'
|
24
|
+
require 'openssl'
|
25
|
+
|
26
|
+
require_relative 'socket_option'
|
27
|
+
require_relative 'cert_option'
|
28
|
+
|
29
|
+
module Fluent
|
30
|
+
module PluginHelper
|
31
|
+
module Server
|
32
|
+
include Fluent::PluginHelper::EventLoop
|
33
|
+
include Fluent::PluginHelper::SocketOption
|
34
|
+
include Fluent::PluginHelper::CertOption
|
35
|
+
|
36
|
+
# This plugin helper doesn't support these things for now:
|
37
|
+
# * TCP/TLS keepalive
|
38
|
+
# * TLS session cache/tickets
|
39
|
+
# * unix domain sockets
|
40
|
+
|
41
|
+
# stop : [-]
|
42
|
+
# shutdown : detach server event handler from event loop (event_loop)
|
43
|
+
# close : close listening sockets
|
44
|
+
# terminate: remote all server instances
|
45
|
+
|
46
|
+
attr_reader :_servers # for tests
|
47
|
+
|
48
|
+
def server_wait_until_start
|
49
|
+
# event_loop_wait_until_start works well for this
|
50
|
+
end
|
51
|
+
|
52
|
+
def server_wait_until_stop
|
53
|
+
sleep 0.1 while @_servers.any?{|si| si.server.attached? }
|
54
|
+
@_servers.each{|si| si.server.close rescue nil }
|
55
|
+
end
|
56
|
+
|
57
|
+
PROTOCOLS = [:tcp, :udp, :tls, :unix]
|
58
|
+
CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
|
59
|
+
|
60
|
+
# server_create_connection(:title, @port) do |conn|
|
61
|
+
# # on connection
|
62
|
+
# source_addr = conn.remote_host
|
63
|
+
# source_port = conn.remote_port
|
64
|
+
# conn.data do |data|
|
65
|
+
# # on data
|
66
|
+
# conn.write resp # ...
|
67
|
+
# conn.close
|
68
|
+
# end
|
69
|
+
# end
|
70
|
+
def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)
|
71
|
+
proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
|
72
|
+
|
73
|
+
raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
|
74
|
+
raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
|
75
|
+
raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
|
76
|
+
raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
|
77
|
+
|
78
|
+
raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
|
79
|
+
|
80
|
+
raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
|
81
|
+
raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
|
82
|
+
|
83
|
+
if proto == :tcp || proto == :tls # default linger_timeout only for server
|
84
|
+
socket_options[:linger_timeout] ||= 0
|
85
|
+
end
|
86
|
+
|
87
|
+
socket_option_validate!(proto, **socket_options)
|
88
|
+
socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
|
89
|
+
|
90
|
+
case proto
|
91
|
+
when :tcp
|
92
|
+
server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
|
93
|
+
when :tls
|
94
|
+
transport_config = if tls_options
|
95
|
+
server_create_transport_section_object(tls_options)
|
96
|
+
elsif @transport_config && @transport_config.protocol == :tls
|
97
|
+
@transport_config
|
98
|
+
else
|
99
|
+
raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
|
100
|
+
end
|
101
|
+
server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)
|
102
|
+
when :unix
|
103
|
+
raise "not implemented yet"
|
104
|
+
else
|
105
|
+
raise "unknown protocol #{proto}"
|
106
|
+
end
|
107
|
+
|
108
|
+
server_attach(title, proto, port, bind, shared, server)
|
109
|
+
end
|
110
|
+
|
111
|
+
# server_create(:title, @port) do |data|
|
112
|
+
# # ...
|
113
|
+
# end
|
114
|
+
# server_create(:title, @port) do |data, conn|
|
115
|
+
# # ...
|
116
|
+
# end
|
117
|
+
# server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
|
118
|
+
# sock.remote_host
|
119
|
+
# sock.remote_port
|
120
|
+
# # ...
|
121
|
+
# end
|
122
|
+
def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
|
123
|
+
proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
|
124
|
+
|
125
|
+
raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
|
126
|
+
raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
|
127
|
+
raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
|
128
|
+
|
129
|
+
raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
|
130
|
+
raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
|
131
|
+
|
132
|
+
raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
|
133
|
+
raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
|
134
|
+
|
135
|
+
if proto == :tcp || proto == :tls # default linger_timeout only for server
|
136
|
+
socket_options[:linger_timeout] ||= 0
|
137
|
+
end
|
138
|
+
|
139
|
+
unless socket
|
140
|
+
socket_option_validate!(proto, **socket_options)
|
141
|
+
socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
|
142
|
+
end
|
143
|
+
|
144
|
+
if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
|
145
|
+
raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
|
146
|
+
end
|
147
|
+
if proto != :udp # UDP options
|
148
|
+
raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
|
149
|
+
raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
|
150
|
+
end
|
151
|
+
|
152
|
+
case proto
|
153
|
+
when :tcp
|
154
|
+
server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
|
155
|
+
conn.data(&callback)
|
156
|
+
end
|
157
|
+
when :tls
|
158
|
+
transport_config = if tls_options
|
159
|
+
server_create_transport_section_object(tls_options)
|
160
|
+
elsif @transport_config && @transport_config.protocol == :tls
|
161
|
+
@transport_config
|
162
|
+
else
|
163
|
+
raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
|
164
|
+
end
|
165
|
+
server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|
|
166
|
+
conn.data(&callback)
|
167
|
+
end
|
168
|
+
when :udp
|
169
|
+
raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
|
170
|
+
if socket
|
171
|
+
sock = socket
|
172
|
+
close_socket = false
|
173
|
+
else
|
174
|
+
sock = server_create_udp_socket(shared, bind, port)
|
175
|
+
socket_option_setter.call(sock)
|
176
|
+
close_socket = true
|
177
|
+
end
|
178
|
+
server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
|
179
|
+
when :unix
|
180
|
+
raise "not implemented yet"
|
181
|
+
else
|
182
|
+
raise "BUG: unknown protocol #{proto}"
|
183
|
+
end
|
184
|
+
|
185
|
+
server_attach(title, proto, port, bind, shared, server)
|
186
|
+
end
|
187
|
+
|
188
|
+
def server_create_tcp(title, port, **kwargs, &callback)
|
189
|
+
server_create(title, port, proto: :tcp, **kwargs, &callback)
|
190
|
+
end
|
191
|
+
|
192
|
+
def server_create_udp(title, port, **kwargs, &callback)
|
193
|
+
server_create(title, port, proto: :udp, **kwargs, &callback)
|
194
|
+
end
|
195
|
+
|
196
|
+
def server_create_tls(title, port, **kwargs, &callback)
|
197
|
+
server_create(title, port, proto: :tls, **kwargs, &callback)
|
198
|
+
end
|
199
|
+
|
200
|
+
def server_create_unix(title, port, **kwargs, &callback)
|
201
|
+
server_create(title, port, proto: :unix, **kwargs, &callback)
|
202
|
+
end
|
203
|
+
|
204
|
+
ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
|
205
|
+
|
206
|
+
def server_attach(title, proto, port, bind, shared, server)
|
207
|
+
@_servers << ServerInfo.new(title, proto, port, bind, shared, server)
|
208
|
+
event_loop_attach(server)
|
209
|
+
end
|
210
|
+
|
211
|
+
def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
|
212
|
+
sock = server_create_tcp_socket(shared, bind, port)
|
213
|
+
socket_option_setter.call(sock)
|
214
|
+
close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
|
215
|
+
server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
|
216
|
+
unless conn.closing
|
217
|
+
@_server_mutex.synchronize do
|
218
|
+
@_server_connections << conn
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
222
|
+
server.listen(backlog) if backlog
|
223
|
+
server
|
224
|
+
end
|
225
|
+
|
226
|
+
def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)
|
227
|
+
context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
|
228
|
+
sock = server_create_tcp_socket(shared, bind, port)
|
229
|
+
socket_option_setter.call(sock)
|
230
|
+
close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
|
231
|
+
server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
|
232
|
+
unless conn.closing
|
233
|
+
@_server_mutex.synchronize do
|
234
|
+
@_server_connections << conn
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
server.listen(backlog) if backlog
|
239
|
+
server
|
240
|
+
end
|
241
|
+
|
242
|
+
SERVER_TRANSPORT_PARAMS = [
|
243
|
+
:protocol, :version, :ciphers, :insecure,
|
244
|
+
:ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
|
245
|
+
:ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
|
246
|
+
:generate_private_key_length,
|
247
|
+
:generate_cert_country, :generate_cert_state, :generate_cert_state,
|
248
|
+
:generate_cert_locality, :generate_cert_common_name,
|
249
|
+
:generate_cert_expiration, :generate_cert_digest,
|
250
|
+
]
|
251
|
+
|
252
|
+
def server_create_transport_section_object(opts)
|
253
|
+
transport_section = configured_section_create(:transport)
|
254
|
+
SERVER_TRANSPORT_PARAMS.each do |param|
|
255
|
+
if opts.has_key?(param)
|
256
|
+
transport_section[param] = opts[param]
|
257
|
+
end
|
258
|
+
end
|
259
|
+
transport_section
|
260
|
+
end
|
261
|
+
|
262
|
+
module ServerTransportParams
|
263
|
+
TLS_DEFAULT_VERSION = :'TLSv1_2'
|
264
|
+
TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
|
265
|
+
### follow httpclient configuration by nahi
|
266
|
+
# OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
|
267
|
+
CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
|
268
|
+
|
269
|
+
include Fluent::Configurable
|
270
|
+
config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
|
271
|
+
config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
|
272
|
+
config_param :version, :enum, list: TLS_SUPPORTED_VERSIONS, default: TLS_DEFAULT_VERSION
|
273
|
+
|
274
|
+
config_param :ciphers, :string, default: CIPHERS_DEFAULT
|
275
|
+
config_param :insecure, :bool, default: false
|
276
|
+
|
277
|
+
# Cert signed by public CA
|
278
|
+
config_param :ca_path, :string, default: nil
|
279
|
+
config_param :cert_path, :string, default: nil
|
280
|
+
config_param :private_key_path, :string, default: nil
|
281
|
+
config_param :private_key_passphrase, :string, default: nil, secret: true
|
282
|
+
config_param :client_cert_auth, :bool, default: false
|
283
|
+
|
284
|
+
# Cert generated and signed by private CA Certificate
|
285
|
+
config_param :ca_cert_path, :string, default: nil
|
286
|
+
config_param :ca_private_key_path, :string, default: nil
|
287
|
+
config_param :ca_private_key_passphrase, :string, default: nil, secret: true
|
288
|
+
|
289
|
+
# Options for generating certs by private CA certs or self-signed
|
290
|
+
config_param :generate_private_key_length, :integer, default: 2048
|
291
|
+
config_param :generate_cert_country, :string, default: 'US'
|
292
|
+
config_param :generate_cert_state, :string, default: 'CA'
|
293
|
+
config_param :generate_cert_locality, :string, default: 'Mountain View'
|
294
|
+
config_param :generate_cert_common_name, :string, default: nil
|
295
|
+
config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later
|
296
|
+
config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
def self.included(mod)
|
301
|
+
mod.include ServerTransportParams
|
302
|
+
end
|
303
|
+
|
304
|
+
def initialize
|
305
|
+
super
|
306
|
+
@_servers = []
|
307
|
+
@_server_connections = []
|
308
|
+
@_server_mutex = Mutex.new
|
309
|
+
end
|
310
|
+
|
311
|
+
def configure(conf)
|
312
|
+
super
|
313
|
+
|
314
|
+
if @transport_config
|
315
|
+
if @transport_config.protocol == :tls
|
316
|
+
cert_option_server_validate!(@transport_config)
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
def stop
|
322
|
+
@_server_mutex.synchronize do
|
323
|
+
@_servers.each do |si|
|
324
|
+
si.server.detach if si.server.attached?
|
325
|
+
# to refuse more connections: (connected sockets are still alive here)
|
326
|
+
si.server.close rescue nil
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
super
|
331
|
+
end
|
332
|
+
|
333
|
+
def shutdown
|
334
|
+
@_server_connections.each do |conn|
|
335
|
+
conn.close rescue nil
|
336
|
+
end
|
337
|
+
|
338
|
+
super
|
339
|
+
end
|
340
|
+
|
341
|
+
def terminate
|
342
|
+
@_servers = []
|
343
|
+
super
|
344
|
+
end
|
345
|
+
|
346
|
+
def server_socket_manager_client
|
347
|
+
socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
|
348
|
+
if Fluent.windows?
|
349
|
+
socket_manager_path = socket_manager_path.to_i
|
350
|
+
end
|
351
|
+
ServerEngine::SocketManager::Client.new(socket_manager_path)
|
352
|
+
end
|
353
|
+
|
354
|
+
def server_create_tcp_socket(shared, bind, port)
|
355
|
+
sock = if shared
|
356
|
+
server_socket_manager_client.listen_tcp(bind, port)
|
357
|
+
else
|
358
|
+
TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
|
359
|
+
end
|
360
|
+
# close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
|
361
|
+
sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
|
362
|
+
sock
|
363
|
+
end
|
364
|
+
|
365
|
+
def server_create_udp_socket(shared, bind, port)
|
366
|
+
sock = if shared
|
367
|
+
server_socket_manager_client.listen_udp(bind, port)
|
368
|
+
else
|
369
|
+
family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
|
370
|
+
usock = UDPSocket.new(family)
|
371
|
+
usock.bind(bind, port)
|
372
|
+
usock
|
373
|
+
end
|
374
|
+
# close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
|
375
|
+
sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
|
376
|
+
sock
|
377
|
+
end
|
378
|
+
|
379
|
+
# Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
|
380
|
+
PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
|
381
|
+
|
382
|
+
class CallbackSocket
|
383
|
+
def initialize(server_type, sock, enabled_events = [], close_socket: true)
|
384
|
+
@server_type = server_type
|
385
|
+
@sock = sock
|
386
|
+
@peeraddr = nil
|
387
|
+
@enabled_events = enabled_events
|
388
|
+
@close_socket = close_socket
|
389
|
+
end
|
390
|
+
|
391
|
+
def remote_addr
|
392
|
+
@peeraddr[3]
|
393
|
+
end
|
394
|
+
|
395
|
+
def remote_host
|
396
|
+
@peeraddr[2]
|
397
|
+
end
|
398
|
+
|
399
|
+
def remote_port
|
400
|
+
@peeraddr[1]
|
401
|
+
end
|
402
|
+
|
403
|
+
def send(data, flags = 0)
|
404
|
+
@sock.send(data, flags)
|
405
|
+
end
|
406
|
+
|
407
|
+
def write(data)
|
408
|
+
raise "not implemented here"
|
409
|
+
end
|
410
|
+
|
411
|
+
def close_after_write_complete
|
412
|
+
@sock.close_after_write_complete = true
|
413
|
+
end
|
414
|
+
|
415
|
+
def close
|
416
|
+
@sock.close if @close_socket
|
417
|
+
end
|
418
|
+
|
419
|
+
def data(&callback)
|
420
|
+
on(:data, &callback)
|
421
|
+
end
|
422
|
+
|
423
|
+
def on(event, &callback)
|
424
|
+
raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
|
425
|
+
case event
|
426
|
+
when :data
|
427
|
+
@sock.data(&callback)
|
428
|
+
when :write_complete
|
429
|
+
cb = ->(){ callback.call(self) }
|
430
|
+
@sock.on_write_complete(&cb)
|
431
|
+
when :close
|
432
|
+
cb = ->(){ callback.call(self) }
|
433
|
+
@sock.on_close(&cb)
|
434
|
+
else
|
435
|
+
raise "BUG: unknown event: #{event}"
|
436
|
+
end
|
437
|
+
end
|
438
|
+
end
|
439
|
+
|
440
|
+
class TCPCallbackSocket < CallbackSocket
|
441
|
+
ENABLED_EVENTS = [:data, :write_complete, :close]
|
442
|
+
|
443
|
+
attr_accessor :buffer
|
444
|
+
|
445
|
+
def initialize(sock)
|
446
|
+
super("tcp", sock, ENABLED_EVENTS)
|
447
|
+
@peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)
|
448
|
+
@buffer = ''
|
449
|
+
end
|
450
|
+
|
451
|
+
def write(data)
|
452
|
+
@sock.write(data)
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
class TLSCallbackSocket < CallbackSocket
|
457
|
+
ENABLED_EVENTS = [:data, :write_complete, :close]
|
458
|
+
|
459
|
+
attr_accessor :buffer
|
460
|
+
|
461
|
+
def initialize(sock)
|
462
|
+
super("tls", sock, ENABLED_EVENTS)
|
463
|
+
@peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
|
464
|
+
@buffer = ''
|
465
|
+
end
|
466
|
+
|
467
|
+
def write(data)
|
468
|
+
@sock.write(data)
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
class UDPCallbackSocket < CallbackSocket
|
473
|
+
ENABLED_EVENTS = []
|
474
|
+
|
475
|
+
def initialize(sock, peeraddr, **kwargs)
|
476
|
+
super("udp", sock, ENABLED_EVENTS, **kwargs)
|
477
|
+
@peeraddr = peeraddr
|
478
|
+
end
|
479
|
+
|
480
|
+
def remote_addr
|
481
|
+
@peeraddr[3]
|
482
|
+
end
|
483
|
+
|
484
|
+
def remote_host
|
485
|
+
@peeraddr[2]
|
486
|
+
end
|
487
|
+
|
488
|
+
def remote_port
|
489
|
+
@peeraddr[1]
|
490
|
+
end
|
491
|
+
|
492
|
+
def write(data)
|
493
|
+
@sock.send(data, 0, @peeraddr[3], @peeraddr[1])
|
494
|
+
end
|
495
|
+
end
|
496
|
+
|
497
|
+
module EventHandler
|
498
|
+
class UDPServer < Coolio::IO
|
499
|
+
attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
|
500
|
+
|
501
|
+
def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
|
502
|
+
raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
|
503
|
+
|
504
|
+
super(sock)
|
505
|
+
|
506
|
+
@sock = sock
|
507
|
+
@max_bytes = max_bytes
|
508
|
+
@flags = flags
|
509
|
+
@close_socket = close_socket
|
510
|
+
@log = log
|
511
|
+
@under_plugin_development = under_plugin_development
|
512
|
+
@callback = callback
|
513
|
+
|
514
|
+
on_readable_impl = case @callback.arity
|
515
|
+
when 1 then :on_readable_without_sock
|
516
|
+
when 2 then :on_readable_with_sock
|
517
|
+
else
|
518
|
+
raise "BUG: callback block must have 1 or 2 arguments"
|
519
|
+
end
|
520
|
+
self.define_singleton_method(:on_readable, method(on_readable_impl))
|
521
|
+
end
|
522
|
+
|
523
|
+
def on_readable_without_sock
|
524
|
+
begin
|
525
|
+
data = @sock.recv(@max_bytes, @flags)
|
526
|
+
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
|
527
|
+
return
|
528
|
+
end
|
529
|
+
@callback.call(data)
|
530
|
+
rescue => e
|
531
|
+
@log.error "unexpected error in processing UDP data", error: e
|
532
|
+
@log.error_backtrace
|
533
|
+
raise if @under_plugin_development
|
534
|
+
end
|
535
|
+
|
536
|
+
def on_readable_with_sock
|
537
|
+
begin
|
538
|
+
data, addr = @sock.recvfrom(@max_bytes)
|
539
|
+
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
|
540
|
+
return
|
541
|
+
end
|
542
|
+
@callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
|
543
|
+
rescue => e
|
544
|
+
@log.error "unexpected error in processing UDP data", error: e
|
545
|
+
@log.error_backtrace
|
546
|
+
raise if @under_plugin_development
|
547
|
+
end
|
548
|
+
end
|
549
|
+
|
550
|
+
class TCPServer < Coolio::TCPSocket
|
551
|
+
attr_reader :closing
|
552
|
+
attr_writer :close_after_write_complete
|
553
|
+
|
554
|
+
def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
|
555
|
+
raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
|
556
|
+
|
557
|
+
socket_option_setter.call(sock)
|
558
|
+
|
559
|
+
@_handler_socket = sock
|
560
|
+
super(sock)
|
561
|
+
|
562
|
+
@log = log
|
563
|
+
@under_plugin_development = under_plugin_development
|
564
|
+
|
565
|
+
@connect_callback = connect_callback
|
566
|
+
@data_callback = nil
|
567
|
+
@close_callback = close_callback
|
568
|
+
|
569
|
+
@callback_connection = nil
|
570
|
+
@close_after_write_complete = false
|
571
|
+
@closing = false
|
572
|
+
|
573
|
+
@mutex = Mutex.new # to serialize #write and #close
|
574
|
+
end
|
575
|
+
|
576
|
+
def to_io
|
577
|
+
@_handler_socket
|
578
|
+
end
|
579
|
+
|
580
|
+
def data(&callback)
|
581
|
+
raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
|
582
|
+
@data_callback = callback
|
583
|
+
on_read_impl = case callback.arity
|
584
|
+
when 1 then :on_read_without_connection
|
585
|
+
when 2 then :on_read_with_connection
|
586
|
+
else
|
587
|
+
raise "BUG: callback block must have 1 or 2 arguments"
|
588
|
+
end
|
589
|
+
self.define_singleton_method(:on_read, method(on_read_impl))
|
590
|
+
end
|
591
|
+
|
592
|
+
def write(data)
|
593
|
+
@mutex.synchronize do
|
594
|
+
super
|
595
|
+
end
|
596
|
+
end
|
597
|
+
|
598
|
+
def on_writable
|
599
|
+
super
|
600
|
+
close if @close_after_write_complete
|
601
|
+
end
|
602
|
+
|
603
|
+
def on_connect
|
604
|
+
@callback_connection = TCPCallbackSocket.new(self)
|
605
|
+
@connect_callback.call(@callback_connection)
|
606
|
+
unless @data_callback
|
607
|
+
raise "connection callback must call #data to set data callback"
|
608
|
+
end
|
609
|
+
end
|
610
|
+
|
611
|
+
def on_read_without_connection(data)
|
612
|
+
@data_callback.call(data)
|
613
|
+
rescue => e
|
614
|
+
@log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
|
615
|
+
@log.error_backtrace
|
616
|
+
close rescue nil
|
617
|
+
raise if @under_plugin_development
|
618
|
+
end
|
619
|
+
|
620
|
+
def on_read_with_connection(data)
|
621
|
+
@data_callback.call(data, @callback_connection)
|
622
|
+
rescue => e
|
623
|
+
@log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
|
624
|
+
@log.error_backtrace
|
625
|
+
close rescue nil
|
626
|
+
raise if @under_plugin_development
|
627
|
+
end
|
628
|
+
|
629
|
+
def close
|
630
|
+
@mutex.synchronize do
|
631
|
+
return if @closing
|
632
|
+
@closing = true
|
633
|
+
@close_callback.call(self)
|
634
|
+
super
|
635
|
+
end
|
636
|
+
end
|
637
|
+
end
|
638
|
+
|
639
|
+
class TLSServer < Coolio::Socket
|
640
|
+
attr_reader :closing
|
641
|
+
attr_writer :close_after_write_complete
|
642
|
+
|
643
|
+
# It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
|
644
|
+
def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
|
645
|
+
raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
|
646
|
+
|
647
|
+
socket_option_setter.call(sock)
|
648
|
+
@_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)
|
649
|
+
@_handler_socket.sync_close = true
|
650
|
+
@_handler_write_buffer = ''.force_encoding('ascii-8bit')
|
651
|
+
@_handler_accepted = false
|
652
|
+
super(@_handler_socket)
|
653
|
+
|
654
|
+
@log = log
|
655
|
+
@under_plugin_development = under_plugin_development
|
656
|
+
|
657
|
+
@connect_callback = connect_callback
|
658
|
+
@data_callback = nil
|
659
|
+
@close_callback = close_callback
|
660
|
+
|
661
|
+
@callback_connection = nil
|
662
|
+
@close_after_write_complete = false
|
663
|
+
@closing = false
|
664
|
+
|
665
|
+
@mutex = Mutex.new # to serialize #write and #close
|
666
|
+
end
|
667
|
+
|
668
|
+
def to_io
|
669
|
+
@_handler_socket.to_io
|
670
|
+
end
|
671
|
+
|
672
|
+
def data(&callback)
|
673
|
+
raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
|
674
|
+
@data_callback = callback
|
675
|
+
on_read_impl = case callback.arity
|
676
|
+
when 1 then :on_read_without_connection
|
677
|
+
when 2 then :on_read_with_connection
|
678
|
+
else
|
679
|
+
raise "BUG: callback block must have 1 or 2 arguments"
|
680
|
+
end
|
681
|
+
self.define_singleton_method(:on_read, method(on_read_impl))
|
682
|
+
end
|
683
|
+
|
684
|
+
def write(data)
|
685
|
+
@mutex.synchronize do
|
686
|
+
@_handler_write_buffer << data
|
687
|
+
schedule_write
|
688
|
+
data.bytesize
|
689
|
+
end
|
690
|
+
end
|
691
|
+
|
692
|
+
if RUBY_VERSION.to_f >= 2.3
|
693
|
+
NONBLOCK_ARG = {exception: false}
|
694
|
+
def try_handshake
|
695
|
+
@_handler_socket.accept_nonblock(NONBLOCK_ARG)
|
696
|
+
end
|
697
|
+
else
|
698
|
+
def try_handshake
|
699
|
+
@_handler_socket.accept_nonblock
|
700
|
+
rescue IO::WaitReadable
|
701
|
+
:wait_readable
|
702
|
+
rescue IO::WaitWritable
|
703
|
+
:wait_writable
|
704
|
+
end
|
705
|
+
end
|
706
|
+
|
707
|
+
def try_tls_accept
|
708
|
+
return true if @_handler_accepted
|
709
|
+
|
710
|
+
begin
|
711
|
+
result = try_handshake # this method call actually try to do handshake via TLS
|
712
|
+
if result == :wait_readable || result == :wait_writable
|
713
|
+
# retry accept_nonblock: there aren't enough data in underlying socket buffer
|
714
|
+
else
|
715
|
+
@_handler_accepted = true
|
716
|
+
|
717
|
+
@callback_connection = TLSCallbackSocket.new(self)
|
718
|
+
@connect_callback.call(@callback_connection)
|
719
|
+
unless @data_callback
|
720
|
+
raise "connection callback must call #data to set data callback"
|
721
|
+
end
|
722
|
+
|
723
|
+
return true
|
724
|
+
end
|
725
|
+
rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
|
726
|
+
@log.trace "unexpected error before accepting TLS connection", error: e
|
727
|
+
close rescue nil
|
728
|
+
end
|
729
|
+
|
730
|
+
false
|
731
|
+
end
|
732
|
+
|
733
|
+
def on_connect
|
734
|
+
try_tls_accept
|
735
|
+
end
|
736
|
+
|
737
|
+
def on_readable
|
738
|
+
if try_tls_accept
|
739
|
+
super
|
740
|
+
end
|
741
|
+
rescue IO::WaitReadable, IO::WaitWritable
|
742
|
+
# ignore and return with doing nothing
|
743
|
+
rescue OpenSSL::SSL::SSLError => e
|
744
|
+
@log.warn "close socket due to unexpected ssl error: #{e}"
|
745
|
+
close rescue nil
|
746
|
+
end
|
747
|
+
|
748
|
+
def on_writable
|
749
|
+
begin
|
750
|
+
@mutex.synchronize do
|
751
|
+
# Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
|
752
|
+
written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
|
753
|
+
@_handler_write_buffer.slice!(0, written_bytes)
|
754
|
+
end
|
755
|
+
|
756
|
+
# No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.
|
757
|
+
# Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.
|
758
|
+
# Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.
|
759
|
+
# If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete
|
760
|
+
# eventually calls TLSServer#close which try to get a lock.
|
761
|
+
super
|
762
|
+
|
763
|
+
close if @close_after_write_complete
|
764
|
+
rescue IO::WaitWritable, IO::WaitReadable
|
765
|
+
return
|
766
|
+
rescue Errno::EINTR
|
767
|
+
return
|
768
|
+
rescue SystemCallError, IOError, SocketError
|
769
|
+
# SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
|
770
|
+
close rescue nil
|
771
|
+
return
|
772
|
+
rescue OpenSSL::SSL::SSLError => e
|
773
|
+
@log.debug "unexpected SSLError while writing data into socket connected via TLS", error: e
|
774
|
+
end
|
775
|
+
end
|
776
|
+
|
777
|
+
def on_read_without_connection(data)
|
778
|
+
@data_callback.call(data)
|
779
|
+
rescue => e
|
780
|
+
@log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
|
781
|
+
@log.error_backtrace
|
782
|
+
close rescue nil
|
783
|
+
raise if @under_plugin_development
|
784
|
+
end
|
785
|
+
|
786
|
+
def on_read_with_connection(data)
|
787
|
+
@data_callback.call(data, @callback_connection)
|
788
|
+
rescue => e
|
789
|
+
@log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
|
790
|
+
@log.error_backtrace
|
791
|
+
close rescue nil
|
792
|
+
raise if @under_plugin_development
|
793
|
+
end
|
794
|
+
|
795
|
+
def close
|
796
|
+
@mutex.synchronize do
|
797
|
+
return if @closing
|
798
|
+
@closing = true
|
799
|
+
@close_callback.call(self)
|
800
|
+
super
|
801
|
+
end
|
802
|
+
end
|
803
|
+
end
|
804
|
+
end
|
805
|
+
end
|
806
|
+
end
|
807
|
+
end
|