fluentd 0.12.40 → 1.6.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +13 -0
- data/.gitignore +5 -0
- data/.gitlab/cicd-template.yaml +10 -0
- data/.gitlab-ci.yml +147 -0
- data/.travis.yml +56 -20
- data/ADOPTERS.md +5 -0
- data/CHANGELOG.md +1369 -0
- data/CONTRIBUTING.md +16 -5
- data/GOVERNANCE.md +55 -0
- data/Gemfile +5 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +7 -0
- data/README.md +23 -11
- data/Rakefile +48 -2
- data/Vagrantfile +17 -0
- data/appveyor.yml +37 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +3 -0
- data/code-of-conduct.md +3 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_dummy_with_compression.conf +23 -0
- data/example/in_forward.conf +7 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +3 -1
- data/example/in_out_forward.conf +17 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_forward.conf +13 -13
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +29 -0
- data/fluentd.gemspec +21 -11
- data/lib/fluent/agent.rb +67 -90
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/ca_generate.rb +181 -0
- data/lib/fluent/command/cat.rb +42 -18
- data/lib/fluent/command/debug.rb +12 -10
- data/lib/fluent/command/fluentd.rb +153 -5
- data/lib/fluent/command/plugin_config_formatter.rb +292 -0
- data/lib/fluent/command/plugin_generator.rb +324 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +718 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/configure_proxy.rb +210 -62
- data/lib/fluent/config/dsl.rb +12 -5
- data/lib/fluent/config/element.rb +107 -9
- data/lib/fluent/config/literal_parser.rb +9 -3
- data/lib/fluent/config/parser.rb +4 -4
- data/lib/fluent/config/section.rb +51 -14
- data/lib/fluent/config/types.rb +28 -13
- data/lib/fluent/config/v1_parser.rb +3 -5
- data/lib/fluent/config.rb +23 -20
- data/lib/fluent/configurable.rb +79 -21
- data/lib/fluent/counter/base_socket.rb +46 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +102 -65
- data/lib/fluent/env.rb +7 -3
- data/lib/fluent/error.rb +30 -0
- data/lib/fluent/event.rb +197 -21
- data/lib/fluent/event_router.rb +93 -10
- data/lib/fluent/filter.rb +2 -50
- data/lib/fluent/formatter.rb +4 -293
- data/lib/fluent/input.rb +2 -32
- data/lib/fluent/label.rb +10 -2
- data/lib/fluent/load.rb +3 -3
- data/lib/fluent/log.rb +348 -81
- data/lib/fluent/match.rb +37 -36
- data/lib/fluent/mixin.rb +12 -176
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +10 -612
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +4 -800
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +192 -0
- data/lib/fluent/plugin/buf_file.rb +128 -174
- data/lib/fluent/plugin/buf_memory.rb +9 -92
- data/lib/fluent/plugin/buffer/chunk.rb +221 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +383 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/buffer.rb +779 -0
- data/lib/fluent/plugin/compressable.rb +92 -0
- data/lib/fluent/plugin/exec_util.rb +3 -108
- data/lib/fluent/plugin/file_util.rb +4 -34
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +117 -34
- data/lib/fluent/plugin/filter_parser.rb +85 -62
- data/lib/fluent/plugin/filter_record_transformer.rb +27 -39
- data/lib/fluent/plugin/filter_stdout.rb +15 -12
- data/lib/fluent/plugin/formatter.rb +50 -0
- data/lib/fluent/plugin/formatter_csv.rb +52 -0
- data/lib/fluent/plugin/formatter_hash.rb +33 -0
- data/lib/fluent/plugin/formatter_json.rb +55 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +51 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +38 -0
- data/lib/fluent/plugin/in_debug_agent.rb +17 -6
- data/lib/fluent/plugin/in_dummy.rb +47 -20
- data/lib/fluent/plugin/in_exec.rb +55 -123
- data/lib/fluent/plugin/in_forward.rb +299 -216
- data/lib/fluent/plugin/in_gc_stat.rb +14 -36
- data/lib/fluent/plugin/in_http.rb +204 -91
- data/lib/fluent/plugin/in_monitor_agent.rb +186 -258
- data/lib/fluent/plugin/in_object_space.rb +13 -41
- data/lib/fluent/plugin/in_syslog.rb +112 -134
- data/lib/fluent/plugin/in_tail.rb +408 -745
- data/lib/fluent/plugin/in_tcp.rb +66 -9
- data/lib/fluent/plugin/in_udp.rb +60 -11
- data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +8 -4
- data/lib/fluent/plugin/input.rb +37 -0
- data/lib/fluent/plugin/multi_output.rb +158 -0
- data/lib/fluent/plugin/out_copy.rb +23 -35
- data/lib/fluent/plugin/out_exec.rb +67 -70
- data/lib/fluent/plugin/out_exec_filter.rb +204 -271
- data/lib/fluent/plugin/out_file.rb +267 -73
- data/lib/fluent/plugin/out_forward.rb +854 -325
- data/lib/fluent/plugin/out_null.rb +42 -9
- data/lib/fluent/plugin/out_relabel.rb +9 -5
- data/lib/fluent/plugin/out_roundrobin.rb +18 -37
- data/lib/fluent/plugin/out_secondary_file.rb +133 -0
- data/lib/fluent/plugin/out_stdout.rb +43 -10
- data/lib/fluent/plugin/out_stream.rb +7 -2
- data/lib/fluent/plugin/output.rb +1498 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +191 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +39 -0
- data/lib/fluent/plugin/parser_json.rb +94 -0
- data/lib/fluent/plugin/parser_ltsv.rb +49 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +106 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +142 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/socket_util.rb +3 -143
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +164 -0
- data/lib/fluent/plugin/string_util.rb +3 -15
- data/lib/fluent/plugin.rb +122 -121
- data/lib/fluent/plugin_helper/cert_option.rb +178 -0
- data/lib/fluent/plugin_helper/child_process.rb +364 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +333 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
- data/lib/fluent/plugin_helper/http_server.rb +76 -0
- data/lib/fluent/plugin_helper/inject.rb +151 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
- data/lib/fluent/plugin_helper/retry_state.rb +205 -0
- data/lib/fluent/plugin_helper/server.rb +807 -0
- data/lib/fluent/plugin_helper/socket.rb +250 -0
- data/lib/fluent/plugin_helper/socket_option.rb +80 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +179 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +73 -0
- data/lib/fluent/plugin_id.rb +80 -0
- data/lib/fluent/process.rb +3 -489
- data/lib/fluent/registry.rb +52 -10
- data/lib/fluent/root_agent.rb +204 -42
- data/lib/fluent/supervisor.rb +597 -359
- data/lib/fluent/system_config.rb +131 -42
- data/lib/fluent/test/base.rb +6 -54
- data/lib/fluent/test/driver/base.rb +224 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +0 -1
- data/lib/fluent/test/formatter_test.rb +4 -1
- data/lib/fluent/test/helpers.rb +58 -10
- data/lib/fluent/test/input_test.rb +27 -19
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +28 -39
- data/lib/fluent/test/parser_test.rb +3 -1
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +33 -1
- data/lib/fluent/time.rb +450 -1
- data/lib/fluent/timezone.rb +27 -3
- data/lib/fluent/{status.rb → unique_id.rb} +15 -24
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +85 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +346 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_fluentd.rb +901 -0
- data/test/command/test_plugin_config_formatter.rb +276 -0
- data/test/command/test_plugin_generator.rb +92 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/test_config_parser.rb +126 -2
- data/test/config/test_configurable.rb +946 -187
- data/test/config/test_configure_proxy.rb +424 -74
- data/test/config/test_dsl.rb +11 -11
- data/test/config/test_element.rb +500 -0
- data/test/config/test_literal_parser.rb +8 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +79 -7
- data/test/config/test_system_config.rb +122 -35
- data/test/config/test_types.rb +38 -0
- data/test/counter/test_client.rb +559 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +89 -6
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +823 -460
- data/test/plugin/test_buf_memory.rb +32 -194
- data/test/plugin/test_buffer.rb +1233 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +844 -0
- data/test/plugin/test_buffer_memory_chunk.rb +338 -0
- data/test/plugin/test_compressable.rb +84 -0
- data/test/plugin/test_filter.rb +357 -0
- data/test/plugin/test_filter_grep.rb +540 -29
- data/test/plugin/test_filter_parser.rb +439 -452
- data/test/plugin/test_filter_record_transformer.rb +123 -166
- data/test/plugin/test_filter_stdout.rb +160 -72
- data/test/plugin/test_formatter_csv.rb +111 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +51 -0
- data/test/plugin/test_formatter_ltsv.rb +62 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_formatter_tsv.rb +68 -0
- data/test/plugin/test_in_debug_agent.rb +24 -1
- data/test/plugin/test_in_dummy.rb +111 -18
- data/test/plugin/test_in_exec.rb +200 -113
- data/test/plugin/test_in_forward.rb +990 -387
- data/test/plugin/test_in_gc_stat.rb +10 -8
- data/test/plugin/test_in_http.rb +600 -224
- data/test/plugin/test_in_monitor_agent.rb +690 -0
- data/test/plugin/test_in_object_space.rb +24 -8
- data/test/plugin/test_in_syslog.rb +154 -215
- data/test/plugin/test_in_tail.rb +1006 -707
- data/test/plugin/test_in_tcp.rb +125 -48
- data/test/plugin/test_in_udp.rb +204 -63
- data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +14 -13
- data/test/plugin/test_input.rb +126 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_copy.rb +117 -112
- data/test/plugin/test_out_exec.rb +258 -53
- data/test/plugin/test_out_exec_filter.rb +538 -115
- data/test/plugin/test_out_file.rb +865 -178
- data/test/plugin/test_out_forward.rb +998 -210
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +36 -29
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +135 -37
- data/test/plugin/test_out_stream.rb +18 -0
- data/test/plugin/test_output.rb +984 -0
- data/test/plugin/test_output_as_buffered.rb +2021 -0
- data/test/plugin/test_output_as_buffered_backup.rb +312 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +911 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +874 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +103 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +145 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +441 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/test_cert_option.rb +16 -0
- data/test/plugin_helper/test_child_process.rb +794 -0
- data/test/plugin_helper/test_compat_parameters.rb +353 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +205 -0
- data/test/plugin_helper/test_inject.rb +519 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +197 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1714 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +132 -0
- data/test/scripts/exec_script.rb +0 -6
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/out_test.rb +23 -15
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +16 -7
- data/test/test_configdsl.rb +2 -2
- data/test/test_event.rb +360 -13
- data/test/test_event_router.rb +108 -11
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +48 -6
- data/test/test_formatter.rb +11 -391
- data/test/test_input.rb +1 -1
- data/test/test_log.rb +591 -31
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +121 -185
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +177 -10
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +101 -0
- data/test/test_process.rb +8 -42
- data/test/test_root_agent.rb +766 -21
- data/test/test_supervisor.rb +481 -0
- data/test/test_test_drivers.rb +135 -0
- data/test/test_time_formatter.rb +282 -0
- data/test/test_time_parser.rb +231 -0
- data/test/test_unique_id.rb +47 -0
- metadata +454 -60
- data/COPYING +0 -14
- data/ChangeLog +0 -666
- data/lib/fluent/buffer.rb +0 -365
- data/lib/fluent/plugin/in_status.rb +0 -76
- data/test/plugin/test_in_status.rb +0 -38
- data/test/test_buffer.rb +0 -624
- data/test/test_parser.rb +0 -1305
@@ -0,0 +1,794 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin_helper/child_process'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
require 'timeout'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
class ChildProcessTest < Test::Unit::TestCase
|
8
|
+
TEST_DEADLOCK_TIMEOUT = 30
|
9
|
+
TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING = 0.1 # This may be shorter than ruby's threading timer, but work well
|
10
|
+
# @nalsh says that ruby's cpu assignments for threads are almost 200ms or so.
|
11
|
+
# Loop interval (expected that it work as specified) should be longer than it.
|
12
|
+
TEST_WAIT_INTERVAL_FOR_LOOP = 0.5
|
13
|
+
|
14
|
+
setup do
|
15
|
+
@d = Dummy.new
|
16
|
+
@d.configure(config_element())
|
17
|
+
@d.start
|
18
|
+
end
|
19
|
+
|
20
|
+
teardown do
|
21
|
+
if @d
|
22
|
+
@d.stop unless @d.stopped?
|
23
|
+
@d.shutdown unless @d.shutdown?
|
24
|
+
@d.close unless @d.closed?
|
25
|
+
@d.terminate unless @d.terminated?
|
26
|
+
@d.log.reset
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
class Dummy < Fluent::Plugin::TestBase
|
31
|
+
helpers :child_process
|
32
|
+
def configure(conf)
|
33
|
+
super
|
34
|
+
@_child_process_kill_timeout = 1
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
test 'can be instantiated' do
|
39
|
+
d1 = Dummy.new
|
40
|
+
assert d1.respond_to?(:_child_process_processes)
|
41
|
+
end
|
42
|
+
|
43
|
+
test 'can be configured and started' do
|
44
|
+
d1 = Dummy.new
|
45
|
+
assert_nothing_raised do
|
46
|
+
d1.configure(config_element())
|
47
|
+
end
|
48
|
+
assert d1.plugin_id
|
49
|
+
assert d1.log
|
50
|
+
|
51
|
+
d1.start
|
52
|
+
end
|
53
|
+
|
54
|
+
test 'can execute external command asyncronously' do
|
55
|
+
m = Mutex.new
|
56
|
+
m.lock
|
57
|
+
ary = []
|
58
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
59
|
+
ran = false
|
60
|
+
@d.child_process_execute(:t0, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
|
61
|
+
m.lock
|
62
|
+
ran = true
|
63
|
+
io.read # discard
|
64
|
+
ary << 2
|
65
|
+
m.unlock
|
66
|
+
end
|
67
|
+
ary << 1
|
68
|
+
m.unlock
|
69
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
70
|
+
m.lock
|
71
|
+
m.unlock
|
72
|
+
end
|
73
|
+
assert_equal [1,2], ary
|
74
|
+
end
|
75
|
+
|
76
|
+
test 'can execute external command at just once, which finishes immediately' do
|
77
|
+
m = Mutex.new
|
78
|
+
t1 = Time.now
|
79
|
+
ary = []
|
80
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
81
|
+
ran = false
|
82
|
+
@d.child_process_execute(:t1, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
|
83
|
+
m.lock
|
84
|
+
ran = true
|
85
|
+
ary << io.read
|
86
|
+
m.unlock
|
87
|
+
end
|
88
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
89
|
+
m.lock
|
90
|
+
m.unlock
|
91
|
+
end
|
92
|
+
assert{ Time.now - t1 < 4.0 }
|
93
|
+
end
|
94
|
+
|
95
|
+
test 'can execute external command at just once, which can handle both of read and write' do
|
96
|
+
m = Mutex.new
|
97
|
+
ary = []
|
98
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
99
|
+
ran = false
|
100
|
+
cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; end'"
|
101
|
+
@d.child_process_execute(:t2, cmd, mode: [:write, :read]) do |writeio, readio|
|
102
|
+
m.lock
|
103
|
+
ran = true
|
104
|
+
|
105
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
106
|
+
writeio.write "my data#{i}\n"
|
107
|
+
writeio.write "my data#{j}\n"
|
108
|
+
writeio.flush
|
109
|
+
end
|
110
|
+
writeio.close
|
111
|
+
|
112
|
+
while line = readio.readline
|
113
|
+
ary << line
|
114
|
+
end
|
115
|
+
m.unlock
|
116
|
+
end
|
117
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
118
|
+
m.lock
|
119
|
+
m.unlock
|
120
|
+
end
|
121
|
+
|
122
|
+
assert_equal [], @d.log.out.logs
|
123
|
+
expected = (1..6).map{|i| "my data#{i}\n" }
|
124
|
+
assert_equal expected, ary
|
125
|
+
end
|
126
|
+
|
127
|
+
test 'can execute external command at just once, which can handle all of read, write and stderr' do
|
128
|
+
m = Mutex.new
|
129
|
+
ary1 = []
|
130
|
+
ary2 = []
|
131
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
132
|
+
ran = false
|
133
|
+
cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; STDERR.puts line.chomp; STDERR.flush rescue nil; end'"
|
134
|
+
@d.child_process_execute(:t2a, cmd, mode: [:write, :read, :stderr]) do |writeio, readio, stderrio|
|
135
|
+
m.lock
|
136
|
+
ran = true
|
137
|
+
|
138
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
139
|
+
writeio.write "my data#{i}\n"
|
140
|
+
writeio.write "my data#{j}\n"
|
141
|
+
writeio.flush
|
142
|
+
end
|
143
|
+
writeio.close
|
144
|
+
|
145
|
+
while (line1 = readio.readline) && (line2 = stderrio.readline)
|
146
|
+
ary1 << line1
|
147
|
+
ary2 << line2
|
148
|
+
end
|
149
|
+
|
150
|
+
m.unlock
|
151
|
+
end
|
152
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
153
|
+
m.lock
|
154
|
+
m.unlock
|
155
|
+
end
|
156
|
+
|
157
|
+
assert_equal [], @d.log.out.logs
|
158
|
+
expected = (1..6).map{|i| "my data#{i}\n" }
|
159
|
+
assert_equal expected, ary1
|
160
|
+
assert_equal expected, ary2
|
161
|
+
end
|
162
|
+
|
163
|
+
test 'can execute external command at just once, which can handle both of write and read (with stderr)' do
|
164
|
+
m = Mutex.new
|
165
|
+
ary = []
|
166
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
167
|
+
ran = false
|
168
|
+
cmd = "ruby"
|
169
|
+
args = ['-e', 'while !STDIN.eof? && line = STDIN.readline; puts "[s]" + line.chomp; STDOUT.flush rescue nil; STDERR.puts "[e]" + line.chomp; STDERR.flush rescue nil; end']
|
170
|
+
@d.child_process_execute(:t2b, cmd, arguments: args, mode: [:write, :read_with_stderr]) do |writeio, readio|
|
171
|
+
m.lock
|
172
|
+
ran = true
|
173
|
+
|
174
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
175
|
+
writeio.write "my data#{i}\n"
|
176
|
+
writeio.write "my data#{j}\n"
|
177
|
+
writeio.flush
|
178
|
+
end
|
179
|
+
writeio.close
|
180
|
+
|
181
|
+
while line = readio.readline
|
182
|
+
ary << line
|
183
|
+
end
|
184
|
+
|
185
|
+
m.unlock
|
186
|
+
end
|
187
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
188
|
+
m.lock
|
189
|
+
m.unlock
|
190
|
+
end
|
191
|
+
|
192
|
+
assert_equal [], @d.log.out.logs
|
193
|
+
expected = (1..6).map{|i| ["[s]my data#{i}\n", "[e]my data#{i}\n"] }.flatten
|
194
|
+
assert_equal expected, ary
|
195
|
+
end
|
196
|
+
|
197
|
+
test 'can execute external command at just once, which runs forever' do
|
198
|
+
m = Mutex.new
|
199
|
+
ary = []
|
200
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
201
|
+
ran = false
|
202
|
+
args = ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush; end"]
|
203
|
+
@d.child_process_execute(:t3, "ruby", arguments: args, mode: [:read]) do |io|
|
204
|
+
m.lock
|
205
|
+
ran = true
|
206
|
+
begin
|
207
|
+
while @d.child_process_running? && line = io.readline
|
208
|
+
ary << line
|
209
|
+
end
|
210
|
+
rescue
|
211
|
+
# ignore
|
212
|
+
ensure
|
213
|
+
m.unlock
|
214
|
+
end
|
215
|
+
end
|
216
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
217
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 10
|
218
|
+
@d.stop # nothing occurs
|
219
|
+
@d.shutdown
|
220
|
+
|
221
|
+
assert{ ary.size > 5 }
|
222
|
+
|
223
|
+
@d.close
|
224
|
+
|
225
|
+
@d.terminate
|
226
|
+
assert @d._child_process_processes.empty?
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
# In windows environment, child_process try KILL at first (because there's no SIGTERM)
|
231
|
+
test 'can execute external command just once, and can terminate it forcedly when shutdown/terminate even if it ignore SIGTERM' do
|
232
|
+
omit "SIGTERM is unavailable on Windows" if Fluent.windows?
|
233
|
+
|
234
|
+
m = Mutex.new
|
235
|
+
ary = []
|
236
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
237
|
+
ran = false
|
238
|
+
@d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
|
239
|
+
m.lock
|
240
|
+
ran = true
|
241
|
+
begin
|
242
|
+
while line = io.readline
|
243
|
+
ary << line
|
244
|
+
end
|
245
|
+
rescue
|
246
|
+
# ignore
|
247
|
+
ensure
|
248
|
+
m.unlock
|
249
|
+
end
|
250
|
+
end
|
251
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
252
|
+
|
253
|
+
assert_equal [], @d.log.out.logs
|
254
|
+
|
255
|
+
@d.stop # nothing occurs
|
256
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
|
257
|
+
lines1 = ary.size
|
258
|
+
assert{ lines1 > 1 }
|
259
|
+
|
260
|
+
pid = @d._child_process_processes.keys.first
|
261
|
+
|
262
|
+
@d.shutdown
|
263
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
|
264
|
+
lines2 = ary.size
|
265
|
+
assert{ lines2 > lines1 }
|
266
|
+
|
267
|
+
@d.close
|
268
|
+
|
269
|
+
assert_nil((Process.waitpid(pid, Process::WNOHANG) rescue nil))
|
270
|
+
|
271
|
+
@d.terminate
|
272
|
+
assert @d._child_process_processes.empty?
|
273
|
+
begin
|
274
|
+
Process.waitpid(pid)
|
275
|
+
rescue Errno::ECHILD
|
276
|
+
end
|
277
|
+
# Process successfully KILLed if test reaches here
|
278
|
+
assert true
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
test 'can execute external command many times, which finishes immediately' do
|
283
|
+
ary = []
|
284
|
+
arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 3
|
285
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
286
|
+
@d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 5, mode: [:read]) do |io|
|
287
|
+
ary << io.read.split("\n").map(&:chomp).join
|
288
|
+
end
|
289
|
+
sleep 13 # 5sec * 2 + 3sec
|
290
|
+
assert_equal [], @d.log.out.logs
|
291
|
+
@d.stop
|
292
|
+
assert_equal [], @d.log.out.logs
|
293
|
+
@d.shutdown; @d.close; @d.terminate
|
294
|
+
assert_equal 2, ary.size
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
test 'can execute external command many times, with leading once executed immediately' do
|
299
|
+
ary = []
|
300
|
+
arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"]
|
301
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
302
|
+
@d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 5, immediate: true, mode: [:read]) do |io|
|
303
|
+
ary << io.read.split("\n").map(&:chomp).join
|
304
|
+
end
|
305
|
+
sleep 8 # 5sec * 1 + 3sec
|
306
|
+
# but expected lines are same with test above
|
307
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
308
|
+
assert_equal 2, ary.size
|
309
|
+
assert_equal [], @d.log.out.logs
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
test 'does not execute long running external command in parallel in default' do
|
314
|
+
ary = []
|
315
|
+
arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10
|
316
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
317
|
+
assert_equal [], @d.log.out.logs
|
318
|
+
@d.log.out.singleton_class.module_eval do
|
319
|
+
define_method(:write){|message|
|
320
|
+
raise "boo" if message.include?('test: {"test":"test"}') || message.include?('test: {"test"=>"test"}')
|
321
|
+
@logs.push message
|
322
|
+
}
|
323
|
+
end
|
324
|
+
|
325
|
+
@d.child_process_execute(:t7, "ruby", arguments: arguments, interval: 2, immediate: true, mode: [:read]) do |io|
|
326
|
+
ary << io.read.split("\n").map(&:chomp).join
|
327
|
+
end
|
328
|
+
sleep 4
|
329
|
+
assert_equal 1, @d._child_process_processes.size
|
330
|
+
@d.stop
|
331
|
+
warn_msg = '[warn]: previous child process is still running. skipped. title=:t7 command="ruby" arguments=["-e", "10.times{ puts \'okay\'; STDOUT.flush rescue nil; sleep 0.5 }"] interval=2 parallel=false' + "\n"
|
332
|
+
logs = @d.log.out.logs
|
333
|
+
assert{ logs.first.end_with?(warn_msg) }
|
334
|
+
assert{ logs.all?{|line| line.end_with?(warn_msg) } }
|
335
|
+
@d.shutdown; @d.close; @d.terminate
|
336
|
+
assert_equal [], @d.log.out.logs
|
337
|
+
end
|
338
|
+
end
|
339
|
+
|
340
|
+
test 'can execute long running external command in parallel if specified' do
|
341
|
+
ary = []
|
342
|
+
arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10 sec
|
343
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
344
|
+
@d.child_process_execute(:t8, "ruby", arguments: arguments, interval: 1, immediate: true, parallel: true, mode: [:read]) do |io|
|
345
|
+
ary << io.read.split("\n").map(&:chomp).join
|
346
|
+
end
|
347
|
+
sleep 4
|
348
|
+
processes = @d._child_process_processes.size
|
349
|
+
assert{ processes >= 3 && processes <= 5 }
|
350
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
351
|
+
assert_equal [], @d.log.out.logs
|
352
|
+
end
|
353
|
+
end
|
354
|
+
|
355
|
+
test 'execute external processes only for writing' do
|
356
|
+
m = Mutex.new
|
357
|
+
unreadable = false
|
358
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
359
|
+
ran = false
|
360
|
+
@d.child_process_execute(:t9, "ruby", arguments: ['-e', 'a=""; while b=STDIN.readline; a+=b; end'], mode: [:write]) do |io|
|
361
|
+
m.lock
|
362
|
+
ran = true
|
363
|
+
begin
|
364
|
+
io.read
|
365
|
+
rescue IOError
|
366
|
+
unreadable = true
|
367
|
+
end
|
368
|
+
50.times do
|
369
|
+
io.write "hahaha\n"
|
370
|
+
end
|
371
|
+
m.unlock
|
372
|
+
end
|
373
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
374
|
+
m.lock
|
375
|
+
m.unlock
|
376
|
+
assert unreadable
|
377
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
378
|
+
assert_equal [], @d.log.out.logs
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
|
+
test 'execute external processes only for reading' do
|
383
|
+
m = Mutex.new
|
384
|
+
unwritable = false
|
385
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
386
|
+
ran = false
|
387
|
+
@d.child_process_execute(:t10, "ruby", arguments: ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end"], mode: [:read]) do |io|
|
388
|
+
m.lock
|
389
|
+
ran = true
|
390
|
+
begin
|
391
|
+
io.write "foobar"
|
392
|
+
rescue IOError
|
393
|
+
unwritable = true
|
394
|
+
end
|
395
|
+
_data = io.readline
|
396
|
+
m.unlock
|
397
|
+
end
|
398
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
399
|
+
m.lock
|
400
|
+
m.unlock
|
401
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
402
|
+
assert unwritable
|
403
|
+
assert_equal [], @d.log.out.logs
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
test 'can control external encodings' do
|
408
|
+
m = Mutex.new
|
409
|
+
encodings = []
|
410
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
411
|
+
ran = false
|
412
|
+
@d.child_process_execute(:t11, "ruby -e 'sleep 10'", external_encoding: 'ascii-8bit') do |r, w|
|
413
|
+
m.lock
|
414
|
+
ran = true
|
415
|
+
encodings << r.external_encoding
|
416
|
+
encodings << w.external_encoding
|
417
|
+
m.unlock
|
418
|
+
end
|
419
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
420
|
+
m.lock
|
421
|
+
assert_equal Encoding::ASCII_8BIT, encodings[0]
|
422
|
+
assert_equal Encoding::ASCII_8BIT, encodings[1]
|
423
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
424
|
+
end
|
425
|
+
end
|
426
|
+
|
427
|
+
test 'can control internal encodings' do
|
428
|
+
m = Mutex.new
|
429
|
+
encodings = []
|
430
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
431
|
+
ran = false
|
432
|
+
@d.child_process_execute(:t12, "ruby -e 'sleep 10'", external_encoding: 'utf-8', internal_encoding: 'ascii-8bit') do |r, w|
|
433
|
+
m.lock
|
434
|
+
ran = true
|
435
|
+
encodings << r.internal_encoding
|
436
|
+
encodings << w.internal_encoding
|
437
|
+
m.unlock
|
438
|
+
end
|
439
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
440
|
+
m.lock
|
441
|
+
assert_equal Encoding::ASCII_8BIT, encodings[0]
|
442
|
+
assert_equal Encoding::ASCII_8BIT, encodings[1]
|
443
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
444
|
+
end
|
445
|
+
end
|
446
|
+
|
447
|
+
test 'can convert encodings from ascii-8bit to utf-8' do
|
448
|
+
m = Mutex.new
|
449
|
+
str = nil
|
450
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
451
|
+
ran = false
|
452
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xA4\xB5\xA4\xC8\xA4\xB7"']
|
453
|
+
@d.child_process_execute(:t13, "ruby", arguments: args, external_encoding: 'euc-jp', internal_encoding: 'windows-31j', mode: [:read]) do |io|
|
454
|
+
m.lock
|
455
|
+
ran = true
|
456
|
+
str = io.read
|
457
|
+
m.unlock
|
458
|
+
end
|
459
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
460
|
+
m.lock
|
461
|
+
assert_equal Encoding.find('windows-31j'), str.encoding
|
462
|
+
expected = "さとし".encode('windows-31j')
|
463
|
+
assert_equal expected, str
|
464
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
test 'can scrub characters without exceptions' do
|
469
|
+
m = Mutex.new
|
470
|
+
str = nil
|
471
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
472
|
+
ran = false
|
473
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
474
|
+
@d.child_process_execute(:t13a, "ruby", arguments: args, mode: [:read]) do |io|
|
475
|
+
m.lock
|
476
|
+
ran = true
|
477
|
+
str = io.read
|
478
|
+
m.unlock
|
479
|
+
end
|
480
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
481
|
+
m.lock
|
482
|
+
assert_equal Encoding.find('utf-8'), str.encoding
|
483
|
+
expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
|
484
|
+
assert_equal expected, str
|
485
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
486
|
+
end
|
487
|
+
end
|
488
|
+
|
489
|
+
test 'can scrub characters without exceptions and replace specified chars' do
|
490
|
+
m = Mutex.new
|
491
|
+
str = nil
|
492
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
493
|
+
ran = false
|
494
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
495
|
+
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
|
496
|
+
m.lock
|
497
|
+
ran = true
|
498
|
+
str = io.read
|
499
|
+
m.unlock
|
500
|
+
end
|
501
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
502
|
+
m.lock
|
503
|
+
assert_equal Encoding.find('utf-8'), str.encoding
|
504
|
+
expected = "??\x00??".force_encoding("utf-8")
|
505
|
+
assert_equal expected, str
|
506
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
unless Fluent.windows?
|
511
|
+
test 'can specify subprocess name' do
|
512
|
+
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
513
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
|
514
|
+
Process.kill(:TERM, io.pid) rescue nil
|
515
|
+
io.close rescue nil
|
516
|
+
|
517
|
+
# Does TravisCI prohibit process renaming?
|
518
|
+
# This test will be passed in such environment
|
519
|
+
pend unless process_naming_enabled
|
520
|
+
|
521
|
+
m = Mutex.new
|
522
|
+
pids = []
|
523
|
+
proc_lines = []
|
524
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
525
|
+
ran = false
|
526
|
+
@d.child_process_execute(:t14, "ruby", arguments:['-e', 'sleep 10; puts "hello"'], subprocess_name: "sleeeeeeeeeper", mode: [:read]) do |readio|
|
527
|
+
m.lock
|
528
|
+
ran = true
|
529
|
+
pids << @d.child_process_id
|
530
|
+
proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
|
531
|
+
m.unlock
|
532
|
+
readio.read
|
533
|
+
end
|
534
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
535
|
+
m.lock
|
536
|
+
pid = pids.first
|
537
|
+
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
538
|
+
assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
539
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
|
544
|
+
test 'can set ENV variables' do
|
545
|
+
m = Mutex.new
|
546
|
+
str = nil
|
547
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
548
|
+
ran = false
|
549
|
+
args = ['-e', 'puts ENV["testing_child_process"]']
|
550
|
+
@d.child_process_execute(:t15a, "ruby", arguments: args, mode: [:read], env: {'testing_child_process' => 'Yes! True!'}) do |io|
|
551
|
+
m.lock
|
552
|
+
ran = true
|
553
|
+
str = io.read
|
554
|
+
m.unlock
|
555
|
+
end
|
556
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
557
|
+
m.lock
|
558
|
+
expected = "Yes! True!\n"
|
559
|
+
assert_equal expected, str
|
560
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
561
|
+
end
|
562
|
+
end
|
563
|
+
|
564
|
+
test 'can unset ENV variables of Fluentd process' do
|
565
|
+
m = Mutex.new
|
566
|
+
str = nil
|
567
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
568
|
+
current_env_path = ENV['PATH']
|
569
|
+
ran = false
|
570
|
+
args = ['-e', 'puts ENV["testing_child_process1"].to_s + ENV["testing_child_process2"].to_s']
|
571
|
+
ENV['testing_child_process1'] = "No! False!"
|
572
|
+
@d.child_process_execute(:t15b, "ruby", arguments: args, mode: [:read], unsetenv: true, env: {'testing_child_process2' => 'Yes! True!', 'PATH' => current_env_path}) do |io|
|
573
|
+
m.lock
|
574
|
+
ran = true
|
575
|
+
str = io.read
|
576
|
+
m.unlock
|
577
|
+
end
|
578
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
579
|
+
m.lock
|
580
|
+
expected = "Yes! True!\n"
|
581
|
+
assert_equal expected, str
|
582
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
583
|
+
end
|
584
|
+
end
|
585
|
+
|
586
|
+
unless Fluent.windows?
|
587
|
+
test 'can change working directory' do
|
588
|
+
# check my real /tmp directory (for mac)
|
589
|
+
cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
|
590
|
+
mytmpdir = open(cmd){|io| io.read.chomp }
|
591
|
+
|
592
|
+
m = Mutex.new
|
593
|
+
str = nil
|
594
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
595
|
+
ran = false
|
596
|
+
args = ['-e', 'puts Dir.pwd']
|
597
|
+
@d.child_process_execute(:t16, "ruby", arguments: args, mode: [:read], chdir: "/tmp") do |io|
|
598
|
+
m.lock
|
599
|
+
ran = true
|
600
|
+
str = io.read.chomp
|
601
|
+
m.unlock
|
602
|
+
end
|
603
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
604
|
+
m.lock
|
605
|
+
assert_equal mytmpdir, str
|
606
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
607
|
+
end
|
608
|
+
end
|
609
|
+
end
|
610
|
+
|
611
|
+
sub_test_case 'on_exit_callback is specified' do
|
612
|
+
setup do
|
613
|
+
@temp = Tempfile.create("child_process_wait_with_on_exit_callback")
|
614
|
+
@temp_path = @temp.path
|
615
|
+
@temp.close
|
616
|
+
end
|
617
|
+
|
618
|
+
teardown do
|
619
|
+
File.unlink @temp_path if File.exist?(@temp_path)
|
620
|
+
end
|
621
|
+
|
622
|
+
test 'can return exit status for child process successfully exits using on_exit_callback' do
|
623
|
+
assert File.exist?(@temp_path)
|
624
|
+
|
625
|
+
block_exits = false
|
626
|
+
callback_called = false
|
627
|
+
exit_status = nil
|
628
|
+
args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '1', @temp_path]
|
629
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
630
|
+
|
631
|
+
str = nil
|
632
|
+
|
633
|
+
pid = nil
|
634
|
+
@d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
|
635
|
+
pid = @d.instance_eval{ child_process_id }
|
636
|
+
str = readio.read.chomp
|
637
|
+
block_exits = true
|
638
|
+
end
|
639
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
|
640
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
|
641
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
|
642
|
+
|
643
|
+
assert callback_called
|
644
|
+
assert exit_status
|
645
|
+
assert_equal 0, exit_status.exitstatus
|
646
|
+
assert !File.exist?(@temp_path)
|
647
|
+
|
648
|
+
assert_equal "yay", str
|
649
|
+
end
|
650
|
+
|
651
|
+
test 'can return exit status with signal code for child process killed by signal using on_exit_callback' do
|
652
|
+
omit "SIGQUIT is unsupported on Windows" if Fluent.windows?
|
653
|
+
|
654
|
+
assert File.exist?(@temp_path)
|
655
|
+
|
656
|
+
block_exits = false
|
657
|
+
callback_called = false
|
658
|
+
exit_status = nil
|
659
|
+
args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '100', @temp_path]
|
660
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
661
|
+
|
662
|
+
str = nil
|
663
|
+
|
664
|
+
pid = nil
|
665
|
+
@d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
|
666
|
+
pid = @d.instance_eval{ child_process_id }
|
667
|
+
sleep 10 # to run child process correctly
|
668
|
+
Process.kill(:QUIT, pid)
|
669
|
+
sleep 1
|
670
|
+
Process.kill(:QUIT, pid) rescue nil # once more to send kill
|
671
|
+
sleep 1
|
672
|
+
Process.kill(:QUIT, pid) rescue nil # just like sync
|
673
|
+
str = readio.read.chomp rescue nil # empty string before EOF
|
674
|
+
block_exits = true
|
675
|
+
end
|
676
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
|
677
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
|
678
|
+
waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
|
679
|
+
|
680
|
+
assert callback_called
|
681
|
+
assert exit_status
|
682
|
+
|
683
|
+
# This test sometimes fails on TravisCI
|
684
|
+
# with [nil, 11] # SIGSEGV
|
685
|
+
# or with [1, nil] # ???
|
686
|
+
assert_equal [nil, 3, true, ""], [exit_status.exitstatus, exit_status.termsig, File.exist?(@temp_path), str] # SIGQUIT
|
687
|
+
# SIGSEGV looks a kind of BUG of ruby...
|
688
|
+
end
|
689
|
+
|
690
|
+
test 'calls on_exit_callback for each process exits for interval call using on_exit_callback' do
|
691
|
+
read_data_list = []
|
692
|
+
exit_status_list = []
|
693
|
+
|
694
|
+
args = ['-e', 'puts "yay"', '1']
|
695
|
+
cb = ->(status){ exit_status_list << status }
|
696
|
+
|
697
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
698
|
+
@d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 2, mode: [:read], on_exit_callback: cb) do |readio|
|
699
|
+
read_data_list << readio.read.chomp
|
700
|
+
end
|
701
|
+
sleep 10
|
702
|
+
end
|
703
|
+
|
704
|
+
assert{ read_data_list.size >= 3 }
|
705
|
+
assert{ exit_status_list.size >= 3 }
|
706
|
+
end
|
707
|
+
|
708
|
+
test 'waits lasting child process until wait_timeout if block is not specified' do
|
709
|
+
assert File.exist?(@temp_path)
|
710
|
+
|
711
|
+
callback_called = false
|
712
|
+
exit_status = nil
|
713
|
+
args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '1', @temp_path]
|
714
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
715
|
+
|
716
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
717
|
+
@d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 5)
|
718
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
|
719
|
+
end
|
720
|
+
|
721
|
+
assert callback_called
|
722
|
+
assert exit_status
|
723
|
+
assert_equal 0, exit_status.exitstatus
|
724
|
+
assert !File.exist?(@temp_path)
|
725
|
+
end
|
726
|
+
|
727
|
+
test 'waits lasting child process until wait_timeout after block rans if block is specified' do
|
728
|
+
assert File.exist?(@temp_path)
|
729
|
+
|
730
|
+
callback_called = false
|
731
|
+
exit_status = nil
|
732
|
+
args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '3', @temp_path]
|
733
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
734
|
+
|
735
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
736
|
+
@d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 10) do
|
737
|
+
sleep 1
|
738
|
+
end
|
739
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
|
740
|
+
end
|
741
|
+
|
742
|
+
assert callback_called
|
743
|
+
assert exit_status
|
744
|
+
assert_equal 0, exit_status.exitstatus
|
745
|
+
assert !File.exist?(@temp_path)
|
746
|
+
end
|
747
|
+
|
748
|
+
test 'kills lasting child process after wait_timeout if block is not specified' do
|
749
|
+
assert File.exist?(@temp_path)
|
750
|
+
|
751
|
+
callback_called = false
|
752
|
+
exit_status = nil
|
753
|
+
args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
|
754
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
755
|
+
|
756
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
757
|
+
@d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 3)
|
758
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
|
759
|
+
end
|
760
|
+
|
761
|
+
assert callback_called
|
762
|
+
assert exit_status
|
763
|
+
unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
|
764
|
+
assert_nil exit_status.exitstatus
|
765
|
+
assert_equal 9, exit_status.termsig # SIGKILL
|
766
|
+
end
|
767
|
+
assert File.exist?(@temp_path)
|
768
|
+
end
|
769
|
+
|
770
|
+
test 'kills lasting child process after block ran and wait_timeout expires if block is specified' do
|
771
|
+
assert File.exist?(@temp_path)
|
772
|
+
|
773
|
+
callback_called = false
|
774
|
+
exit_status = nil
|
775
|
+
args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
|
776
|
+
cb = ->(status){ exit_status = status; callback_called = true }
|
777
|
+
|
778
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
779
|
+
@d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 3) do
|
780
|
+
sleep 3
|
781
|
+
end
|
782
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
|
783
|
+
end
|
784
|
+
|
785
|
+
assert callback_called
|
786
|
+
assert exit_status
|
787
|
+
unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
|
788
|
+
assert_nil exit_status.exitstatus
|
789
|
+
assert_equal 9, exit_status.termsig # SIGKILL
|
790
|
+
end
|
791
|
+
assert File.exist?(@temp_path)
|
792
|
+
end
|
793
|
+
end
|
794
|
+
end
|