fluentd-hubspot 0.14.14.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +28 -0
- data/.travis.yml +51 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +42 -0
- data/COPYING +14 -0
- data/ChangeLog +593 -0
- data/Gemfile +9 -0
- data/README.md +76 -0
- data/Rakefile +74 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +43 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/code-of-conduct.md +3 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_dummy_with_compression.conf +23 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/secondary_file.conf +41 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +163 -0
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/binlog_reader.rb +234 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +330 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +301 -0
- data/lib/fluent/command/plugin_config_formatter.rb +258 -0
- data/lib/fluent/command/plugin_generator.rb +301 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/detach_process_mixin.rb +25 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +59 -0
- data/lib/fluent/compat/output.rb +728 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +418 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +223 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +200 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +266 -0
- data/lib/fluent/env.rb +28 -0
- data/lib/fluent/error.rb +30 -0
- data/lib/fluent/event.rb +334 -0
- data/lib/fluent/event_router.rb +269 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +35 -0
- data/lib/fluent/log.rb +546 -0
- data/lib/fluent/match.rb +178 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +183 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +165 -0
- data/lib/fluent/plugin/buf_file.rb +184 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +617 -0
- data/lib/fluent/plugin/buffer/chunk.rb +221 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +364 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/compressable.rb +92 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_parser.rb +119 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +322 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +50 -0
- data/lib/fluent/plugin/formatter_csv.rb +52 -0
- data/lib/fluent/plugin/formatter_hash.rb +33 -0
- data/lib/fluent/plugin/formatter_json.rb +55 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +51 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +75 -0
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +139 -0
- data/lib/fluent/plugin/in_exec.rb +108 -0
- data/lib/fluent/plugin/in_forward.rb +455 -0
- data/lib/fluent/plugin/in_gc_stat.rb +56 -0
- data/lib/fluent/plugin/in_http.rb +433 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +448 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_syslog.rb +209 -0
- data/lib/fluent/plugin/in_tail.rb +905 -0
- data/lib/fluent/plugin/in_tcp.rb +85 -0
- data/lib/fluent/plugin/in_udp.rb +81 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +37 -0
- data/lib/fluent/plugin/multi_output.rb +157 -0
- data/lib/fluent/plugin/out_copy.rb +46 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +317 -0
- data/lib/fluent/plugin/out_file.rb +302 -0
- data/lib/fluent/plugin/out_forward.rb +912 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +133 -0
- data/lib/fluent/plugin/out_stdout.rb +75 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1291 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +191 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +39 -0
- data/lib/fluent/plugin/parser_json.rb +81 -0
- data/lib/fluent/plugin/parser_ltsv.rb +42 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +105 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +63 -0
- data/lib/fluent/plugin/parser_syslog.rb +121 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +159 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +70 -0
- data/lib/fluent/plugin_helper/cert_option.rb +159 -0
- data/lib/fluent/plugin_helper/child_process.rb +364 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +331 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +93 -0
- data/lib/fluent/plugin_helper/event_loop.rb +161 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/inject.rb +151 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +201 -0
- data/lib/fluent/plugin_helper/server.rb +738 -0
- data/lib/fluent/plugin_helper/socket.rb +241 -0
- data/lib/fluent/plugin_helper/socket_option.rb +69 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +179 -0
- data/lib/fluent/plugin_helper/timer.rb +91 -0
- data/lib/fluent/plugin_id.rb +80 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +323 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +741 -0
- data/lib/fluent/system_config.rb +159 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +224 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/time.rb +412 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +346 -0
- data/test/command/test_fluentd.rb +618 -0
- data/test/command/test_plugin_config_formatter.rb +275 -0
- data/test/command/test_plugin_generator.rb +66 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +513 -0
- data/test/config/test_configurable.rb +1587 -0
- data/test/config/test_configure_proxy.rb +566 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +168 -0
- data/test/config/test_types.rb +191 -0
- data/test/helper.rb +153 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +843 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1220 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +844 -0
- data/test/plugin/test_buffer_memory_chunk.rb +338 -0
- data/test/plugin/test_compressable.rb +84 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +357 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_parser.rb +700 -0
- data/test/plugin/test_filter_record_transformer.rb +556 -0
- data/test/plugin/test_filter_stdout.rb +202 -0
- data/test/plugin/test_formatter_csv.rb +111 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +51 -0
- data/test/plugin/test_formatter_ltsv.rb +59 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +192 -0
- data/test/plugin/test_in_exec.rb +245 -0
- data/test/plugin/test_in_forward.rb +1120 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +588 -0
- data/test/plugin/test_in_monitor_agent.rb +516 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +271 -0
- data/test/plugin/test_in_tail.rb +1216 -0
- data/test/plugin/test_in_tcp.rb +118 -0
- data/test/plugin/test_in_udp.rb +152 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +126 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +310 -0
- data/test/plugin/test_out_exec_filter.rb +613 -0
- data/test/plugin/test_out_file.rb +873 -0
- data/test/plugin/test_out_forward.rb +685 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +442 -0
- data/test/plugin/test_out_stdout.rb +170 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +870 -0
- data/test/plugin/test_output_as_buffered.rb +1932 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +877 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +46 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +103 -0
- data/test/plugin/test_parser_json.rb +114 -0
- data/test/plugin/test_parser_labeled_tsv.rb +128 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +281 -0
- data/test/plugin/test_parser_syslog.rb +242 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +794 -0
- data/test/plugin_helper/test_compat_parameters.rb +331 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_inject.rb +519 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_retry_state.rb +422 -0
- data/test/plugin_helper/test_server.rb +1677 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +132 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +515 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +186 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +312 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +828 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +273 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +253 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +101 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +611 -0
- data/test/test_supervisor.rb +373 -0
- data/test/test_test_drivers.rb +135 -0
- data/test/test_time_formatter.rb +282 -0
- data/test/test_time_parser.rb +211 -0
- data/test/test_unique_id.rb +47 -0
- metadata +898 -0
@@ -0,0 +1,170 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_stdout'
|
4
|
+
|
5
|
+
class StdoutOutputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
]
|
12
|
+
TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
|
13
|
+
|
14
|
+
def create_driver(conf = CONFIG)
|
15
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
sub_test_case 'non-buffered' do
|
19
|
+
test 'configure' do
|
20
|
+
d = create_driver
|
21
|
+
assert_equal 1, d.instance.formatter_configs.size # init: true
|
22
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
23
|
+
assert_equal 'json', d.instance.formatter.output_type
|
24
|
+
end
|
25
|
+
|
26
|
+
test 'configure output_type' do
|
27
|
+
d = create_driver(CONFIG + "\noutput_type json")
|
28
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
29
|
+
assert_equal 'json', d.instance.formatter.output_type
|
30
|
+
|
31
|
+
d = create_driver(CONFIG + "\noutput_type hash")
|
32
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
33
|
+
assert_equal 'hash', d.instance.formatter.output_type
|
34
|
+
|
35
|
+
assert_raise(Fluent::ConfigError) do
|
36
|
+
d = create_driver(CONFIG + "\noutput_type foo")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
test 'emit with default configuration' do
|
41
|
+
d = create_driver
|
42
|
+
time = event_time()
|
43
|
+
out = capture_log do
|
44
|
+
d.run(default_tag: 'test') do
|
45
|
+
d.feed(time, {'test' => 'test1'})
|
46
|
+
end
|
47
|
+
end
|
48
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
|
49
|
+
end
|
50
|
+
|
51
|
+
data('oj' => 'oj', 'yajl' => 'yajl')
|
52
|
+
test 'emit in json format' do |data|
|
53
|
+
d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
|
54
|
+
time = event_time()
|
55
|
+
out = capture_log do
|
56
|
+
d.run(default_tag: 'test') do
|
57
|
+
d.feed(time, {'test' => 'test1'})
|
58
|
+
end
|
59
|
+
end
|
60
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
|
61
|
+
|
62
|
+
if data == 'yajl'
|
63
|
+
# NOTE: Float::NAN is not jsonable
|
64
|
+
assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
|
65
|
+
else
|
66
|
+
out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
|
67
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":NaN}\n", out
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
test 'emit in hash format' do
|
72
|
+
d = create_driver(CONFIG + "\noutput_type hash")
|
73
|
+
time = event_time()
|
74
|
+
out = capture_log do
|
75
|
+
d.run(default_tag: 'test') do
|
76
|
+
d.feed(time, {'test' => 'test2'})
|
77
|
+
end
|
78
|
+
end
|
79
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out
|
80
|
+
|
81
|
+
# NOTE: Float::NAN is not jsonable, but hash string can output it.
|
82
|
+
out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
|
83
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
sub_test_case 'buffered' do
|
88
|
+
test 'configure' do
|
89
|
+
d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
|
90
|
+
assert_equal 1, d.instance.formatter_configs.size
|
91
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
92
|
+
assert_equal 'json', d.instance.formatter.output_type
|
93
|
+
assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
|
94
|
+
assert d.instance.buffer_config.flush_at_shutdown
|
95
|
+
assert_equal ['tag'], d.instance.buffer_config.chunk_keys
|
96
|
+
assert d.instance.chunk_key_tag
|
97
|
+
assert !d.instance.chunk_key_time
|
98
|
+
assert_equal [], d.instance.chunk_keys
|
99
|
+
end
|
100
|
+
|
101
|
+
test 'configure with output_type' do
|
102
|
+
d = create_driver(config_element("ROOT", "", {"output_type" => "json"}, [config_element("buffer")]))
|
103
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
104
|
+
assert_equal 'json', d.instance.formatter.output_type
|
105
|
+
|
106
|
+
d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
|
107
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
108
|
+
assert_equal 'hash', d.instance.formatter.output_type
|
109
|
+
|
110
|
+
assert_raise(Fluent::ConfigError) do
|
111
|
+
create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
sub_test_case "emit with default config" do
|
116
|
+
test '#write(synchronous)' do
|
117
|
+
d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
|
118
|
+
time = event_time()
|
119
|
+
|
120
|
+
out = capture_log do
|
121
|
+
d.run(default_tag: 'test', flush: true) do
|
122
|
+
d.feed(time, {'test' => 'test'})
|
123
|
+
end
|
124
|
+
end
|
125
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
sub_test_case "emit json" do
|
130
|
+
data('oj' => 'oj', 'yajl' => 'yajl')
|
131
|
+
test '#write(synchronous)' do |data|
|
132
|
+
d = create_driver(config_element("ROOT", "", {"output_type" => "json", "json_parser" => data}, [config_element("buffer")]))
|
133
|
+
time = event_time()
|
134
|
+
|
135
|
+
out = capture_log do
|
136
|
+
d.run(default_tag: 'test', flush: true) do
|
137
|
+
d.feed(time, {'test' => 'test'})
|
138
|
+
end
|
139
|
+
end
|
140
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
sub_test_case 'emit hash' do
|
145
|
+
test '#write(synchronous)' do
|
146
|
+
d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
|
147
|
+
time = event_time()
|
148
|
+
|
149
|
+
out = capture_log do
|
150
|
+
d.run(default_tag: 'test', flush: true) do
|
151
|
+
d.feed(time, {'test' => 'test'})
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# Capture the log output of the block given
|
161
|
+
def capture_log(&block)
|
162
|
+
tmp = $log
|
163
|
+
$log = StringIO.new
|
164
|
+
yield
|
165
|
+
return $log.string
|
166
|
+
ensure
|
167
|
+
$log = tmp
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test'
|
3
|
+
require 'fluent/plugin/out_stream'
|
4
|
+
|
5
|
+
module StreamOutputTest
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_write
|
11
|
+
d = create_driver
|
12
|
+
|
13
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
14
|
+
d.emit({"a"=>1}, time)
|
15
|
+
d.emit({"a"=>2}, time)
|
16
|
+
|
17
|
+
expect = ["test",
|
18
|
+
[time,{"a"=>1}].to_msgpack +
|
19
|
+
[time,{"a"=>2}].to_msgpack
|
20
|
+
].to_msgpack
|
21
|
+
|
22
|
+
result = d.run
|
23
|
+
assert_equal(expect, result)
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_write_event_time
|
27
|
+
d = create_driver
|
28
|
+
|
29
|
+
time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
|
30
|
+
d.emit({"a"=>1}, time)
|
31
|
+
d.emit({"a"=>2}, time)
|
32
|
+
|
33
|
+
expect = ["test",
|
34
|
+
Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>1}]).to_s +
|
35
|
+
Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>2}]).to_s
|
36
|
+
]
|
37
|
+
expect = Fluent::Engine.msgpack_factory.packer.write(expect).to_s
|
38
|
+
|
39
|
+
result = d.run
|
40
|
+
assert_equal(expect, result)
|
41
|
+
end
|
42
|
+
|
43
|
+
def create_driver(klass, conf)
|
44
|
+
Fluent::Test::BufferedOutputTestDriver.new(klass) do
|
45
|
+
def write(chunk)
|
46
|
+
chunk.read
|
47
|
+
end
|
48
|
+
end.configure(conf)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
class TcpOutputTest < Test::Unit::TestCase
|
53
|
+
include StreamOutputTest
|
54
|
+
|
55
|
+
PORT = unused_port
|
56
|
+
CONFIG = %[
|
57
|
+
port #{PORT}
|
58
|
+
host 127.0.0.1
|
59
|
+
send_timeout 51
|
60
|
+
]
|
61
|
+
|
62
|
+
def create_driver(conf=CONFIG)
|
63
|
+
super(Fluent::TcpOutput, conf)
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_configure
|
67
|
+
d = create_driver
|
68
|
+
assert_equal PORT, d.instance.port
|
69
|
+
assert_equal '127.0.0.1', d.instance.host
|
70
|
+
assert_equal 51, d.instance.send_timeout
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class UnixOutputTest < Test::Unit::TestCase
|
75
|
+
include StreamOutputTest
|
76
|
+
|
77
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_unix#{ENV['TEST_ENV_NUMBER']}"
|
78
|
+
CONFIG = %[
|
79
|
+
path #{TMP_DIR}/unix
|
80
|
+
send_timeout 52
|
81
|
+
]
|
82
|
+
|
83
|
+
def create_driver(conf=CONFIG)
|
84
|
+
super(Fluent::UnixOutput, conf)
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_configure
|
88
|
+
d = create_driver
|
89
|
+
assert_equal "#{TMP_DIR}/unix", d.instance.path
|
90
|
+
assert_equal 52, d.instance.send_timeout
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
@@ -0,0 +1,870 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/output'
|
3
|
+
require 'fluent/plugin/buffer'
|
4
|
+
require 'fluent/event'
|
5
|
+
|
6
|
+
require 'json'
|
7
|
+
require 'time'
|
8
|
+
require 'timeout'
|
9
|
+
|
10
|
+
module FluentPluginOutputTest
|
11
|
+
class DummyBareOutput < Fluent::Plugin::Output
|
12
|
+
def register(name, &block)
|
13
|
+
instance_variable_set("@#{name}", block)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
class DummySyncOutput < DummyBareOutput
|
17
|
+
def initialize
|
18
|
+
super
|
19
|
+
@process = nil
|
20
|
+
end
|
21
|
+
def process(tag, es)
|
22
|
+
@process ? @process.call(tag, es) : nil
|
23
|
+
end
|
24
|
+
end
|
25
|
+
class DummyAsyncOutput < DummyBareOutput
|
26
|
+
def initialize
|
27
|
+
super
|
28
|
+
@format = nil
|
29
|
+
@write = nil
|
30
|
+
end
|
31
|
+
def format(tag, time, record)
|
32
|
+
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
33
|
+
end
|
34
|
+
def write(chunk)
|
35
|
+
@write ? @write.call(chunk) : nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
class DummyAsyncStandardOutput < DummyBareOutput
|
39
|
+
def initialize
|
40
|
+
super
|
41
|
+
@write = nil
|
42
|
+
end
|
43
|
+
def write(chunk)
|
44
|
+
@write ? @write.call(chunk) : nil
|
45
|
+
end
|
46
|
+
end
|
47
|
+
class DummyDelayedOutput < DummyBareOutput
|
48
|
+
def initialize
|
49
|
+
super
|
50
|
+
@format = nil
|
51
|
+
@try_write = nil
|
52
|
+
end
|
53
|
+
def format(tag, time, record)
|
54
|
+
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
55
|
+
end
|
56
|
+
def try_write(chunk)
|
57
|
+
@try_write ? @try_write.call(chunk) : nil
|
58
|
+
end
|
59
|
+
end
|
60
|
+
class DummyDelayedStandardOutput < DummyBareOutput
|
61
|
+
def initialize
|
62
|
+
super
|
63
|
+
@try_write = nil
|
64
|
+
end
|
65
|
+
def try_write(chunk)
|
66
|
+
@try_write ? @try_write.call(chunk) : nil
|
67
|
+
end
|
68
|
+
end
|
69
|
+
class DummyFullFeatureOutput < DummyBareOutput
|
70
|
+
def initialize
|
71
|
+
super
|
72
|
+
@prefer_buffered_processing = nil
|
73
|
+
@prefer_delayed_commit = nil
|
74
|
+
@process = nil
|
75
|
+
@format = nil
|
76
|
+
@write = nil
|
77
|
+
@try_write = nil
|
78
|
+
end
|
79
|
+
def prefer_buffered_processing
|
80
|
+
@prefer_buffered_processing ? @prefer_buffered_processing.call : false
|
81
|
+
end
|
82
|
+
def prefer_delayed_commit
|
83
|
+
@prefer_delayed_commit ? @prefer_delayed_commit.call : false
|
84
|
+
end
|
85
|
+
def process(tag, es)
|
86
|
+
@process ? @process.call(tag, es) : nil
|
87
|
+
end
|
88
|
+
def format(tag, time, record)
|
89
|
+
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
90
|
+
end
|
91
|
+
def write(chunk)
|
92
|
+
@write ? @write.call(chunk) : nil
|
93
|
+
end
|
94
|
+
def try_write(chunk)
|
95
|
+
@try_write ? @try_write.call(chunk) : nil
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
class OutputTest < Test::Unit::TestCase
|
101
|
+
class << self
|
102
|
+
def startup
|
103
|
+
$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))
|
104
|
+
require 'fluent/plugin/out_test'
|
105
|
+
end
|
106
|
+
|
107
|
+
def shutdown
|
108
|
+
$LOAD_PATH.shift
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def create_output(type=:full)
|
113
|
+
case type
|
114
|
+
when :bare then FluentPluginOutputTest::DummyBareOutput.new
|
115
|
+
when :sync then FluentPluginOutputTest::DummySyncOutput.new
|
116
|
+
when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new
|
117
|
+
when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new
|
118
|
+
when :delayed then FluentPluginOutputTest::DummyDelayedOutput.new
|
119
|
+
when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new
|
120
|
+
when :full then FluentPluginOutputTest::DummyFullFeatureOutput.new
|
121
|
+
else
|
122
|
+
raise ArgumentError, "unknown type: #{type}"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
def create_metadata(timekey: nil, tag: nil, variables: nil)
|
126
|
+
Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
|
127
|
+
end
|
128
|
+
def waiting(seconds)
|
129
|
+
begin
|
130
|
+
Timeout.timeout(seconds) do
|
131
|
+
yield
|
132
|
+
end
|
133
|
+
rescue Timeout::Error
|
134
|
+
STDERR.print(*@i.log.out.logs)
|
135
|
+
raise
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
sub_test_case 'basic output feature' do
|
140
|
+
setup do
|
141
|
+
@i = create_output(:full)
|
142
|
+
end
|
143
|
+
|
144
|
+
test 'are not available with multi workers configuration in default' do
|
145
|
+
assert_false @i.multi_workers_ready?
|
146
|
+
end
|
147
|
+
|
148
|
+
test '#implement? can return features for plugin instances' do
|
149
|
+
i1 = FluentPluginOutputTest::DummyBareOutput.new
|
150
|
+
assert !i1.implement?(:synchronous)
|
151
|
+
assert !i1.implement?(:buffered)
|
152
|
+
assert !i1.implement?(:delayed_commit)
|
153
|
+
assert !i1.implement?(:custom_format)
|
154
|
+
|
155
|
+
i2 = FluentPluginOutputTest::DummySyncOutput.new
|
156
|
+
assert i2.implement?(:synchronous)
|
157
|
+
assert !i2.implement?(:buffered)
|
158
|
+
assert !i2.implement?(:delayed_commit)
|
159
|
+
assert !i2.implement?(:custom_format)
|
160
|
+
|
161
|
+
i3 = FluentPluginOutputTest::DummyAsyncOutput.new
|
162
|
+
assert !i3.implement?(:synchronous)
|
163
|
+
assert i3.implement?(:buffered)
|
164
|
+
assert !i3.implement?(:delayed_commit)
|
165
|
+
assert i3.implement?(:custom_format)
|
166
|
+
|
167
|
+
i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new
|
168
|
+
assert !i4.implement?(:synchronous)
|
169
|
+
assert i4.implement?(:buffered)
|
170
|
+
assert !i4.implement?(:delayed_commit)
|
171
|
+
assert !i4.implement?(:custom_format)
|
172
|
+
|
173
|
+
i5 = FluentPluginOutputTest::DummyDelayedOutput.new
|
174
|
+
assert !i5.implement?(:synchronous)
|
175
|
+
assert !i5.implement?(:buffered)
|
176
|
+
assert i5.implement?(:delayed_commit)
|
177
|
+
assert i5.implement?(:custom_format)
|
178
|
+
|
179
|
+
i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new
|
180
|
+
assert !i6.implement?(:synchronous)
|
181
|
+
assert !i6.implement?(:buffered)
|
182
|
+
assert i6.implement?(:delayed_commit)
|
183
|
+
assert !i6.implement?(:custom_format)
|
184
|
+
|
185
|
+
i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new
|
186
|
+
assert i6.implement?(:synchronous)
|
187
|
+
assert i6.implement?(:buffered)
|
188
|
+
assert i6.implement?(:delayed_commit)
|
189
|
+
assert i6.implement?(:custom_format)
|
190
|
+
end
|
191
|
+
|
192
|
+
test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do
|
193
|
+
assert !@i.configured?
|
194
|
+
@i.configure(config_element())
|
195
|
+
assert @i.configured?
|
196
|
+
assert !@i.started?
|
197
|
+
@i.start
|
198
|
+
assert @i.started?
|
199
|
+
assert !@i.after_started?
|
200
|
+
@i.after_start
|
201
|
+
assert @i.after_started?
|
202
|
+
assert !@i.stopped?
|
203
|
+
@i.stop
|
204
|
+
assert @i.stopped?
|
205
|
+
assert !@i.before_shutdown?
|
206
|
+
@i.before_shutdown
|
207
|
+
assert @i.before_shutdown?
|
208
|
+
assert !@i.shutdown?
|
209
|
+
@i.shutdown
|
210
|
+
assert @i.shutdown?
|
211
|
+
assert !@i.after_shutdown?
|
212
|
+
@i.after_shutdown
|
213
|
+
assert @i.after_shutdown?
|
214
|
+
assert !@i.closed?
|
215
|
+
@i.close
|
216
|
+
assert @i.closed?
|
217
|
+
assert !@i.terminated?
|
218
|
+
@i.terminate
|
219
|
+
assert @i.terminated?
|
220
|
+
end
|
221
|
+
|
222
|
+
test '#extract_placeholders does nothing if chunk key is not specified' do
|
223
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
224
|
+
assert !@i.chunk_key_time
|
225
|
+
assert !@i.chunk_key_tag
|
226
|
+
assert_equal [], @i.chunk_keys
|
227
|
+
tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
|
228
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
229
|
+
v = {key1: "value1", key2: "value2"}
|
230
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
231
|
+
assert_equal tmpl, @i.extract_placeholders(tmpl, m)
|
232
|
+
end
|
233
|
+
|
234
|
+
test '#extract_placeholders can extract time if time key and range are configured' do
|
235
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
|
236
|
+
assert @i.chunk_key_time
|
237
|
+
assert !@i.chunk_key_tag
|
238
|
+
assert_equal [], @i.chunk_keys
|
239
|
+
tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
|
240
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
241
|
+
v = {key1: "value1", key2: "value2"}
|
242
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
243
|
+
assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
|
244
|
+
end
|
245
|
+
|
246
|
+
test '#extract_placeholders can extract tag and parts of tag if tag is configured' do
|
247
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
|
248
|
+
assert !@i.chunk_key_time
|
249
|
+
assert @i.chunk_key_tag
|
250
|
+
assert_equal [], @i.chunk_keys
|
251
|
+
tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
|
252
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
253
|
+
v = {key1: "value1", key2: "value2"}
|
254
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
255
|
+
assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
|
256
|
+
end
|
257
|
+
|
258
|
+
test '#extract_placeholders can extract variables if variables are configured' do
|
259
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
|
260
|
+
assert !@i.chunk_key_time
|
261
|
+
assert !@i.chunk_key_tag
|
262
|
+
assert_equal ['key1','key2'], @i.chunk_keys
|
263
|
+
tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
|
264
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
265
|
+
v = {key1: "value1", key2: "value2"}
|
266
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
267
|
+
assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
|
268
|
+
end
|
269
|
+
|
270
|
+
test '#extract_placeholders can extract all chunk keys if configured' do
|
271
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
|
272
|
+
assert @i.chunk_key_time
|
273
|
+
assert @i.chunk_key_tag
|
274
|
+
assert_equal ['key1','key2'], @i.chunk_keys
|
275
|
+
tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
|
276
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
277
|
+
v = {key1: "value1", key2: "value2"}
|
278
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
279
|
+
assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m)
|
280
|
+
end
|
281
|
+
|
282
|
+
test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do
|
283
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
|
284
|
+
assert @i.chunk_key_time
|
285
|
+
assert @i.chunk_key_tag
|
286
|
+
assert_equal ['key1','key2'], @i.chunk_keys
|
287
|
+
tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
|
288
|
+
t = event_time('2016-04-11 20:30:00 +0900')
|
289
|
+
v = {key1: "value1", key2: "value2"}
|
290
|
+
m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
|
291
|
+
assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
|
292
|
+
end
|
293
|
+
|
294
|
+
sub_test_case '#placeholder_validators' do
|
295
|
+
test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
|
296
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
297
|
+
validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
|
298
|
+
assert_equal 3, validators.size
|
299
|
+
assert_equal 1, validators.select(&:time?).size
|
300
|
+
assert_equal 1, validators.select(&:tag?).size
|
301
|
+
assert_equal 1, validators.select(&:keys?).size
|
302
|
+
end
|
303
|
+
|
304
|
+
test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
|
305
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
|
306
|
+
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
307
|
+
assert_equal 3, validators.size
|
308
|
+
assert_equal 1, validators.select(&:time?).size
|
309
|
+
assert_equal 1, validators.select(&:tag?).size
|
310
|
+
assert_equal 1, validators.select(&:keys?).size
|
311
|
+
end
|
312
|
+
|
313
|
+
test 'returns a validator for time if a template has timestamp placeholders' do
|
314
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
315
|
+
validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
|
316
|
+
assert_equal 1, validators.size
|
317
|
+
assert_equal 1, validators.select(&:time?).size
|
318
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
|
319
|
+
validators.first.validate!
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
test 'returns a validator for time if a plugin is configured with time key' do
|
324
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
|
325
|
+
validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
|
326
|
+
assert_equal 1, validators.size
|
327
|
+
assert_equal 1, validators.select(&:time?).size
|
328
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
|
329
|
+
validators.first.validate!
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
test 'returns a validator for tag if a template has tag placeholders' do
|
334
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
335
|
+
validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
|
336
|
+
assert_equal 1, validators.size
|
337
|
+
assert_equal 1, validators.select(&:tag?).size
|
338
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
|
339
|
+
validators.first.validate!
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
test 'returns a validator for tag if a plugin is configured with tag key' do
|
344
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
|
345
|
+
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
346
|
+
assert_equal 1, validators.size
|
347
|
+
assert_equal 1, validators.select(&:tag?).size
|
348
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
|
349
|
+
validators.first.validate!
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
test 'returns a validator for variable keys if a template has variable placeholders' do
|
354
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
355
|
+
validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
|
356
|
+
assert_equal 1, validators.size
|
357
|
+
assert_equal 1, validators.select(&:keys?).size
|
358
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
|
359
|
+
validators.first.validate!
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
test 'returns a validator for variable keys if a plugin is configured with variable keys' do
|
364
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
|
365
|
+
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
366
|
+
assert_equal 1, validators.size
|
367
|
+
assert_equal 1, validators.select(&:keys?).size
|
368
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
|
369
|
+
validators.first.validate!
|
370
|
+
end
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
sub_test_case '#placeholder_validate!' do
|
375
|
+
test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
|
376
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
377
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
|
378
|
+
@i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
|
+
test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
|
383
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
|
384
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have timestamp placeholders for timekey 180") do
|
385
|
+
@i.placeholder_validate!(:path, "/my/path/file.log")
|
386
|
+
end
|
387
|
+
assert_nothing_raised do
|
388
|
+
@i.placeholder_validate!(:path, "/my/path/%Y%m%d/file.%H%M.log")
|
389
|
+
end
|
390
|
+
end
|
391
|
+
|
392
|
+
test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
|
393
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
|
394
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y%m%d_%H.log' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
|
395
|
+
@i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
|
396
|
+
end
|
397
|
+
assert_nothing_raised do
|
398
|
+
@i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H%M.log")
|
399
|
+
end
|
400
|
+
end
|
401
|
+
|
402
|
+
test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
|
403
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
404
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.${tag[2]}.log' has tag placeholders, but chunk key 'tag' is not configured") do
|
405
|
+
@i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
|
406
|
+
end
|
407
|
+
end
|
408
|
+
|
409
|
+
test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
|
410
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
|
411
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
|
412
|
+
@i.placeholder_validate!(:path, "/my/path/file.log")
|
413
|
+
end
|
414
|
+
assert_nothing_raised do
|
415
|
+
@i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
|
416
|
+
end
|
417
|
+
end
|
418
|
+
|
419
|
+
test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
|
420
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
421
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.${username}.log' has placeholders, but chunk keys doesn't have keys service,username") do
|
422
|
+
@i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
|
427
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
|
428
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys service,username") do
|
429
|
+
@i.placeholder_validate!(:path, "/my/path/file.log")
|
430
|
+
end
|
431
|
+
assert_nothing_raised do
|
432
|
+
@i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
|
433
|
+
end
|
434
|
+
end
|
435
|
+
|
436
|
+
test 'raise configuration error for a template and configuration with keys mismatch' do
|
437
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
|
438
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.${username}.log' doesn't have enough placeholders for keys service") do
|
439
|
+
@i.placeholder_validate!(:path, "/my/path/file.${username}.log")
|
440
|
+
end
|
441
|
+
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.log' doesn't have enough placeholders for keys username") do
|
442
|
+
@i.placeholder_validate!(:path, "/my/path/${service}/file.log")
|
443
|
+
end
|
444
|
+
assert_nothing_raised do
|
445
|
+
@i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
|
446
|
+
end
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
450
|
+
test '#get_placeholders_time returns seconds,title and example placeholder for a template' do
|
451
|
+
s, t, e = @i.get_placeholders_time("/path/to/dir/yay")
|
452
|
+
assert_nil s
|
453
|
+
assert_nil t
|
454
|
+
assert_nil e
|
455
|
+
|
456
|
+
s, t, e = @i.get_placeholders_time("/path/to/%Y%m%d/yay")
|
457
|
+
assert_equal 86400, s
|
458
|
+
assert_equal :day, t
|
459
|
+
assert_equal '%d', e
|
460
|
+
s, t, e = @i.get_placeholders_time("my birthiday! at %F")
|
461
|
+
assert_equal 86400, s
|
462
|
+
assert_equal :day, t
|
463
|
+
assert_equal '%d', e
|
464
|
+
|
465
|
+
s, t, e = @i.get_placeholders_time("myfile.%Y-%m-%d_%H.log")
|
466
|
+
assert_equal 3600, s
|
467
|
+
assert_equal :hour, t
|
468
|
+
assert_equal '%H', e
|
469
|
+
|
470
|
+
s, t, e = @i.get_placeholders_time("part-%Y%m%d-%H%M.ts")
|
471
|
+
assert_equal 60, s
|
472
|
+
assert_equal :minute, t
|
473
|
+
assert_equal '%M', e
|
474
|
+
|
475
|
+
s, t, e = @i.get_placeholders_time("my first data at %F %T %z")
|
476
|
+
assert_equal 1, s
|
477
|
+
assert_equal :second, t
|
478
|
+
assert_equal '%S', e
|
479
|
+
end
|
480
|
+
|
481
|
+
test '#get_placeholders_tag returns a list of tag part position for a template' do
|
482
|
+
assert_equal [], @i.get_placeholders_tag("db.table")
|
483
|
+
assert_equal [], @i.get_placeholders_tag("db.table_${non_tag}")
|
484
|
+
assert_equal [-1], @i.get_placeholders_tag("table_${tag}")
|
485
|
+
assert_equal [0, 1], @i.get_placeholders_tag("db_${tag[0]}.table_${tag[1]}")
|
486
|
+
assert_equal [-1, 0], @i.get_placeholders_tag("/treedir/${tag[0]}/${tag}")
|
487
|
+
end
|
488
|
+
|
489
|
+
test '#get_placeholders_keys returns a list of keys for a template' do
|
490
|
+
assert_equal [], @i.get_placeholders_keys("/path/to/my/data/file.log")
|
491
|
+
assert_equal [], @i.get_placeholders_keys("/path/to/my/${tag}/file.log")
|
492
|
+
assert_equal ['key1', 'key2'], @i.get_placeholders_keys("/path/to/${key2}/${tag}/file.${key1}.log")
|
493
|
+
assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
|
494
|
+
end
|
495
|
+
|
496
|
+
test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
|
497
|
+
tag = 'test.output'
|
498
|
+
time = event_time('2016-04-12 15:31:23 -0700')
|
499
|
+
timekey = event_time('2016-04-12 15:00:00 -0700')
|
500
|
+
record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
|
501
|
+
|
502
|
+
i1 = create_output(:buffered)
|
503
|
+
i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
|
504
|
+
assert_equal create_metadata(), i1.metadata(tag, time, record)
|
505
|
+
|
506
|
+
i2 = create_output(:buffered)
|
507
|
+
i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))
|
508
|
+
assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)
|
509
|
+
|
510
|
+
i3 = create_output(:buffered)
|
511
|
+
i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
|
512
|
+
assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)
|
513
|
+
|
514
|
+
i4 = create_output(:buffered)
|
515
|
+
i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))
|
516
|
+
assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record)
|
517
|
+
|
518
|
+
i5 = create_output(:buffered)
|
519
|
+
i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))
|
520
|
+
assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record)
|
521
|
+
|
522
|
+
i6 = create_output(:buffered)
|
523
|
+
i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
|
524
|
+
assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)
|
525
|
+
|
526
|
+
i7 = create_output(:buffered)
|
527
|
+
i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
|
528
|
+
assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)
|
529
|
+
|
530
|
+
i8 = create_output(:buffered)
|
531
|
+
i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
|
532
|
+
assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
|
533
|
+
end
|
534
|
+
|
535
|
+
test '#emit calls #process via #emit_sync for non-buffered output' do
|
536
|
+
i = create_output(:sync)
|
537
|
+
process_called = false
|
538
|
+
i.register(:process){|tag, es| process_called = true }
|
539
|
+
i.configure(config_element())
|
540
|
+
i.start
|
541
|
+
i.after_start
|
542
|
+
|
543
|
+
t = event_time()
|
544
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
545
|
+
|
546
|
+
assert process_called
|
547
|
+
|
548
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
549
|
+
end
|
550
|
+
|
551
|
+
test '#emit calls #format for buffered output' do
|
552
|
+
i = create_output(:buffered)
|
553
|
+
format_called_times = 0
|
554
|
+
i.register(:format){|tag, time, record| format_called_times += 1; '' }
|
555
|
+
i.configure(config_element())
|
556
|
+
i.start
|
557
|
+
i.after_start
|
558
|
+
|
559
|
+
t = event_time()
|
560
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
561
|
+
|
562
|
+
assert_equal 2, format_called_times
|
563
|
+
|
564
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
565
|
+
end
|
566
|
+
|
567
|
+
test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do
|
568
|
+
i = create_output(:full)
|
569
|
+
|
570
|
+
process_called = false
|
571
|
+
format_called_times = 0
|
572
|
+
i.register(:process){|tag, es| process_called = true }
|
573
|
+
i.register(:format){|tag, time, record| format_called_times += 1; '' }
|
574
|
+
|
575
|
+
i.configure(config_element())
|
576
|
+
i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
|
577
|
+
i.start
|
578
|
+
i.after_start
|
579
|
+
|
580
|
+
assert !i.prefer_buffered_processing
|
581
|
+
|
582
|
+
t = event_time()
|
583
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
584
|
+
|
585
|
+
waiting(4){ Thread.pass until process_called }
|
586
|
+
|
587
|
+
assert process_called
|
588
|
+
assert_equal 0, format_called_times
|
589
|
+
|
590
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
591
|
+
end
|
592
|
+
|
593
|
+
test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do
|
594
|
+
i = create_output(:full)
|
595
|
+
|
596
|
+
process_called = false
|
597
|
+
format_called_times = 0
|
598
|
+
i.register(:process){|tag, es| process_called = true }
|
599
|
+
i.register(:format){|tag, time, record| format_called_times += 1; '' }
|
600
|
+
|
601
|
+
i.configure(config_element())
|
602
|
+
i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
|
603
|
+
i.start
|
604
|
+
i.after_start
|
605
|
+
|
606
|
+
assert i.prefer_buffered_processing
|
607
|
+
|
608
|
+
t = event_time()
|
609
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
610
|
+
|
611
|
+
assert !process_called
|
612
|
+
assert_equal 2, format_called_times
|
613
|
+
|
614
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
615
|
+
end
|
616
|
+
|
617
|
+
test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do
|
618
|
+
i = create_output(:buffered)
|
619
|
+
write_called = false
|
620
|
+
i.register(:write){ |chunk| write_called = true }
|
621
|
+
|
622
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
|
623
|
+
i.start
|
624
|
+
i.after_start
|
625
|
+
|
626
|
+
t = event_time()
|
627
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
628
|
+
i.force_flush
|
629
|
+
|
630
|
+
waiting(4){ Thread.pass until write_called }
|
631
|
+
|
632
|
+
assert write_called
|
633
|
+
|
634
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
635
|
+
end
|
636
|
+
|
637
|
+
test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
|
638
|
+
i = create_output(:delayed)
|
639
|
+
try_write_called = false
|
640
|
+
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
641
|
+
|
642
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
|
643
|
+
i.start
|
644
|
+
i.after_start
|
645
|
+
|
646
|
+
t = event_time()
|
647
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
648
|
+
i.force_flush
|
649
|
+
|
650
|
+
waiting(4){ Thread.pass until try_write_called }
|
651
|
+
|
652
|
+
assert try_write_called
|
653
|
+
|
654
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
655
|
+
end
|
656
|
+
|
657
|
+
test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
|
658
|
+
i = create_output(:full)
|
659
|
+
write_called = false
|
660
|
+
try_write_called = false
|
661
|
+
i.register(:write){ |chunk| write_called = true }
|
662
|
+
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
663
|
+
|
664
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
|
665
|
+
i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
|
666
|
+
i.start
|
667
|
+
i.after_start
|
668
|
+
|
669
|
+
assert !i.prefer_delayed_commit
|
670
|
+
|
671
|
+
t = event_time()
|
672
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
673
|
+
i.force_flush
|
674
|
+
|
675
|
+
waiting(4){ Thread.pass until write_called || try_write_called }
|
676
|
+
|
677
|
+
assert write_called
|
678
|
+
assert !try_write_called
|
679
|
+
|
680
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
681
|
+
end
|
682
|
+
|
683
|
+
test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
|
684
|
+
i = create_output(:full)
|
685
|
+
write_called = false
|
686
|
+
try_write_called = false
|
687
|
+
i.register(:write){ |chunk| write_called = true }
|
688
|
+
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
689
|
+
|
690
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
|
691
|
+
i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
|
692
|
+
i.start
|
693
|
+
i.after_start
|
694
|
+
|
695
|
+
assert i.prefer_delayed_commit
|
696
|
+
|
697
|
+
t = event_time()
|
698
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
699
|
+
i.force_flush
|
700
|
+
|
701
|
+
waiting(4){ Thread.pass until write_called || try_write_called }
|
702
|
+
|
703
|
+
assert !write_called
|
704
|
+
assert try_write_called
|
705
|
+
|
706
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
707
|
+
end
|
708
|
+
|
709
|
+
test 'flush_interval is ignored when flush_mode is not interval' do
|
710
|
+
mock(@i.log).warn("'flush_interval' is ignored because default 'flush_mode' is not 'interval': 'lazy'")
|
711
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'flush_interval' => 10})]))
|
712
|
+
end
|
713
|
+
|
714
|
+
data(:lazy => 'lazy', :immediate => 'immediate')
|
715
|
+
test 'flush_interval and non-interval flush_mode is exclusive ' do |mode|
|
716
|
+
assert_raise Fluent::ConfigError.new("'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{mode}'") do
|
717
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'flush_mode' => mode, 'flush_interval' => 10})]))
|
718
|
+
end
|
719
|
+
end
|
720
|
+
|
721
|
+
test 'flush_mode is set to interval when flush_interval with v0.12 configuration is given' do
|
722
|
+
mock(@i.log).info("'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour")
|
723
|
+
@i.configure(config_element('ROOT', '', {'flush_interval' => 60}, []))
|
724
|
+
assert_equal :interval, @i.instance_variable_get(:@flush_mode)
|
725
|
+
end
|
726
|
+
|
727
|
+
test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
|
728
|
+
o = create_output(:buffered)
|
729
|
+
mock(o.log).warn("secondary type should be same with primary one",
|
730
|
+
{ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
|
731
|
+
|
732
|
+
o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
|
733
|
+
assert_not_nil o.instance_variable_get(:@secondary)
|
734
|
+
end
|
735
|
+
|
736
|
+
test "don't warn if primary type is the same as secondary type" do
|
737
|
+
o = Fluent::Plugin::TestOutput.new
|
738
|
+
mock(o.log).warn("secondary type should be same with primary one",
|
739
|
+
{ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
|
740
|
+
|
741
|
+
o.configure(config_element('ROOT','',{'name' => "cool2"},
|
742
|
+
[config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
|
743
|
+
config_element('buffer','',{'@type'=>'memory'})]
|
744
|
+
))
|
745
|
+
assert_not_nil o.instance_variable_get(:@secondary)
|
746
|
+
end
|
747
|
+
|
748
|
+
test "don't warn if primary type is different from secondary type and both don't have custom_format" do
|
749
|
+
o = create_output(:standard)
|
750
|
+
mock(o.log).warn("secondary type should be same with primary one",
|
751
|
+
{ primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
|
752
|
+
|
753
|
+
o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
|
754
|
+
assert_not_nil o.instance_variable_get(:@secondary)
|
755
|
+
end
|
756
|
+
end
|
757
|
+
|
758
|
+
sub_test_case 'sync output feature' do
|
759
|
+
setup do
|
760
|
+
@i = create_output(:sync)
|
761
|
+
end
|
762
|
+
|
763
|
+
test 'raises configuration error if <buffer> section is specified' do
|
764
|
+
assert_raise Fluent::ConfigError do
|
765
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
|
766
|
+
end
|
767
|
+
end
|
768
|
+
|
769
|
+
test 'raises configuration error if <secondary> section is specified' do
|
770
|
+
assert_raise Fluent::ConfigError do
|
771
|
+
@i.configure(config_element('ROOT','',{},[config_element('secondary','')]))
|
772
|
+
end
|
773
|
+
end
|
774
|
+
|
775
|
+
test '#process is called for each event streams' do
|
776
|
+
ary = []
|
777
|
+
@i.register(:process){|tag, es| ary << [tag, es] }
|
778
|
+
@i.configure(config_element())
|
779
|
+
@i.start
|
780
|
+
@i.after_start
|
781
|
+
|
782
|
+
t = event_time()
|
783
|
+
es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
|
784
|
+
5.times do
|
785
|
+
@i.emit_events('tag', es)
|
786
|
+
end
|
787
|
+
assert_equal 5, ary.size
|
788
|
+
|
789
|
+
@i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
|
790
|
+
end
|
791
|
+
end
|
792
|
+
|
793
|
+
sub_test_case '#generate_format_proc' do
|
794
|
+
test "when output doesn't have <buffer>" do
|
795
|
+
i = create_output(:sync)
|
796
|
+
i.configure(config_element('ROOT', '', {}, []))
|
797
|
+
assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
|
798
|
+
end
|
799
|
+
|
800
|
+
test "when output doesn't have <buffer> and time_as_integer is true" do
|
801
|
+
i = create_output(:sync)
|
802
|
+
i.configure(config_element('ROOT', '', {'time_as_integer' => true}))
|
803
|
+
assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
|
804
|
+
end
|
805
|
+
|
806
|
+
test 'when output has <buffer> and compress is gzip' do
|
807
|
+
i = create_output(:buffered)
|
808
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))
|
809
|
+
assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM, i.generate_format_proc
|
810
|
+
end
|
811
|
+
|
812
|
+
test 'when output has <buffer> and compress is gzip and time_as_integer is true' do
|
813
|
+
i = create_output(:buffered)
|
814
|
+
i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))
|
815
|
+
assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
|
816
|
+
end
|
817
|
+
|
818
|
+
test 'when output has <buffer> and compress is text' do
|
819
|
+
i = create_output(:buffered)
|
820
|
+
i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))
|
821
|
+
assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
|
822
|
+
end
|
823
|
+
end
|
824
|
+
|
825
|
+
sub_test_case 'slow_flush_log_threshold' do
|
826
|
+
def invoke_slow_flush_log_threshold_test(i)
|
827
|
+
i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
|
828
|
+
i.start
|
829
|
+
i.after_start
|
830
|
+
|
831
|
+
t = event_time()
|
832
|
+
i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
|
833
|
+
i.force_flush
|
834
|
+
|
835
|
+
waiting(4) { Thread.pass until i.test_finished? }
|
836
|
+
|
837
|
+
yield
|
838
|
+
ensure
|
839
|
+
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
840
|
+
end
|
841
|
+
|
842
|
+
test '#write flush took longer time than slow_flush_log_threshold' do
|
843
|
+
i = create_output(:buffered)
|
844
|
+
write_called = false
|
845
|
+
i.register(:write) { |chunk| sleep 3 }
|
846
|
+
i.define_singleton_method(:test_finished?) { write_called }
|
847
|
+
i.define_singleton_method(:try_flush) { super(); write_called = true }
|
848
|
+
|
849
|
+
invoke_slow_flush_log_threshold_test(i) {
|
850
|
+
assert write_called
|
851
|
+
logs = i.log.out.logs
|
852
|
+
assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
|
853
|
+
}
|
854
|
+
end
|
855
|
+
|
856
|
+
test '#try_write flush took longer time than slow_flush_log_threshold' do
|
857
|
+
i = create_output(:delayed)
|
858
|
+
try_write_called = false
|
859
|
+
i.register(:try_write){ |chunk| sleep 3 }
|
860
|
+
i.define_singleton_method(:test_finished?) { try_write_called }
|
861
|
+
i.define_singleton_method(:try_flush) { super(); try_write_called = true }
|
862
|
+
|
863
|
+
invoke_slow_flush_log_threshold_test(i) {
|
864
|
+
assert try_write_called
|
865
|
+
logs = i.log.out.logs
|
866
|
+
assert{ logs.any?{|log| log.include?("buffer flush took longer time than slow_flush_log_threshold: elapsed_time") } }
|
867
|
+
}
|
868
|
+
end
|
869
|
+
end
|
870
|
+
end
|