fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
@@ -0,0 +1,66 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser'
|
4
|
+
|
5
|
+
class SyslogParserTest < ::Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
@parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::SyslogParser)
|
9
|
+
@expected = {
|
10
|
+
'host' => '192.168.0.1',
|
11
|
+
'ident' => 'fluentd',
|
12
|
+
'pid' => '11111',
|
13
|
+
'message' => '[error] Syslog test'
|
14
|
+
}
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_parse
|
18
|
+
@parser.configure({})
|
19
|
+
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
20
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
21
|
+
assert_equal(@expected, record)
|
22
|
+
}
|
23
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
|
24
|
+
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_parse_with_time_format
|
28
|
+
@parser.configure('time_format' => '%b %d %M:%S:%H')
|
29
|
+
@parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
30
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
31
|
+
assert_equal(@expected, record)
|
32
|
+
}
|
33
|
+
assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_parse_with_priority
|
37
|
+
@parser.configure('with_priority' => true)
|
38
|
+
@parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
39
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
40
|
+
assert_equal(@expected.merge('pri' => 6), record)
|
41
|
+
}
|
42
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
43
|
+
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_parse_without_colon
|
47
|
+
@parser.configure({})
|
48
|
+
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
|
49
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
50
|
+
assert_equal(@expected, record)
|
51
|
+
}
|
52
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
|
53
|
+
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_parse_with_keep_time_key
|
57
|
+
@parser.configure(
|
58
|
+
'time_format' => '%b %d %M:%S:%H',
|
59
|
+
'keep_time_key'=>'true',
|
60
|
+
)
|
61
|
+
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
62
|
+
@parser.instance.parse(text) do |time, record|
|
63
|
+
assert_equal "Feb 28 00:00:12", record['time']
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser'
|
4
|
+
|
5
|
+
class TimeParserTest < ::Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_call_with_parse
|
11
|
+
parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
|
12
|
+
|
13
|
+
assert(parser.instance.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))
|
14
|
+
|
15
|
+
time = event_time('2013-09-18 12:00:00 +0900')
|
16
|
+
assert_equal(time, parser.instance.parse('2013-09-18 12:00:00 +0900'))
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_parse_with_strptime
|
20
|
+
parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z'))
|
21
|
+
|
22
|
+
assert(parser.instance.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))
|
23
|
+
|
24
|
+
time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
|
25
|
+
assert_equal(time, parser.instance.parse('28/Feb/2013:12:00:00 +0900'))
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_parse_nsec_with_strptime
|
29
|
+
parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z'))
|
30
|
+
|
31
|
+
assert(parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))
|
32
|
+
|
33
|
+
time = event_time('28/Feb/2013:12:00:00:123456789 +0900', format: '%d/%b/%Y:%H:%M:%S:%N %z')
|
34
|
+
assert_equal_event_time(time, parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900'))
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_parse_with_invalid_argument
|
38
|
+
parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
|
39
|
+
|
40
|
+
[[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
|
41
|
+
assert_raise Fluent::ParserError do
|
42
|
+
parser.instance.parse(v)
|
43
|
+
end
|
44
|
+
}
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser_tsv'
|
4
|
+
|
5
|
+
class TSVParserTest < ::Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_driver(conf={})
|
11
|
+
Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser).configure(conf)
|
12
|
+
end
|
13
|
+
|
14
|
+
data('array param' => '["a","b"]', 'string param' => 'a,b')
|
15
|
+
def test_config_params(param)
|
16
|
+
d = create_driver
|
17
|
+
|
18
|
+
assert_equal "\t", d.instance.delimiter
|
19
|
+
|
20
|
+
d = create_driver(
|
21
|
+
'keys' => param,
|
22
|
+
'delimiter' => ',',
|
23
|
+
)
|
24
|
+
|
25
|
+
assert_equal ['a', 'b'], d.instance.keys
|
26
|
+
assert_equal ",", d.instance.delimiter
|
27
|
+
end
|
28
|
+
|
29
|
+
data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
|
30
|
+
def test_parse(param)
|
31
|
+
d = create_driver('keys' => param, 'time_key' => 'time')
|
32
|
+
d.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
|
33
|
+
assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
|
34
|
+
assert_equal({
|
35
|
+
'a' => '192.168.0.1',
|
36
|
+
'b' => '111',
|
37
|
+
}, record)
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_parse_with_time
|
42
|
+
time_at_start = Time.now.to_i
|
43
|
+
|
44
|
+
d = create_driver('keys' => 'a,b')
|
45
|
+
d.instance.parse("192.168.0.1\t111") { |time, record|
|
46
|
+
assert time && time >= time_at_start, "parser puts current time without time input"
|
47
|
+
assert_equal({
|
48
|
+
'a' => '192.168.0.1',
|
49
|
+
'b' => '111',
|
50
|
+
}, record)
|
51
|
+
}
|
52
|
+
|
53
|
+
d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
|
54
|
+
d.instance.estimate_current_event = false
|
55
|
+
d.configure('keys' => 'a,b', 'time_key' => 'time')
|
56
|
+
d.instance.parse("192.168.0.1\t111") { |time, record|
|
57
|
+
assert_equal({
|
58
|
+
'a' => '192.168.0.1',
|
59
|
+
'b' => '111',
|
60
|
+
}, record)
|
61
|
+
assert_nil time, "parser returns nil w/o time and if configured so"
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
data(
|
66
|
+
'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
|
67
|
+
'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
|
68
|
+
'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
|
69
|
+
'2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
|
70
|
+
'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
|
71
|
+
'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
|
72
|
+
def test_black_column(data)
|
73
|
+
line, expected = data
|
74
|
+
|
75
|
+
d = create_driver('keys' => '1,2,3')
|
76
|
+
d.instance.parse(line) { |time, record|
|
77
|
+
assert_equal(expected, record)
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_parse_with_keep_time_key
|
82
|
+
d = create_driver(
|
83
|
+
'keys'=>'time',
|
84
|
+
'time_key'=>'time',
|
85
|
+
'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
|
86
|
+
'keep_time_key'=>'true',
|
87
|
+
)
|
88
|
+
text = '28/Feb/2013:12:00:00 +0900'
|
89
|
+
d.instance.parse(text) do |time, record|
|
90
|
+
assert_equal text, record['time']
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
|
95
|
+
def test_parse_with_null_value_pattern(param)
|
96
|
+
d = create_driver(
|
97
|
+
'keys'=>param,
|
98
|
+
'null_value_pattern'=>'^(-|null|NULL)$'
|
99
|
+
)
|
100
|
+
d.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
|
101
|
+
assert_nil record['a']
|
102
|
+
assert_nil record['b']
|
103
|
+
assert_nil record['c']
|
104
|
+
assert_equal record['d'], ''
|
105
|
+
assert_equal record['e'], '--'
|
106
|
+
assert_equal record['f'], 'nuLL'
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
data('array param' => '["a","b"]', 'string param' => 'a,b')
|
111
|
+
def test_parse_with_null_empty_string(param)
|
112
|
+
d = create_driver(
|
113
|
+
'keys'=>param,
|
114
|
+
'null_empty_string'=>true
|
115
|
+
)
|
116
|
+
d.instance.parse("\t ") do |time, record|
|
117
|
+
assert_nil record['a']
|
118
|
+
assert_equal record['b'], ' '
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/storage'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
|
5
|
+
class DummyPlugin < Fluent::Plugin::TestBase
|
6
|
+
end
|
7
|
+
|
8
|
+
class BareStorage < Fluent::Plugin::Storage
|
9
|
+
Fluent::Plugin.register_storage('bare', self)
|
10
|
+
end
|
11
|
+
|
12
|
+
class BasicStorage < Fluent::Plugin::Storage
|
13
|
+
Fluent::Plugin.register_storage('example', self)
|
14
|
+
|
15
|
+
attr_reader :data, :saved
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
super
|
19
|
+
@data = @saved = nil
|
20
|
+
end
|
21
|
+
def load
|
22
|
+
@data = {}
|
23
|
+
end
|
24
|
+
def save
|
25
|
+
@saved = @data.dup
|
26
|
+
end
|
27
|
+
def get(key)
|
28
|
+
@data[key]
|
29
|
+
end
|
30
|
+
def fetch(key, defval)
|
31
|
+
@data.fetch(key, defval)
|
32
|
+
end
|
33
|
+
def put(key, value)
|
34
|
+
@data[key] = value
|
35
|
+
end
|
36
|
+
def delete(key)
|
37
|
+
@data.delete(key)
|
38
|
+
end
|
39
|
+
def update(key, &block)
|
40
|
+
@data[key] = block.call(@data[key])
|
41
|
+
end
|
42
|
+
def close
|
43
|
+
@data = {}
|
44
|
+
super
|
45
|
+
end
|
46
|
+
def terminate
|
47
|
+
@saved = {}
|
48
|
+
super
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
class StorageTest < Test::Unit::TestCase
|
53
|
+
sub_test_case 'BareStorage' do
|
54
|
+
setup do
|
55
|
+
plugin = DummyPlugin.new
|
56
|
+
@s = BareStorage.new
|
57
|
+
@s.configure(config_element())
|
58
|
+
@s.owner = plugin
|
59
|
+
end
|
60
|
+
|
61
|
+
test 'is configured with plugin information and system config' do
|
62
|
+
plugin = DummyPlugin.new
|
63
|
+
plugin.system_config_override({'process_name' => 'mytest'})
|
64
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
|
65
|
+
s = BareStorage.new
|
66
|
+
s.configure(config_element())
|
67
|
+
s.owner = plugin
|
68
|
+
|
69
|
+
assert_equal 'mytest', s.owner.system_config.process_name
|
70
|
+
assert_equal '1', s.instance_eval{ @_plugin_id }
|
71
|
+
assert_equal true, s.instance_eval{ @_plugin_id_configured }
|
72
|
+
end
|
73
|
+
|
74
|
+
test 'does NOT have features for high-performance/high-consistent storages' do
|
75
|
+
assert_equal false, @s.persistent_always?
|
76
|
+
assert_equal false, @s.synchronized?
|
77
|
+
end
|
78
|
+
|
79
|
+
test 'does have default values which is conservative for almost all users' do
|
80
|
+
assert_equal false, @s.persistent
|
81
|
+
assert_equal true, @s.autosave
|
82
|
+
assert_equal 10, @s.autosave_interval
|
83
|
+
assert_equal true, @s.save_at_shutdown
|
84
|
+
end
|
85
|
+
|
86
|
+
test 'load/save doesn NOT anything: just as memory storage' do
|
87
|
+
assert_nothing_raised{ @s.load }
|
88
|
+
assert_nothing_raised{ @s.save }
|
89
|
+
end
|
90
|
+
|
91
|
+
test 'all operations are not defined yet' do
|
92
|
+
assert_raise NotImplementedError do
|
93
|
+
@s.get('key')
|
94
|
+
end
|
95
|
+
assert_raise NotImplementedError do
|
96
|
+
@s.fetch('key', 'value')
|
97
|
+
end
|
98
|
+
assert_raise NotImplementedError do
|
99
|
+
@s.put('key', 'value')
|
100
|
+
end
|
101
|
+
assert_raise NotImplementedError do
|
102
|
+
@s.delete('key')
|
103
|
+
end
|
104
|
+
assert_raise NotImplementedError do
|
105
|
+
@s.update('key'){ |v| v + '2' }
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
sub_test_case 'ExampleStorage' do
|
111
|
+
setup do
|
112
|
+
plugin = DummyPlugin.new
|
113
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
|
114
|
+
@s = BasicStorage.new
|
115
|
+
@s.configure(config_element())
|
116
|
+
@s.owner = plugin
|
117
|
+
end
|
118
|
+
|
119
|
+
test 'load/save works well as plugin internal state operations' do
|
120
|
+
plugin = DummyPlugin.new
|
121
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '0'}))
|
122
|
+
s = BasicStorage.new
|
123
|
+
s.owner = plugin
|
124
|
+
|
125
|
+
assert_nothing_raised{ s.load }
|
126
|
+
assert s.data
|
127
|
+
assert_nil s.saved
|
128
|
+
|
129
|
+
assert_nothing_raised{ s.save }
|
130
|
+
assert s.saved
|
131
|
+
assert{ s.data == s.saved }
|
132
|
+
assert{ s.data.object_id != s.saved.object_id }
|
133
|
+
end
|
134
|
+
|
135
|
+
test 'all operations work well' do
|
136
|
+
@s.load
|
137
|
+
|
138
|
+
assert_nil @s.get('key')
|
139
|
+
assert_equal 'value', @s.fetch('key', 'value')
|
140
|
+
assert_nil @s.get('key')
|
141
|
+
|
142
|
+
assert_equal 'value', @s.put('key', 'value')
|
143
|
+
assert_equal 'value', @s.get('key')
|
144
|
+
|
145
|
+
assert_equal 'valuevalue', @s.update('key'){|v| v * 2 }
|
146
|
+
|
147
|
+
assert_equal 'valuevalue', @s.delete('key')
|
148
|
+
end
|
149
|
+
|
150
|
+
test 'close and terminate work to operate internal states' do
|
151
|
+
@s.load
|
152
|
+
@s.put('k1', 'v1')
|
153
|
+
@s.put('k2', 'v2')
|
154
|
+
assert_equal 2, @s.data.size
|
155
|
+
@s.save
|
156
|
+
assert_equal @s.data.size, @s.saved.size
|
157
|
+
|
158
|
+
assert_nothing_raised{ @s.close }
|
159
|
+
assert @s.data.empty?
|
160
|
+
assert !@s.saved.empty?
|
161
|
+
|
162
|
+
assert_nothing_raised{ @s.terminate }
|
163
|
+
assert @s.data.empty?
|
164
|
+
assert @s.saved.empty?
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/string_util'
|
3
|
+
|
4
|
+
class StringUtilTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
@null_value_pattern = Regexp.new("^(-|null|NULL)$")
|
7
|
+
end
|
8
|
+
|
9
|
+
sub_test_case 'valid string' do
|
10
|
+
test 'null string' do
|
11
|
+
assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "null").to_s, "null"
|
12
|
+
assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "NULL").to_s, "NULL"
|
13
|
+
assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "-").to_s, "-"
|
14
|
+
end
|
15
|
+
|
16
|
+
test 'normal string' do
|
17
|
+
assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "fluentd"), nil
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
sub_test_case 'invalid string' do
|
22
|
+
test 'normal string' do
|
23
|
+
assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "\xff"), nil
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,608 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin_helper/child_process'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
require 'timeout'
|
5
|
+
|
6
|
+
class ChildProcessTest < Test::Unit::TestCase
|
7
|
+
TEST_DEADLOCK_TIMEOUT = 30
|
8
|
+
TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING = 0.1 # This may be shorter than ruby's threading timer, but work well
|
9
|
+
# @nalsh says that ruby's cpu assignments for threads are almost 200ms or so.
|
10
|
+
# Loop interval (expected that it work as specified) should be longer than it.
|
11
|
+
TEST_WAIT_INTERVAL_FOR_LOOP = 0.5
|
12
|
+
|
13
|
+
setup do
|
14
|
+
@d = Dummy.new
|
15
|
+
@d.configure(config_element())
|
16
|
+
@d.start
|
17
|
+
end
|
18
|
+
|
19
|
+
teardown do
|
20
|
+
if @d
|
21
|
+
@d.stop unless @d.stopped?
|
22
|
+
@d.shutdown unless @d.shutdown?
|
23
|
+
@d.close unless @d.closed?
|
24
|
+
@d.terminate unless @d.terminated?
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class Dummy < Fluent::Plugin::TestBase
|
29
|
+
helpers :child_process
|
30
|
+
def configure(conf)
|
31
|
+
super
|
32
|
+
@_child_process_kill_timeout = 1
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
test 'can be instantiated' do
|
37
|
+
d1 = Dummy.new
|
38
|
+
assert d1.respond_to?(:_child_process_processes)
|
39
|
+
end
|
40
|
+
|
41
|
+
test 'can be configured and started' do
|
42
|
+
d1 = Dummy.new
|
43
|
+
assert_nothing_raised do
|
44
|
+
d1.configure(config_element())
|
45
|
+
end
|
46
|
+
assert d1.plugin_id
|
47
|
+
assert d1.log
|
48
|
+
|
49
|
+
d1.start
|
50
|
+
end
|
51
|
+
|
52
|
+
test 'can execute external command asyncronously' do
|
53
|
+
m = Mutex.new
|
54
|
+
m.lock
|
55
|
+
ary = []
|
56
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
57
|
+
ran = false
|
58
|
+
@d.child_process_execute(:t0, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
|
59
|
+
m.lock
|
60
|
+
ran = true
|
61
|
+
io.read # discard
|
62
|
+
ary << 2
|
63
|
+
m.unlock
|
64
|
+
end
|
65
|
+
ary << 1
|
66
|
+
m.unlock
|
67
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
68
|
+
m.lock
|
69
|
+
m.unlock
|
70
|
+
end
|
71
|
+
assert_equal [1,2], ary
|
72
|
+
end
|
73
|
+
|
74
|
+
test 'can execute external command at just once, which finishes immediately' do
|
75
|
+
m = Mutex.new
|
76
|
+
t1 = Time.now
|
77
|
+
ary = []
|
78
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
79
|
+
ran = false
|
80
|
+
@d.child_process_execute(:t1, 'echo', arguments: ['foo', 'bar'], mode: [:read]) do |io|
|
81
|
+
m.lock
|
82
|
+
ran = true
|
83
|
+
ary << io.read
|
84
|
+
m.unlock
|
85
|
+
end
|
86
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
87
|
+
m.lock
|
88
|
+
m.unlock
|
89
|
+
end
|
90
|
+
assert{ Time.now - t1 < 4.0 }
|
91
|
+
end
|
92
|
+
|
93
|
+
test 'can execute external command at just once, which can handle both of read and write' do
|
94
|
+
m = Mutex.new
|
95
|
+
ary = []
|
96
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
97
|
+
ran = false
|
98
|
+
cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; end'"
|
99
|
+
@d.child_process_execute(:t2, cmd, mode: [:write, :read]) do |writeio, readio|
|
100
|
+
m.lock
|
101
|
+
ran = true
|
102
|
+
|
103
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
104
|
+
writeio.write "my data#{i}\n"
|
105
|
+
writeio.write "my data#{j}\n"
|
106
|
+
writeio.flush
|
107
|
+
end
|
108
|
+
writeio.close
|
109
|
+
|
110
|
+
while line = readio.readline
|
111
|
+
ary << line
|
112
|
+
end
|
113
|
+
m.unlock
|
114
|
+
end
|
115
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
116
|
+
m.lock
|
117
|
+
m.unlock
|
118
|
+
end
|
119
|
+
|
120
|
+
assert_equal [], @d.log.out.logs
|
121
|
+
expected = (1..6).map{|i| "my data#{i}\n" }
|
122
|
+
assert_equal expected, ary
|
123
|
+
end
|
124
|
+
|
125
|
+
test 'can execute external command at just once, which can handle all of read, write and stderr' do
|
126
|
+
m = Mutex.new
|
127
|
+
ary1 = []
|
128
|
+
ary2 = []
|
129
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
130
|
+
ran = false
|
131
|
+
cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; puts line.chomp; STDOUT.flush rescue nil; STDERR.puts line.chomp; STDERR.flush rescue nil; end'"
|
132
|
+
@d.child_process_execute(:t2a, cmd, mode: [:write, :read, :stderr]) do |writeio, readio, stderrio|
|
133
|
+
m.lock
|
134
|
+
ran = true
|
135
|
+
|
136
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
137
|
+
writeio.write "my data#{i}\n"
|
138
|
+
writeio.write "my data#{j}\n"
|
139
|
+
writeio.flush
|
140
|
+
end
|
141
|
+
writeio.close
|
142
|
+
|
143
|
+
while (line1 = readio.readline) && (line2 = stderrio.readline)
|
144
|
+
ary1 << line1
|
145
|
+
ary2 << line2
|
146
|
+
end
|
147
|
+
|
148
|
+
m.unlock
|
149
|
+
end
|
150
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
151
|
+
m.lock
|
152
|
+
m.unlock
|
153
|
+
end
|
154
|
+
|
155
|
+
assert_equal [], @d.log.out.logs
|
156
|
+
expected = (1..6).map{|i| "my data#{i}\n" }
|
157
|
+
assert_equal expected, ary1
|
158
|
+
assert_equal expected, ary2
|
159
|
+
end
|
160
|
+
|
161
|
+
test 'can execute external command at just once, which can handle both of write and read (with stderr)' do
|
162
|
+
m = Mutex.new
|
163
|
+
ary = []
|
164
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
165
|
+
ran = false
|
166
|
+
cmd = "ruby"
|
167
|
+
args = ['-e', 'while !STDIN.eof? && line = STDIN.readline; puts "[s]" + line.chomp; STDOUT.flush rescue nil; STDERR.puts "[e]" + line.chomp; STDERR.flush rescue nil; end']
|
168
|
+
@d.child_process_execute(:t2b, cmd, arguments: args, mode: [:write, :read_with_stderr]) do |writeio, readio|
|
169
|
+
m.lock
|
170
|
+
ran = true
|
171
|
+
|
172
|
+
[[1,2],[3,4],[5,6]].each do |i,j|
|
173
|
+
writeio.write "my data#{i}\n"
|
174
|
+
writeio.write "my data#{j}\n"
|
175
|
+
writeio.flush
|
176
|
+
end
|
177
|
+
writeio.close
|
178
|
+
|
179
|
+
while line = readio.readline
|
180
|
+
ary << line
|
181
|
+
end
|
182
|
+
|
183
|
+
m.unlock
|
184
|
+
end
|
185
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
186
|
+
m.lock
|
187
|
+
m.unlock
|
188
|
+
end
|
189
|
+
|
190
|
+
assert_equal [], @d.log.out.logs
|
191
|
+
expected = (1..6).map{|i| ["[s]my data#{i}\n", "[e]my data#{i}\n"] }.flatten
|
192
|
+
assert_equal expected, ary
|
193
|
+
end
|
194
|
+
|
195
|
+
test 'can execute external command at just once, which runs forever' do
|
196
|
+
m = Mutex.new
|
197
|
+
ary = []
|
198
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
199
|
+
ran = false
|
200
|
+
args = ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush; end"]
|
201
|
+
@d.child_process_execute(:t3, "ruby", arguments: args, mode: [:read]) do |io|
|
202
|
+
m.lock
|
203
|
+
ran = true
|
204
|
+
begin
|
205
|
+
while @d.child_process_running? && line = io.readline
|
206
|
+
ary << line
|
207
|
+
end
|
208
|
+
rescue
|
209
|
+
# ignore
|
210
|
+
ensure
|
211
|
+
m.unlock
|
212
|
+
end
|
213
|
+
end
|
214
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
215
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 10
|
216
|
+
@d.stop # nothing occures
|
217
|
+
@d.shutdown
|
218
|
+
|
219
|
+
assert{ ary.size > 5 }
|
220
|
+
|
221
|
+
@d.close
|
222
|
+
|
223
|
+
@d.terminate
|
224
|
+
assert @d._child_process_processes.empty?
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
unless Fluent.windows?
|
229
|
+
# In windows environment, child_process try KILL at first (because there's no SIGTERM)
|
230
|
+
test 'can execute external command just once, and can terminate it forcedly when shutdown/terminate even if it ignore SIGTERM' do
|
231
|
+
m = Mutex.new
|
232
|
+
ary = []
|
233
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
234
|
+
ran = false
|
235
|
+
@d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
|
236
|
+
m.lock
|
237
|
+
ran = true
|
238
|
+
begin
|
239
|
+
while line = io.readline
|
240
|
+
ary << line
|
241
|
+
end
|
242
|
+
rescue
|
243
|
+
# ignore
|
244
|
+
ensure
|
245
|
+
m.unlock
|
246
|
+
end
|
247
|
+
end
|
248
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
249
|
+
|
250
|
+
assert_equal [], @d.log.out.logs
|
251
|
+
|
252
|
+
@d.stop # nothing occures
|
253
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
|
254
|
+
lines1 = ary.size
|
255
|
+
assert{ lines1 > 1 }
|
256
|
+
|
257
|
+
pid = @d._child_process_processes.keys.first
|
258
|
+
|
259
|
+
@d.shutdown
|
260
|
+
sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
|
261
|
+
lines2 = ary.size
|
262
|
+
assert{ lines2 > lines1 }
|
263
|
+
|
264
|
+
@d.close
|
265
|
+
|
266
|
+
assert_nil((Process.waitpid(pid, Process::WNOHANG) rescue nil))
|
267
|
+
|
268
|
+
@d.terminate
|
269
|
+
assert @d._child_process_processes.empty?
|
270
|
+
begin
|
271
|
+
Process.waitpid(pid)
|
272
|
+
rescue Errno::ECHILD
|
273
|
+
end
|
274
|
+
# Process successfully KILLed if test reaches here
|
275
|
+
assert true
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
test 'can execute external command many times, which finishes immediately' do
|
281
|
+
ary = []
|
282
|
+
arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 3
|
283
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
284
|
+
@d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 5, mode: [:read]) do |io|
|
285
|
+
ary << io.read.split("\n").map(&:chomp).join
|
286
|
+
end
|
287
|
+
sleep 13 # 5sec * 2 + 3sec
|
288
|
+
assert_equal [], @d.log.out.logs
|
289
|
+
@d.stop
|
290
|
+
assert_equal [], @d.log.out.logs
|
291
|
+
@d.shutdown; @d.close; @d.terminate
|
292
|
+
assert_equal 2, ary.size
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
test 'can execute external command many times, with leading once executed immediately' do
|
297
|
+
ary = []
|
298
|
+
arguments = ["-e", "3.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"]
|
299
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
300
|
+
@d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 5, immediate: true, mode: [:read]) do |io|
|
301
|
+
ary << io.read.split("\n").map(&:chomp).join
|
302
|
+
end
|
303
|
+
sleep 8 # 5sec * 1 + 3sec
|
304
|
+
# but expected lines are same with test above
|
305
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
306
|
+
assert_equal 2, ary.size
|
307
|
+
assert_equal [], @d.log.out.logs
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
test 'does not execute long running external command in parallel in default' do
|
312
|
+
ary = []
|
313
|
+
arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10
|
314
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
315
|
+
assert_equal [], @d.log.out.logs
|
316
|
+
@d.log.out.singleton_class.module_eval do
|
317
|
+
define_method(:write){|message|
|
318
|
+
raise "boo" if message.include?('test: {"test":"test"}') || message.include?('test: {"test"=>"test"}')
|
319
|
+
@logs.push message
|
320
|
+
}
|
321
|
+
end
|
322
|
+
|
323
|
+
@d.child_process_execute(:t7, "ruby", arguments: arguments, interval: 2, immediate: true, mode: [:read]) do |io|
|
324
|
+
ary << io.read.split("\n").map(&:chomp).join
|
325
|
+
end
|
326
|
+
sleep 4
|
327
|
+
assert_equal 1, @d._child_process_processes.size
|
328
|
+
@d.stop
|
329
|
+
warn_msg = '[warn]: previous child process is still running. skipped. title=:t7 command="ruby" arguments=["-e", "10.times{ puts \'okay\'; STDOUT.flush rescue nil; sleep 0.5 }"] interval=2 parallel=false' + "\n"
|
330
|
+
logs = @d.log.out.logs
|
331
|
+
assert{ logs.first.end_with?(warn_msg) }
|
332
|
+
assert{ logs.all?{|line| line.end_with?(warn_msg) } }
|
333
|
+
@d.shutdown; @d.close; @d.terminate
|
334
|
+
assert_equal [], @d.log.out.logs
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
test 'can execute long running external command in parallel if specified' do
|
339
|
+
ary = []
|
340
|
+
arguments = ["-e", "10.times{ puts 'okay'; STDOUT.flush rescue nil; sleep #{TEST_WAIT_INTERVAL_FOR_LOOP} }"] # 0.5 * 10 sec
|
341
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
342
|
+
@d.child_process_execute(:t8, "ruby", arguments: arguments, interval: 1, immediate: true, parallel: true, mode: [:read]) do |io|
|
343
|
+
ary << io.read.split("\n").map(&:chomp).join
|
344
|
+
end
|
345
|
+
sleep 4
|
346
|
+
processes = @d._child_process_processes.size
|
347
|
+
assert{ processes >= 3 && processes <= 5 }
|
348
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
349
|
+
assert_equal [], @d.log.out.logs
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
test 'execute external processes only for writing' do
|
354
|
+
m = Mutex.new
|
355
|
+
unreadable = false
|
356
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
357
|
+
ran = false
|
358
|
+
@d.child_process_execute(:t9, "ruby", arguments: ['-e', 'a=""; while b=STDIN.readline; a+=b; end'], mode: [:write]) do |io|
|
359
|
+
m.lock
|
360
|
+
ran = true
|
361
|
+
begin
|
362
|
+
io.read
|
363
|
+
rescue IOError
|
364
|
+
unreadable = true
|
365
|
+
end
|
366
|
+
50.times do
|
367
|
+
io.write "hahaha\n"
|
368
|
+
end
|
369
|
+
m.unlock
|
370
|
+
end
|
371
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
372
|
+
m.lock
|
373
|
+
m.unlock
|
374
|
+
assert unreadable
|
375
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
376
|
+
assert_equal [], @d.log.out.logs
|
377
|
+
end
|
378
|
+
end
|
379
|
+
|
380
|
+
test 'execute external processes only for reading' do
|
381
|
+
m = Mutex.new
|
382
|
+
unwritable = false
|
383
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
384
|
+
ran = false
|
385
|
+
@d.child_process_execute(:t10, "ruby", arguments: ["-e", "while sleep #{TEST_WAIT_INTERVAL_FOR_LOOP}; puts 1; STDOUT.flush rescue nil; end"], mode: [:read]) do |io|
|
386
|
+
m.lock
|
387
|
+
ran = true
|
388
|
+
begin
|
389
|
+
io.write "foobar"
|
390
|
+
rescue IOError
|
391
|
+
unwritable = true
|
392
|
+
end
|
393
|
+
_data = io.readline
|
394
|
+
m.unlock
|
395
|
+
end
|
396
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
397
|
+
m.lock
|
398
|
+
m.unlock
|
399
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
400
|
+
assert unwritable
|
401
|
+
assert_equal [], @d.log.out.logs
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
test 'can control external encodings' do
|
406
|
+
m = Mutex.new
|
407
|
+
encodings = []
|
408
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
409
|
+
ran = false
|
410
|
+
@d.child_process_execute(:t11, "ruby -e 'sleep 10'", external_encoding: 'ascii-8bit') do |r, w|
|
411
|
+
m.lock
|
412
|
+
ran = true
|
413
|
+
encodings << r.external_encoding
|
414
|
+
encodings << w.external_encoding
|
415
|
+
m.unlock
|
416
|
+
end
|
417
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
418
|
+
m.lock
|
419
|
+
assert_equal Encoding::ASCII_8BIT, encodings[0]
|
420
|
+
assert_equal Encoding::ASCII_8BIT, encodings[1]
|
421
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
422
|
+
end
|
423
|
+
end
|
424
|
+
|
425
|
+
test 'can control internal encodings' do
|
426
|
+
m = Mutex.new
|
427
|
+
encodings = []
|
428
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
429
|
+
ran = false
|
430
|
+
@d.child_process_execute(:t12, "ruby -e 'sleep 10'", external_encoding: 'utf-8', internal_encoding: 'ascii-8bit') do |r, w|
|
431
|
+
m.lock
|
432
|
+
ran = true
|
433
|
+
encodings << r.internal_encoding
|
434
|
+
encodings << w.internal_encoding
|
435
|
+
m.unlock
|
436
|
+
end
|
437
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
438
|
+
m.lock
|
439
|
+
assert_equal Encoding::ASCII_8BIT, encodings[0]
|
440
|
+
assert_equal Encoding::ASCII_8BIT, encodings[1]
|
441
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
442
|
+
end
|
443
|
+
end
|
444
|
+
|
445
|
+
test 'can convert encodings from ascii-8bit to utf-8' do
|
446
|
+
m = Mutex.new
|
447
|
+
str = nil
|
448
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
449
|
+
ran = false
|
450
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xA4\xB5\xA4\xC8\xA4\xB7"']
|
451
|
+
@d.child_process_execute(:t13, "ruby", arguments: args, external_encoding: 'euc-jp', internal_encoding: 'windows-31j', mode: [:read]) do |io|
|
452
|
+
m.lock
|
453
|
+
ran = true
|
454
|
+
str = io.read
|
455
|
+
m.unlock
|
456
|
+
end
|
457
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
458
|
+
m.lock
|
459
|
+
assert_equal Encoding.find('windows-31j'), str.encoding
|
460
|
+
expected = "さとし".encode('windows-31j')
|
461
|
+
assert_equal expected, str
|
462
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
463
|
+
end
|
464
|
+
end
|
465
|
+
|
466
|
+
test 'can scrub characters without exceptions' do
|
467
|
+
m = Mutex.new
|
468
|
+
str = nil
|
469
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
470
|
+
ran = false
|
471
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
472
|
+
@d.child_process_execute(:t13a, "ruby", arguments: args, mode: [:read]) do |io|
|
473
|
+
m.lock
|
474
|
+
ran = true
|
475
|
+
str = io.read
|
476
|
+
m.unlock
|
477
|
+
end
|
478
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
479
|
+
m.lock
|
480
|
+
assert_equal Encoding.find('utf-8'), str.encoding
|
481
|
+
expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
|
482
|
+
assert_equal expected, str
|
483
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
484
|
+
end
|
485
|
+
end
|
486
|
+
|
487
|
+
test 'can scrub characters without exceptions and replace specified chars' do
|
488
|
+
m = Mutex.new
|
489
|
+
str = nil
|
490
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
491
|
+
ran = false
|
492
|
+
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
493
|
+
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
|
494
|
+
m.lock
|
495
|
+
ran = true
|
496
|
+
str = io.read
|
497
|
+
m.unlock
|
498
|
+
end
|
499
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
500
|
+
m.lock
|
501
|
+
assert_equal Encoding.find('utf-8'), str.encoding
|
502
|
+
expected = "??\x00??".force_encoding("utf-8")
|
503
|
+
assert_equal expected, str
|
504
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
505
|
+
end
|
506
|
+
end
|
507
|
+
|
508
|
+
unless Fluent.windows?
|
509
|
+
test 'can specify subprocess name' do
|
510
|
+
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
511
|
+
process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
|
512
|
+
Process.kill(:TERM, io.pid) rescue nil
|
513
|
+
io.close rescue nil
|
514
|
+
|
515
|
+
# Does TravisCI prohibit process renaming?
|
516
|
+
# This test will be passed in such environment
|
517
|
+
pend unless process_naming_enabled
|
518
|
+
|
519
|
+
m = Mutex.new
|
520
|
+
pids = []
|
521
|
+
proc_lines = []
|
522
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
523
|
+
ran = false
|
524
|
+
@d.child_process_execute(:t14, "ruby", arguments:['-e', 'sleep 10; puts "hello"'], subprocess_name: "sleeeeeeeeeper", mode: [:read]) do |readio|
|
525
|
+
m.lock
|
526
|
+
ran = true
|
527
|
+
pids << @d.child_process_id
|
528
|
+
proc_lines += open("|ps"){|_io| _io.readlines }
|
529
|
+
m.unlock
|
530
|
+
readio.read
|
531
|
+
end
|
532
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
533
|
+
m.lock
|
534
|
+
pid = pids.first
|
535
|
+
# 51358 ttys001 0:00.00 sleeper -e sleep 10
|
536
|
+
assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[3] == "sleeeeeeeeeper" }
|
537
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
538
|
+
end
|
539
|
+
end
|
540
|
+
end
|
541
|
+
|
542
|
+
test 'can set ENV variables' do
|
543
|
+
m = Mutex.new
|
544
|
+
str = nil
|
545
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
546
|
+
ran = false
|
547
|
+
args = ['-e', 'puts ENV["testing_child_process"]']
|
548
|
+
@d.child_process_execute(:t15a, "ruby", arguments: args, mode: [:read], env: {'testing_child_process' => 'Yes! True!'}) do |io|
|
549
|
+
m.lock
|
550
|
+
ran = true
|
551
|
+
str = io.read
|
552
|
+
m.unlock
|
553
|
+
end
|
554
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
555
|
+
m.lock
|
556
|
+
expected = "Yes! True!\n"
|
557
|
+
assert_equal expected, str
|
558
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
559
|
+
end
|
560
|
+
end
|
561
|
+
|
562
|
+
test 'can unset ENV variables of Fluentd process' do
|
563
|
+
m = Mutex.new
|
564
|
+
str = nil
|
565
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
566
|
+
current_env_path = ENV['PATH']
|
567
|
+
ran = false
|
568
|
+
args = ['-e', 'puts ENV["testing_child_process1"].to_s + ENV["testing_child_process2"].to_s']
|
569
|
+
ENV['testing_child_process1'] = "No! False!"
|
570
|
+
@d.child_process_execute(:t15b, "ruby", arguments: args, mode: [:read], unsetenv: true, env: {'testing_child_process2' => 'Yes! True!', 'PATH' => current_env_path}) do |io|
|
571
|
+
m.lock
|
572
|
+
ran = true
|
573
|
+
str = io.read
|
574
|
+
m.unlock
|
575
|
+
end
|
576
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
577
|
+
m.lock
|
578
|
+
expected = "Yes! True!\n"
|
579
|
+
assert_equal expected, str
|
580
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
581
|
+
end
|
582
|
+
end
|
583
|
+
|
584
|
+
unless Fluent.windows?
|
585
|
+
test 'can change working directory' do
|
586
|
+
# check my real /tmp directory (for mac)
|
587
|
+
cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
|
588
|
+
mytmpdir = open(cmd){|io| io.read.chomp }
|
589
|
+
|
590
|
+
m = Mutex.new
|
591
|
+
str = nil
|
592
|
+
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
593
|
+
ran = false
|
594
|
+
args = ['-e', 'puts Dir.pwd']
|
595
|
+
@d.child_process_execute(:t16, "ruby", arguments: args, mode: [:read], chdir: "/tmp") do |io|
|
596
|
+
m.lock
|
597
|
+
ran = true
|
598
|
+
str = io.read.chomp
|
599
|
+
m.unlock
|
600
|
+
end
|
601
|
+
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
602
|
+
m.lock
|
603
|
+
assert_equal mytmpdir, str
|
604
|
+
@d.stop; @d.shutdown; @d.close; @d.terminate
|
605
|
+
end
|
606
|
+
end
|
607
|
+
end
|
608
|
+
end
|