fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +1 -0
- data/SECURITY.md +2 -2
- data/fluent.conf +14 -14
- data/lib/fluent/command/cap_ctl.rb +4 -4
- data/lib/fluent/compat/call_super_mixin.rb +3 -3
- data/lib/fluent/compat/propagate_default.rb +4 -4
- data/lib/fluent/config/yaml_parser/parser.rb +4 -0
- data/lib/fluent/log/console_adapter.rb +4 -2
- data/lib/fluent/plugin/in_exec.rb +14 -2
- data/lib/fluent/plugin/in_http.rb +1 -1
- data/lib/fluent/plugin/in_sample.rb +13 -7
- data/lib/fluent/plugin/in_tail.rb +65 -23
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +8 -0
- data/lib/fluent/plugin/out_http.rb +12 -0
- data/lib/fluent/plugin/parser_json.rb +4 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
- metadata +25 -472
- data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
- data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
- data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
- data/.github/ISSUE_TEMPLATE/config.yml +0 -5
- data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
- data/.github/ISSUE_TEMPLATE.md +0 -17
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
- data/.github/workflows/stale-actions.yml +0 -24
- data/.github/workflows/test-ruby-head.yml +0 -31
- data/.github/workflows/test.yml +0 -32
- data/.gitignore +0 -30
- data/Gemfile +0 -9
- data/fluentd.gemspec +0 -62
- data/test/command/test_binlog_reader.rb +0 -362
- data/test/command/test_ca_generate.rb +0 -70
- data/test/command/test_cap_ctl.rb +0 -100
- data/test/command/test_cat.rb +0 -128
- data/test/command/test_ctl.rb +0 -56
- data/test/command/test_fluentd.rb +0 -1291
- data/test/command/test_plugin_config_formatter.rb +0 -397
- data/test/command/test_plugin_generator.rb +0 -109
- data/test/compat/test_calls_super.rb +0 -166
- data/test/compat/test_parser.rb +0 -92
- data/test/config/assertions.rb +0 -42
- data/test/config/test_config_parser.rb +0 -551
- data/test/config/test_configurable.rb +0 -1784
- data/test/config/test_configure_proxy.rb +0 -604
- data/test/config/test_dsl.rb +0 -415
- data/test/config/test_element.rb +0 -518
- data/test/config/test_literal_parser.rb +0 -309
- data/test/config/test_plugin_configuration.rb +0 -56
- data/test/config/test_section.rb +0 -191
- data/test/config/test_system_config.rb +0 -195
- data/test/config/test_types.rb +0 -408
- data/test/counter/test_client.rb +0 -563
- data/test/counter/test_error.rb +0 -44
- data/test/counter/test_mutex_hash.rb +0 -179
- data/test/counter/test_server.rb +0 -589
- data/test/counter/test_store.rb +0 -258
- data/test/counter/test_validator.rb +0 -137
- data/test/helper.rb +0 -155
- data/test/helpers/fuzzy_assert.rb +0 -89
- data/test/helpers/process_extenstion.rb +0 -33
- data/test/log/test_console_adapter.rb +0 -117
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/log_numeric/01.log +0 -0
- data/test/plugin/data/log_numeric/02.log +0 -0
- data/test/plugin/data/log_numeric/12.log +0 -0
- data/test/plugin/data/log_numeric/14.log +0 -0
- data/test/plugin/data/sd_file/config +0 -11
- data/test/plugin/data/sd_file/config.json +0 -17
- data/test/plugin/data/sd_file/config.yaml +0 -11
- data/test/plugin/data/sd_file/config.yml +0 -11
- data/test/plugin/data/sd_file/invalid_config.yml +0 -7
- data/test/plugin/in_tail/test_fifo.rb +0 -121
- data/test/plugin/in_tail/test_io_handler.rb +0 -150
- data/test/plugin/in_tail/test_position_file.rb +0 -346
- data/test/plugin/out_forward/test_ack_handler.rb +0 -140
- data/test/plugin/out_forward/test_connection_manager.rb +0 -145
- data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
- data/test/plugin/out_forward/test_load_balancer.rb +0 -106
- data/test/plugin/out_forward/test_socket_cache.rb +0 -174
- data/test/plugin/test_bare_output.rb +0 -131
- data/test/plugin/test_base.rb +0 -247
- data/test/plugin/test_buf_file.rb +0 -1314
- data/test/plugin/test_buf_file_single.rb +0 -898
- data/test/plugin/test_buf_memory.rb +0 -42
- data/test/plugin/test_buffer.rb +0 -1493
- data/test/plugin/test_buffer_chunk.rb +0 -209
- data/test/plugin/test_buffer_file_chunk.rb +0 -871
- data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
- data/test/plugin/test_buffer_memory_chunk.rb +0 -339
- data/test/plugin/test_compressable.rb +0 -87
- data/test/plugin/test_file_util.rb +0 -96
- data/test/plugin/test_filter.rb +0 -368
- data/test/plugin/test_filter_grep.rb +0 -697
- data/test/plugin/test_filter_parser.rb +0 -731
- data/test/plugin/test_filter_record_transformer.rb +0 -577
- data/test/plugin/test_filter_stdout.rb +0 -207
- data/test/plugin/test_formatter_csv.rb +0 -136
- data/test/plugin/test_formatter_hash.rb +0 -38
- data/test/plugin/test_formatter_json.rb +0 -61
- data/test/plugin/test_formatter_ltsv.rb +0 -70
- data/test/plugin/test_formatter_msgpack.rb +0 -28
- data/test/plugin/test_formatter_out_file.rb +0 -116
- data/test/plugin/test_formatter_single_value.rb +0 -44
- data/test/plugin/test_formatter_tsv.rb +0 -76
- data/test/plugin/test_in_debug_agent.rb +0 -49
- data/test/plugin/test_in_exec.rb +0 -261
- data/test/plugin/test_in_forward.rb +0 -1178
- data/test/plugin/test_in_gc_stat.rb +0 -62
- data/test/plugin/test_in_http.rb +0 -1124
- data/test/plugin/test_in_monitor_agent.rb +0 -922
- data/test/plugin/test_in_object_space.rb +0 -66
- data/test/plugin/test_in_sample.rb +0 -190
- data/test/plugin/test_in_syslog.rb +0 -505
- data/test/plugin/test_in_tail.rb +0 -3429
- data/test/plugin/test_in_tcp.rb +0 -328
- data/test/plugin/test_in_udp.rb +0 -296
- data/test/plugin/test_in_unix.rb +0 -181
- data/test/plugin/test_input.rb +0 -137
- data/test/plugin/test_metadata.rb +0 -89
- data/test/plugin/test_metrics.rb +0 -294
- data/test/plugin/test_metrics_local.rb +0 -96
- data/test/plugin/test_multi_output.rb +0 -204
- data/test/plugin/test_out_copy.rb +0 -308
- data/test/plugin/test_out_exec.rb +0 -312
- data/test/plugin/test_out_exec_filter.rb +0 -606
- data/test/plugin/test_out_file.rb +0 -1038
- data/test/plugin/test_out_forward.rb +0 -1349
- data/test/plugin/test_out_http.rb +0 -557
- data/test/plugin/test_out_null.rb +0 -105
- data/test/plugin/test_out_relabel.rb +0 -28
- data/test/plugin/test_out_roundrobin.rb +0 -146
- data/test/plugin/test_out_secondary_file.rb +0 -458
- data/test/plugin/test_out_stdout.rb +0 -205
- data/test/plugin/test_out_stream.rb +0 -103
- data/test/plugin/test_output.rb +0 -1334
- data/test/plugin/test_output_as_buffered.rb +0 -2024
- data/test/plugin/test_output_as_buffered_backup.rb +0 -363
- data/test/plugin/test_output_as_buffered_compress.rb +0 -179
- data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
- data/test/plugin/test_output_as_buffered_retries.rb +0 -966
- data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
- data/test/plugin/test_output_as_standard.rb +0 -374
- data/test/plugin/test_owned_by.rb +0 -34
- data/test/plugin/test_parser.rb +0 -399
- data/test/plugin/test_parser_apache.rb +0 -42
- data/test/plugin/test_parser_apache2.rb +0 -47
- data/test/plugin/test_parser_apache_error.rb +0 -45
- data/test/plugin/test_parser_csv.rb +0 -200
- data/test/plugin/test_parser_json.rb +0 -244
- data/test/plugin/test_parser_labeled_tsv.rb +0 -160
- data/test/plugin/test_parser_msgpack.rb +0 -127
- data/test/plugin/test_parser_multiline.rb +0 -111
- data/test/plugin/test_parser_nginx.rb +0 -88
- data/test/plugin/test_parser_none.rb +0 -52
- data/test/plugin/test_parser_regexp.rb +0 -284
- data/test/plugin/test_parser_syslog.rb +0 -650
- data/test/plugin/test_parser_tsv.rb +0 -122
- data/test/plugin/test_sd_file.rb +0 -228
- data/test/plugin/test_sd_srv.rb +0 -230
- data/test/plugin/test_storage.rb +0 -166
- data/test/plugin/test_storage_local.rb +0 -335
- data/test/plugin/test_string_util.rb +0 -26
- data/test/plugin_helper/data/cert/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
- data/test/plugin_helper/data/cert/cert.pem +0 -19
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
- data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
- data/test/plugin_helper/http_server/test_app.rb +0 -65
- data/test/plugin_helper/http_server/test_route.rb +0 -32
- data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
- data/test/plugin_helper/test_cert_option.rb +0 -25
- data/test/plugin_helper/test_child_process.rb +0 -862
- data/test/plugin_helper/test_compat_parameters.rb +0 -358
- data/test/plugin_helper/test_event_emitter.rb +0 -80
- data/test/plugin_helper/test_event_loop.rb +0 -52
- data/test/plugin_helper/test_extract.rb +0 -194
- data/test/plugin_helper/test_formatter.rb +0 -255
- data/test/plugin_helper/test_http_server_helper.rb +0 -372
- data/test/plugin_helper/test_inject.rb +0 -561
- data/test/plugin_helper/test_metrics.rb +0 -137
- data/test/plugin_helper/test_parser.rb +0 -264
- data/test/plugin_helper/test_record_accessor.rb +0 -238
- data/test/plugin_helper/test_retry_state.rb +0 -1006
- data/test/plugin_helper/test_server.rb +0 -1895
- data/test/plugin_helper/test_service_discovery.rb +0 -165
- data/test/plugin_helper/test_socket.rb +0 -146
- data/test/plugin_helper/test_storage.rb +0 -542
- data/test/plugin_helper/test_thread.rb +0 -164
- data/test/plugin_helper/test_timer.rb +0 -130
- data/test/scripts/exec_script.rb +0 -32
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
- data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
- data/test/scripts/fluent/plugin/out_test.rb +0 -81
- data/test/scripts/fluent/plugin/out_test2.rb +0 -80
- data/test/scripts/fluent/plugin/parser_known.rb +0 -4
- data/test/test_capability.rb +0 -74
- data/test/test_clock.rb +0 -164
- data/test/test_config.rb +0 -369
- data/test/test_configdsl.rb +0 -148
- data/test/test_daemonizer.rb +0 -91
- data/test/test_engine.rb +0 -203
- data/test/test_event.rb +0 -531
- data/test/test_event_router.rb +0 -348
- data/test/test_event_time.rb +0 -199
- data/test/test_file_wrapper.rb +0 -53
- data/test/test_filter.rb +0 -121
- data/test/test_fluent_log_event_router.rb +0 -99
- data/test/test_formatter.rb +0 -369
- data/test/test_input.rb +0 -31
- data/test/test_log.rb +0 -1076
- data/test/test_match.rb +0 -148
- data/test/test_mixin.rb +0 -351
- data/test/test_msgpack_factory.rb +0 -50
- data/test/test_oj_options.rb +0 -55
- data/test/test_output.rb +0 -278
- data/test/test_plugin.rb +0 -251
- data/test/test_plugin_classes.rb +0 -370
- data/test/test_plugin_helper.rb +0 -81
- data/test/test_plugin_id.rb +0 -119
- data/test/test_process.rb +0 -14
- data/test/test_root_agent.rb +0 -951
- data/test/test_static_config_analysis.rb +0 -177
- data/test/test_supervisor.rb +0 -821
- data/test/test_test_drivers.rb +0 -136
- data/test/test_time_formatter.rb +0 -301
- data/test/test_time_parser.rb +0 -362
- data/test/test_tls.rb +0 -65
- data/test/test_unique_id.rb +0 -47
- data/test/test_variable_store.rb +0 -65
@@ -1,606 +0,0 @@
|
|
1
|
-
require_relative '../helper'
|
2
|
-
require 'fluent/test/driver/output'
|
3
|
-
require 'fluent/plugin/out_exec_filter'
|
4
|
-
require 'fileutils'
|
5
|
-
|
6
|
-
class ExecFilterOutputTest < Test::Unit::TestCase
|
7
|
-
def setup
|
8
|
-
Fluent::Test.setup
|
9
|
-
end
|
10
|
-
|
11
|
-
CONFIG = %[
|
12
|
-
command cat
|
13
|
-
num_children 3
|
14
|
-
<inject>
|
15
|
-
tag_key tag
|
16
|
-
time_key time_in
|
17
|
-
time_type string
|
18
|
-
time_format %Y-%m-%d %H:%M:%S
|
19
|
-
</inject>
|
20
|
-
<format>
|
21
|
-
keys ["time_in", "tag", "k1"]
|
22
|
-
</format>
|
23
|
-
<parse>
|
24
|
-
keys ["time_out", "tag", "k2"]
|
25
|
-
</parse>
|
26
|
-
<extract>
|
27
|
-
tag_key tag
|
28
|
-
time_key time_out
|
29
|
-
time_type string
|
30
|
-
time_format %Y-%m-%d %H:%M:%S
|
31
|
-
</extract>
|
32
|
-
]
|
33
|
-
|
34
|
-
CONFIG_COMPAT = %[
|
35
|
-
command cat
|
36
|
-
in_keys time_in,tag,k1
|
37
|
-
out_keys time_out,tag,k2
|
38
|
-
tag_key tag
|
39
|
-
in_time_key time_in
|
40
|
-
out_time_key time_out
|
41
|
-
time_format %Y-%m-%d %H:%M:%S
|
42
|
-
localtime
|
43
|
-
num_children 3
|
44
|
-
]
|
45
|
-
|
46
|
-
def create_driver(conf)
|
47
|
-
Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)
|
48
|
-
end
|
49
|
-
|
50
|
-
SED_SUPPORT_UNBUFFERED_OPTION = ->(){
|
51
|
-
system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
|
52
|
-
$?.success?
|
53
|
-
}.call
|
54
|
-
SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''
|
55
|
-
|
56
|
-
data(
|
57
|
-
'with sections' => CONFIG,
|
58
|
-
'traditional' => CONFIG_COMPAT,
|
59
|
-
)
|
60
|
-
test 'configure' do |conf|
|
61
|
-
d = create_driver(conf)
|
62
|
-
|
63
|
-
assert_false d.instance.parser.estimate_current_event
|
64
|
-
|
65
|
-
assert_equal ["time_in","tag","k1"], d.instance.formatter.keys
|
66
|
-
assert_equal ["time_out","tag","k2"], d.instance.parser.keys
|
67
|
-
assert_equal "tag", d.instance.inject_config.tag_key
|
68
|
-
assert_equal "tag", d.instance.extract_config.tag_key
|
69
|
-
assert_equal "time_in", d.instance.inject_config.time_key
|
70
|
-
assert_equal "time_out", d.instance.extract_config.time_key
|
71
|
-
assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
|
72
|
-
assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
|
73
|
-
assert_equal true, d.instance.inject_config.localtime
|
74
|
-
assert_equal 3, d.instance.num_children
|
75
|
-
|
76
|
-
d = create_driver %[
|
77
|
-
command sed -l -e s/foo/bar/
|
78
|
-
in_keys time,k1
|
79
|
-
out_keys time,k2
|
80
|
-
tag xxx
|
81
|
-
time_key time
|
82
|
-
num_children 3
|
83
|
-
]
|
84
|
-
assert_equal "sed -l -e s/foo/bar/", d.instance.command
|
85
|
-
|
86
|
-
d = create_driver(conf + %[
|
87
|
-
remove_prefix before
|
88
|
-
add_prefix after
|
89
|
-
])
|
90
|
-
assert_equal "before", d.instance.remove_prefix
|
91
|
-
assert_equal "after" , d.instance.add_prefix
|
92
|
-
end
|
93
|
-
|
94
|
-
data(
|
95
|
-
'with sections' => CONFIG,
|
96
|
-
'traditional' => CONFIG_COMPAT,
|
97
|
-
)
|
98
|
-
test 'emit events with TSV format' do |conf|
|
99
|
-
d = create_driver(conf)
|
100
|
-
time = event_time("2011-01-02 13:14:15")
|
101
|
-
|
102
|
-
d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
|
103
|
-
# sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }
|
104
|
-
d.feed(time, {"k1"=>1})
|
105
|
-
d.feed(time, {"k1"=>2})
|
106
|
-
end
|
107
|
-
|
108
|
-
assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
|
109
|
-
assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
|
110
|
-
|
111
|
-
events = d.events
|
112
|
-
assert_equal 2, events.length
|
113
|
-
assert_equal_event_time time, events[0][1]
|
114
|
-
assert_equal ["test", time, {"k2"=>"1"}], events[0]
|
115
|
-
assert_equal_event_time time, events[1][1]
|
116
|
-
assert_equal ["test", time, {"k2"=>"2"}], events[1]
|
117
|
-
end
|
118
|
-
|
119
|
-
CONFIG_WITHOUT_TIME_FORMAT = %[
|
120
|
-
command cat
|
121
|
-
num_children 3
|
122
|
-
tag xxx
|
123
|
-
<inject>
|
124
|
-
time_key time
|
125
|
-
time_type unixtime
|
126
|
-
</inject>
|
127
|
-
<format>
|
128
|
-
keys time,k1
|
129
|
-
</format>
|
130
|
-
<parse>
|
131
|
-
keys time,k2
|
132
|
-
time_key time
|
133
|
-
time_type unixtime
|
134
|
-
</parse>
|
135
|
-
]
|
136
|
-
CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[
|
137
|
-
command cat
|
138
|
-
in_keys time,k1
|
139
|
-
out_keys time,k2
|
140
|
-
tag xxx
|
141
|
-
time_key time
|
142
|
-
num_children 3
|
143
|
-
]
|
144
|
-
|
145
|
-
data(
|
146
|
-
'with sections' => CONFIG_WITHOUT_TIME_FORMAT,
|
147
|
-
'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,
|
148
|
-
)
|
149
|
-
test 'emit events without time format configuration' do |conf|
|
150
|
-
d = create_driver(conf)
|
151
|
-
time = event_time("2011-01-02 13:14:15 +0900")
|
152
|
-
|
153
|
-
d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
|
154
|
-
d.feed(time, {"k1"=>1})
|
155
|
-
d.feed(time, {"k1"=>2})
|
156
|
-
end
|
157
|
-
|
158
|
-
assert_equal "1293941655\t1\n", d.formatted[0]
|
159
|
-
assert_equal "1293941655\t2\n", d.formatted[1]
|
160
|
-
|
161
|
-
events = d.events
|
162
|
-
assert_equal 2, events.length
|
163
|
-
assert_equal_event_time time, events[0][1]
|
164
|
-
assert_equal ["xxx", time, {"k2"=>"1"}], events[0]
|
165
|
-
assert_equal_event_time time, events[1][1]
|
166
|
-
assert_equal ["xxx", time, {"k2"=>"2"}], events[1]
|
167
|
-
end
|
168
|
-
|
169
|
-
CONFIG_TO_DO_GREP = %[
|
170
|
-
command grep --line-buffered -v poo
|
171
|
-
num_children 3
|
172
|
-
tag xxx
|
173
|
-
<inject>
|
174
|
-
time_key time
|
175
|
-
time_type unixtime
|
176
|
-
</inject>
|
177
|
-
<format>
|
178
|
-
keys time, val1
|
179
|
-
</format>
|
180
|
-
<parse>
|
181
|
-
keys time, val2
|
182
|
-
time_key time
|
183
|
-
time_type unixtime
|
184
|
-
</parse>
|
185
|
-
]
|
186
|
-
CONFIG_TO_DO_GREP_COMPAT = %[
|
187
|
-
command grep --line-buffered -v poo
|
188
|
-
in_keys time,val1
|
189
|
-
out_keys time,val2
|
190
|
-
tag xxx
|
191
|
-
time_key time
|
192
|
-
num_children 3
|
193
|
-
]
|
194
|
-
|
195
|
-
data(
|
196
|
-
'with sections' => CONFIG_TO_DO_GREP,
|
197
|
-
'traditional' => CONFIG_TO_DO_GREP_COMPAT,
|
198
|
-
)
|
199
|
-
test 'emit events through grep command' do |conf|
|
200
|
-
d = create_driver(conf)
|
201
|
-
time = event_time("2011-01-02 13:14:15 +0900")
|
202
|
-
|
203
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
|
204
|
-
d.feed(time, {"val1"=>"sed-ed value poo"})
|
205
|
-
d.feed(time, {"val1"=>"sed-ed value foo"})
|
206
|
-
end
|
207
|
-
|
208
|
-
assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
|
209
|
-
assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
|
210
|
-
|
211
|
-
events = d.events
|
212
|
-
assert_equal 1, events.length
|
213
|
-
assert_equal_event_time time, events[0][1]
|
214
|
-
assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], events[0]
|
215
|
-
end
|
216
|
-
|
217
|
-
CONFIG_TO_DO_SED = %[
|
218
|
-
command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
|
219
|
-
num_children 3
|
220
|
-
tag xxx
|
221
|
-
<inject>
|
222
|
-
time_key time
|
223
|
-
time_type unixtime
|
224
|
-
</inject>
|
225
|
-
<format>
|
226
|
-
keys time, val1
|
227
|
-
</format>
|
228
|
-
<parse>
|
229
|
-
keys time, val2
|
230
|
-
time_key time
|
231
|
-
time_type unixtime
|
232
|
-
</parse>
|
233
|
-
]
|
234
|
-
CONFIG_TO_DO_SED_COMPAT = %[
|
235
|
-
command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
|
236
|
-
in_keys time,val1
|
237
|
-
out_keys time,val2
|
238
|
-
tag xxx
|
239
|
-
time_key time
|
240
|
-
num_children 3
|
241
|
-
]
|
242
|
-
|
243
|
-
data(
|
244
|
-
'with sections' => CONFIG_TO_DO_SED,
|
245
|
-
'traditional' => CONFIG_TO_DO_SED_COMPAT,
|
246
|
-
)
|
247
|
-
test 'emit events through sed command' do |conf|
|
248
|
-
d = create_driver(conf)
|
249
|
-
time = event_time("2011-01-02 13:14:15 +0900")
|
250
|
-
|
251
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
|
252
|
-
d.feed(time, {"val1"=>"sed-ed value poo"})
|
253
|
-
d.feed(time, {"val1"=>"sed-ed value foo"})
|
254
|
-
end
|
255
|
-
|
256
|
-
assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
|
257
|
-
assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
|
258
|
-
|
259
|
-
events = d.events
|
260
|
-
assert_equal 2, events.length
|
261
|
-
assert_equal_event_time time, events[0][1]
|
262
|
-
assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], events[0]
|
263
|
-
assert_equal_event_time time, events[1][1]
|
264
|
-
assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], events[1]
|
265
|
-
end
|
266
|
-
|
267
|
-
CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[
|
268
|
-
command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
|
269
|
-
num_children 3
|
270
|
-
remove_prefix input
|
271
|
-
add_prefix output
|
272
|
-
<inject>
|
273
|
-
tag_key tag
|
274
|
-
time_key time
|
275
|
-
</inject>
|
276
|
-
<format>
|
277
|
-
keys tag, time, val1
|
278
|
-
</format>
|
279
|
-
<parse>
|
280
|
-
keys tag, time, val2
|
281
|
-
</parse>
|
282
|
-
<extract>
|
283
|
-
tag_key tag
|
284
|
-
time_key time
|
285
|
-
</extract>
|
286
|
-
]
|
287
|
-
CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[
|
288
|
-
command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
|
289
|
-
in_keys tag,time,val1
|
290
|
-
remove_prefix input
|
291
|
-
out_keys tag,time,val2
|
292
|
-
add_prefix output
|
293
|
-
tag_key tag
|
294
|
-
time_key time
|
295
|
-
num_children 3
|
296
|
-
]
|
297
|
-
|
298
|
-
data(
|
299
|
-
'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,
|
300
|
-
'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,
|
301
|
-
)
|
302
|
-
test 'emit events with add/remove tag prefix' do |conf|
|
303
|
-
d = create_driver(conf)
|
304
|
-
|
305
|
-
time = event_time("2011-01-02 13:14:15 +0900")
|
306
|
-
|
307
|
-
d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do
|
308
|
-
d.feed(time, {"val1"=>"sed-ed value foo"})
|
309
|
-
d.feed(time, {"val1"=>"sed-ed value poo"})
|
310
|
-
end
|
311
|
-
|
312
|
-
assert_equal "test\t1293941655\tsed-ed value foo\n", d.formatted[0]
|
313
|
-
assert_equal "test\t1293941655\tsed-ed value poo\n", d.formatted[1]
|
314
|
-
|
315
|
-
events = d.events
|
316
|
-
assert_equal 2, events.length
|
317
|
-
assert_equal_event_time time, events[0][1]
|
318
|
-
assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], events[0]
|
319
|
-
assert_equal_event_time time, events[1][1]
|
320
|
-
assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], events[1]
|
321
|
-
end
|
322
|
-
|
323
|
-
CONFIG_JSON = %[
|
324
|
-
command cat
|
325
|
-
<format>
|
326
|
-
@type tsv
|
327
|
-
keys message
|
328
|
-
</format>
|
329
|
-
<parse>
|
330
|
-
@type json
|
331
|
-
stream_buffer_size 1
|
332
|
-
</parse>
|
333
|
-
<extract>
|
334
|
-
tag_key tag
|
335
|
-
time_key time
|
336
|
-
</extract>
|
337
|
-
]
|
338
|
-
CONFIG_JSON_COMPAT = %[
|
339
|
-
command cat
|
340
|
-
in_keys message
|
341
|
-
out_format json
|
342
|
-
out_stream_buffer_size 1
|
343
|
-
time_key time
|
344
|
-
tag_key tag
|
345
|
-
]
|
346
|
-
|
347
|
-
data(
|
348
|
-
'with sections' => CONFIG_JSON,
|
349
|
-
'traditional' => CONFIG_JSON_COMPAT,
|
350
|
-
)
|
351
|
-
test 'using json format' do |conf|
|
352
|
-
d = create_driver(conf)
|
353
|
-
time = event_time("2011-01-02 13:14:15 +0900")
|
354
|
-
|
355
|
-
d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
|
356
|
-
i = d.instance
|
357
|
-
assert{ i.router }
|
358
|
-
d.feed(time, {"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]})
|
359
|
-
end
|
360
|
-
|
361
|
-
assert_equal '{"time":1293941655,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
|
362
|
-
|
363
|
-
events = d.events
|
364
|
-
assert_equal 1, events.length
|
365
|
-
assert_equal_event_time time, events[0][1]
|
366
|
-
assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
|
367
|
-
end
|
368
|
-
|
369
|
-
CONFIG_JSON_WITH_FLOAT_TIME = %[
|
370
|
-
command cat
|
371
|
-
<format>
|
372
|
-
@type tsv
|
373
|
-
keys message
|
374
|
-
</format>
|
375
|
-
<parse>
|
376
|
-
@type json
|
377
|
-
stream_buffer_size 1
|
378
|
-
</parse>
|
379
|
-
<extract>
|
380
|
-
tag_key tag
|
381
|
-
time_key time
|
382
|
-
</extract>
|
383
|
-
]
|
384
|
-
CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[
|
385
|
-
command cat
|
386
|
-
in_keys message
|
387
|
-
out_format json
|
388
|
-
out_stream_buffer_size 1
|
389
|
-
time_key time
|
390
|
-
tag_key tag
|
391
|
-
]
|
392
|
-
|
393
|
-
data(
|
394
|
-
'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,
|
395
|
-
'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,
|
396
|
-
)
|
397
|
-
test 'using json format with float time' do |conf|
|
398
|
-
d = create_driver(conf)
|
399
|
-
time = event_time("2011-01-02 13:14:15.123 +0900")
|
400
|
-
|
401
|
-
d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
|
402
|
-
d.feed(time + 10, {"message"=>%[{"time":#{time.sec}.#{time.nsec},"tag":"t1","k1":"v1"}]})
|
403
|
-
end
|
404
|
-
|
405
|
-
assert_equal '{"time":1293941655.123000000,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
|
406
|
-
|
407
|
-
events = d.events
|
408
|
-
assert_equal 1, events.length
|
409
|
-
assert_equal_event_time time, events[0][1]
|
410
|
-
assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
|
411
|
-
end
|
412
|
-
|
413
|
-
CONFIG_JSON_WITH_TIME_FORMAT = %[
|
414
|
-
command cat
|
415
|
-
<format>
|
416
|
-
@type tsv
|
417
|
-
keys message
|
418
|
-
</format>
|
419
|
-
<parse>
|
420
|
-
@type json
|
421
|
-
stream_buffer_size 1
|
422
|
-
</parse>
|
423
|
-
<extract>
|
424
|
-
tag_key tag
|
425
|
-
time_key time
|
426
|
-
time_type string
|
427
|
-
time_format %d/%b/%Y %H:%M:%S.%N %z
|
428
|
-
</extract>
|
429
|
-
]
|
430
|
-
CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[
|
431
|
-
command cat
|
432
|
-
in_keys message
|
433
|
-
out_format json
|
434
|
-
out_stream_buffer_size 1
|
435
|
-
time_key time
|
436
|
-
time_format %d/%b/%Y %H:%M:%S.%N %z
|
437
|
-
tag_key tag
|
438
|
-
]
|
439
|
-
|
440
|
-
data(
|
441
|
-
'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,
|
442
|
-
'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,
|
443
|
-
)
|
444
|
-
test 'using json format with custom time format' do |conf|
|
445
|
-
d = create_driver(conf)
|
446
|
-
time_str = "28/Feb/2013 12:00:00.123456789 +0900"
|
447
|
-
time = event_time(time_str, format: "%d/%b/%Y %H:%M:%S.%N %z")
|
448
|
-
|
449
|
-
d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
|
450
|
-
d.feed(time + 10, {"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]})
|
451
|
-
end
|
452
|
-
|
453
|
-
assert_equal '{"time":"28/Feb/2013 12:00:00.123456789 +0900","tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
|
454
|
-
|
455
|
-
events = d.events
|
456
|
-
assert_equal 1, events.length
|
457
|
-
assert_equal_event_time time, events[0][1]
|
458
|
-
assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
|
459
|
-
end
|
460
|
-
|
461
|
-
CONFIG_ROUND_ROBIN = %[
|
462
|
-
command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
|
463
|
-
num_children 2
|
464
|
-
<inject>
|
465
|
-
tag_key tag
|
466
|
-
time_key time_in
|
467
|
-
time_type string
|
468
|
-
time_format %Y-%m-%d %H:%M:%S
|
469
|
-
</inject>
|
470
|
-
<format>
|
471
|
-
keys ["time_in", "tag", "k1"]
|
472
|
-
</format>
|
473
|
-
<parse>
|
474
|
-
keys ["time_out", "tag", "k2", "child_pid"]
|
475
|
-
</parse>
|
476
|
-
<extract>
|
477
|
-
tag_key tag
|
478
|
-
time_key time_out
|
479
|
-
time_type string
|
480
|
-
time_format %Y-%m-%d %H:%M:%S
|
481
|
-
</extract>
|
482
|
-
]
|
483
|
-
CONFIG_ROUND_ROBIN_COMPAT = %[
|
484
|
-
command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
|
485
|
-
in_keys time_in,tag,k1
|
486
|
-
out_keys time_out,tag,k2,child_pid
|
487
|
-
tag_key tag
|
488
|
-
in_time_key time_in
|
489
|
-
out_time_key time_out
|
490
|
-
time_format %Y-%m-%d %H:%M:%S
|
491
|
-
localtime
|
492
|
-
num_children 2
|
493
|
-
]
|
494
|
-
|
495
|
-
data(
|
496
|
-
'with sections' => CONFIG_ROUND_ROBIN,
|
497
|
-
'traditional' => CONFIG_ROUND_ROBIN_COMPAT,
|
498
|
-
)
|
499
|
-
test 'using child processes by round robin' do |conf|
|
500
|
-
d = create_driver(conf)
|
501
|
-
time = event_time('2011-01-02 13:14:15')
|
502
|
-
|
503
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true, shutdown: false){ d.feed(time, {"k1" => 0}) }
|
504
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 1}) }
|
505
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 2}) }
|
506
|
-
d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 3}) }
|
507
|
-
|
508
|
-
assert_equal "2011-01-02 13:14:15\ttest\t0\n", d.formatted[0]
|
509
|
-
assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[1]
|
510
|
-
assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[2]
|
511
|
-
assert_equal "2011-01-02 13:14:15\ttest\t3\n", d.formatted[3]
|
512
|
-
|
513
|
-
events = d.events
|
514
|
-
assert_equal 4, events.length
|
515
|
-
|
516
|
-
pid_list = []
|
517
|
-
events.each do |event|
|
518
|
-
pid = event[2]['child_pid']
|
519
|
-
pid_list << pid unless pid_list.include?(pid)
|
520
|
-
end
|
521
|
-
assert_equal 2, pid_list.size, "the number of pids should be same with number of child processes: #{pid_list.inspect}"
|
522
|
-
|
523
|
-
assert_equal pid_list[0], events[0][2]['child_pid']
|
524
|
-
assert_equal pid_list[1], events[1][2]['child_pid']
|
525
|
-
assert_equal pid_list[0], events[2][2]['child_pid']
|
526
|
-
assert_equal pid_list[1], events[3][2]['child_pid']
|
527
|
-
|
528
|
-
ensure
|
529
|
-
d.run(start: false, shutdown: true)
|
530
|
-
end
|
531
|
-
|
532
|
-
# child process exits per 3 lines
|
533
|
-
CONFIG_RESPAWN = %[
|
534
|
-
command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
|
535
|
-
num_children 2
|
536
|
-
child_respawn -1
|
537
|
-
<inject>
|
538
|
-
tag_key tag
|
539
|
-
time_key time_in
|
540
|
-
time_type unixtime
|
541
|
-
</inject>
|
542
|
-
<format>
|
543
|
-
keys ["time_in", "tag", "k1"]
|
544
|
-
</format>
|
545
|
-
<parse>
|
546
|
-
keys ["time_out", "tag", "k2", "child_pid"]
|
547
|
-
</parse>
|
548
|
-
<extract>
|
549
|
-
tag_key tag
|
550
|
-
time_key time_out
|
551
|
-
time_type unixtime
|
552
|
-
</extract>
|
553
|
-
]
|
554
|
-
|
555
|
-
CONFIG_RESPAWN_COMPAT = %[
|
556
|
-
command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
|
557
|
-
num_children 2
|
558
|
-
child_respawn -1
|
559
|
-
in_keys time_in,tag,k1
|
560
|
-
out_keys time_out,tag,k2,child_pid
|
561
|
-
tag_key tag
|
562
|
-
in_time_key time_in
|
563
|
-
out_time_key time_out
|
564
|
-
# time_format %Y-%m-%d %H:%M:%S
|
565
|
-
# localtime
|
566
|
-
]
|
567
|
-
|
568
|
-
data(
|
569
|
-
'with sections' => CONFIG_RESPAWN,
|
570
|
-
'traditional' => CONFIG_RESPAWN_COMPAT,
|
571
|
-
)
|
572
|
-
test 'emit events via child processes which exits sometimes' do |conf|
|
573
|
-
d = create_driver(conf)
|
574
|
-
time = event_time("2011-01-02 13:14:15")
|
575
|
-
countup = 0
|
576
|
-
|
577
|
-
d.run(start: true, shutdown: false)
|
578
|
-
assert_equal 2, d.instance.instance_eval{ @_child_process_processes.size }
|
579
|
-
|
580
|
-
2.times do
|
581
|
-
d.run(default_tag: 'test', expect_emits: 3, timeout: 3, force_flush_retry: true, start: false, shutdown: false) do
|
582
|
-
d.feed(time, { "k1" => countup }); countup += 1
|
583
|
-
d.feed(time, { "k1" => countup }); countup += 1
|
584
|
-
d.feed(time, { "k1" => countup }); countup += 1
|
585
|
-
end
|
586
|
-
end
|
587
|
-
|
588
|
-
events = d.events
|
589
|
-
assert_equal 6, events.length
|
590
|
-
|
591
|
-
pid_list = []
|
592
|
-
events.each do |event|
|
593
|
-
pid = event[2]['child_pid']
|
594
|
-
pid_list << pid unless pid_list.include?(pid)
|
595
|
-
end
|
596
|
-
|
597
|
-
# the number of pids should be same with number of child processes
|
598
|
-
assert_equal 2, pid_list.size
|
599
|
-
logs = d.instance.log.out.logs
|
600
|
-
assert_equal 2, logs.count { |l| l.include?('child process exits with error code') }
|
601
|
-
assert_equal 2, logs.count { |l| l.include?('respawning child process') }
|
602
|
-
|
603
|
-
ensure
|
604
|
-
d.run(start: false, shutdown: true)
|
605
|
-
end
|
606
|
-
end
|