fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +1 -0
- data/SECURITY.md +2 -2
- data/fluent.conf +14 -14
- data/lib/fluent/command/cap_ctl.rb +4 -4
- data/lib/fluent/compat/call_super_mixin.rb +3 -3
- data/lib/fluent/compat/propagate_default.rb +4 -4
- data/lib/fluent/config/yaml_parser/parser.rb +4 -0
- data/lib/fluent/log/console_adapter.rb +4 -2
- data/lib/fluent/plugin/in_exec.rb +14 -2
- data/lib/fluent/plugin/in_http.rb +1 -1
- data/lib/fluent/plugin/in_sample.rb +13 -7
- data/lib/fluent/plugin/in_tail.rb +65 -23
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +8 -0
- data/lib/fluent/plugin/out_http.rb +12 -0
- data/lib/fluent/plugin/parser_json.rb +4 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
- metadata +25 -472
- data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
- data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
- data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
- data/.github/ISSUE_TEMPLATE/config.yml +0 -5
- data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
- data/.github/ISSUE_TEMPLATE.md +0 -17
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
- data/.github/workflows/stale-actions.yml +0 -24
- data/.github/workflows/test-ruby-head.yml +0 -31
- data/.github/workflows/test.yml +0 -32
- data/.gitignore +0 -30
- data/Gemfile +0 -9
- data/fluentd.gemspec +0 -62
- data/test/command/test_binlog_reader.rb +0 -362
- data/test/command/test_ca_generate.rb +0 -70
- data/test/command/test_cap_ctl.rb +0 -100
- data/test/command/test_cat.rb +0 -128
- data/test/command/test_ctl.rb +0 -56
- data/test/command/test_fluentd.rb +0 -1291
- data/test/command/test_plugin_config_formatter.rb +0 -397
- data/test/command/test_plugin_generator.rb +0 -109
- data/test/compat/test_calls_super.rb +0 -166
- data/test/compat/test_parser.rb +0 -92
- data/test/config/assertions.rb +0 -42
- data/test/config/test_config_parser.rb +0 -551
- data/test/config/test_configurable.rb +0 -1784
- data/test/config/test_configure_proxy.rb +0 -604
- data/test/config/test_dsl.rb +0 -415
- data/test/config/test_element.rb +0 -518
- data/test/config/test_literal_parser.rb +0 -309
- data/test/config/test_plugin_configuration.rb +0 -56
- data/test/config/test_section.rb +0 -191
- data/test/config/test_system_config.rb +0 -195
- data/test/config/test_types.rb +0 -408
- data/test/counter/test_client.rb +0 -563
- data/test/counter/test_error.rb +0 -44
- data/test/counter/test_mutex_hash.rb +0 -179
- data/test/counter/test_server.rb +0 -589
- data/test/counter/test_store.rb +0 -258
- data/test/counter/test_validator.rb +0 -137
- data/test/helper.rb +0 -155
- data/test/helpers/fuzzy_assert.rb +0 -89
- data/test/helpers/process_extenstion.rb +0 -33
- data/test/log/test_console_adapter.rb +0 -117
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/log_numeric/01.log +0 -0
- data/test/plugin/data/log_numeric/02.log +0 -0
- data/test/plugin/data/log_numeric/12.log +0 -0
- data/test/plugin/data/log_numeric/14.log +0 -0
- data/test/plugin/data/sd_file/config +0 -11
- data/test/plugin/data/sd_file/config.json +0 -17
- data/test/plugin/data/sd_file/config.yaml +0 -11
- data/test/plugin/data/sd_file/config.yml +0 -11
- data/test/plugin/data/sd_file/invalid_config.yml +0 -7
- data/test/plugin/in_tail/test_fifo.rb +0 -121
- data/test/plugin/in_tail/test_io_handler.rb +0 -150
- data/test/plugin/in_tail/test_position_file.rb +0 -346
- data/test/plugin/out_forward/test_ack_handler.rb +0 -140
- data/test/plugin/out_forward/test_connection_manager.rb +0 -145
- data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
- data/test/plugin/out_forward/test_load_balancer.rb +0 -106
- data/test/plugin/out_forward/test_socket_cache.rb +0 -174
- data/test/plugin/test_bare_output.rb +0 -131
- data/test/plugin/test_base.rb +0 -247
- data/test/plugin/test_buf_file.rb +0 -1314
- data/test/plugin/test_buf_file_single.rb +0 -898
- data/test/plugin/test_buf_memory.rb +0 -42
- data/test/plugin/test_buffer.rb +0 -1493
- data/test/plugin/test_buffer_chunk.rb +0 -209
- data/test/plugin/test_buffer_file_chunk.rb +0 -871
- data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
- data/test/plugin/test_buffer_memory_chunk.rb +0 -339
- data/test/plugin/test_compressable.rb +0 -87
- data/test/plugin/test_file_util.rb +0 -96
- data/test/plugin/test_filter.rb +0 -368
- data/test/plugin/test_filter_grep.rb +0 -697
- data/test/plugin/test_filter_parser.rb +0 -731
- data/test/plugin/test_filter_record_transformer.rb +0 -577
- data/test/plugin/test_filter_stdout.rb +0 -207
- data/test/plugin/test_formatter_csv.rb +0 -136
- data/test/plugin/test_formatter_hash.rb +0 -38
- data/test/plugin/test_formatter_json.rb +0 -61
- data/test/plugin/test_formatter_ltsv.rb +0 -70
- data/test/plugin/test_formatter_msgpack.rb +0 -28
- data/test/plugin/test_formatter_out_file.rb +0 -116
- data/test/plugin/test_formatter_single_value.rb +0 -44
- data/test/plugin/test_formatter_tsv.rb +0 -76
- data/test/plugin/test_in_debug_agent.rb +0 -49
- data/test/plugin/test_in_exec.rb +0 -261
- data/test/plugin/test_in_forward.rb +0 -1178
- data/test/plugin/test_in_gc_stat.rb +0 -62
- data/test/plugin/test_in_http.rb +0 -1124
- data/test/plugin/test_in_monitor_agent.rb +0 -922
- data/test/plugin/test_in_object_space.rb +0 -66
- data/test/plugin/test_in_sample.rb +0 -190
- data/test/plugin/test_in_syslog.rb +0 -505
- data/test/plugin/test_in_tail.rb +0 -3429
- data/test/plugin/test_in_tcp.rb +0 -328
- data/test/plugin/test_in_udp.rb +0 -296
- data/test/plugin/test_in_unix.rb +0 -181
- data/test/plugin/test_input.rb +0 -137
- data/test/plugin/test_metadata.rb +0 -89
- data/test/plugin/test_metrics.rb +0 -294
- data/test/plugin/test_metrics_local.rb +0 -96
- data/test/plugin/test_multi_output.rb +0 -204
- data/test/plugin/test_out_copy.rb +0 -308
- data/test/plugin/test_out_exec.rb +0 -312
- data/test/plugin/test_out_exec_filter.rb +0 -606
- data/test/plugin/test_out_file.rb +0 -1038
- data/test/plugin/test_out_forward.rb +0 -1349
- data/test/plugin/test_out_http.rb +0 -557
- data/test/plugin/test_out_null.rb +0 -105
- data/test/plugin/test_out_relabel.rb +0 -28
- data/test/plugin/test_out_roundrobin.rb +0 -146
- data/test/plugin/test_out_secondary_file.rb +0 -458
- data/test/plugin/test_out_stdout.rb +0 -205
- data/test/plugin/test_out_stream.rb +0 -103
- data/test/plugin/test_output.rb +0 -1334
- data/test/plugin/test_output_as_buffered.rb +0 -2024
- data/test/plugin/test_output_as_buffered_backup.rb +0 -363
- data/test/plugin/test_output_as_buffered_compress.rb +0 -179
- data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
- data/test/plugin/test_output_as_buffered_retries.rb +0 -966
- data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
- data/test/plugin/test_output_as_standard.rb +0 -374
- data/test/plugin/test_owned_by.rb +0 -34
- data/test/plugin/test_parser.rb +0 -399
- data/test/plugin/test_parser_apache.rb +0 -42
- data/test/plugin/test_parser_apache2.rb +0 -47
- data/test/plugin/test_parser_apache_error.rb +0 -45
- data/test/plugin/test_parser_csv.rb +0 -200
- data/test/plugin/test_parser_json.rb +0 -244
- data/test/plugin/test_parser_labeled_tsv.rb +0 -160
- data/test/plugin/test_parser_msgpack.rb +0 -127
- data/test/plugin/test_parser_multiline.rb +0 -111
- data/test/plugin/test_parser_nginx.rb +0 -88
- data/test/plugin/test_parser_none.rb +0 -52
- data/test/plugin/test_parser_regexp.rb +0 -284
- data/test/plugin/test_parser_syslog.rb +0 -650
- data/test/plugin/test_parser_tsv.rb +0 -122
- data/test/plugin/test_sd_file.rb +0 -228
- data/test/plugin/test_sd_srv.rb +0 -230
- data/test/plugin/test_storage.rb +0 -166
- data/test/plugin/test_storage_local.rb +0 -335
- data/test/plugin/test_string_util.rb +0 -26
- data/test/plugin_helper/data/cert/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
- data/test/plugin_helper/data/cert/cert.pem +0 -19
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
- data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
- data/test/plugin_helper/http_server/test_app.rb +0 -65
- data/test/plugin_helper/http_server/test_route.rb +0 -32
- data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
- data/test/plugin_helper/test_cert_option.rb +0 -25
- data/test/plugin_helper/test_child_process.rb +0 -862
- data/test/plugin_helper/test_compat_parameters.rb +0 -358
- data/test/plugin_helper/test_event_emitter.rb +0 -80
- data/test/plugin_helper/test_event_loop.rb +0 -52
- data/test/plugin_helper/test_extract.rb +0 -194
- data/test/plugin_helper/test_formatter.rb +0 -255
- data/test/plugin_helper/test_http_server_helper.rb +0 -372
- data/test/plugin_helper/test_inject.rb +0 -561
- data/test/plugin_helper/test_metrics.rb +0 -137
- data/test/plugin_helper/test_parser.rb +0 -264
- data/test/plugin_helper/test_record_accessor.rb +0 -238
- data/test/plugin_helper/test_retry_state.rb +0 -1006
- data/test/plugin_helper/test_server.rb +0 -1895
- data/test/plugin_helper/test_service_discovery.rb +0 -165
- data/test/plugin_helper/test_socket.rb +0 -146
- data/test/plugin_helper/test_storage.rb +0 -542
- data/test/plugin_helper/test_thread.rb +0 -164
- data/test/plugin_helper/test_timer.rb +0 -130
- data/test/scripts/exec_script.rb +0 -32
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
- data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
- data/test/scripts/fluent/plugin/out_test.rb +0 -81
- data/test/scripts/fluent/plugin/out_test2.rb +0 -80
- data/test/scripts/fluent/plugin/parser_known.rb +0 -4
- data/test/test_capability.rb +0 -74
- data/test/test_clock.rb +0 -164
- data/test/test_config.rb +0 -369
- data/test/test_configdsl.rb +0 -148
- data/test/test_daemonizer.rb +0 -91
- data/test/test_engine.rb +0 -203
- data/test/test_event.rb +0 -531
- data/test/test_event_router.rb +0 -348
- data/test/test_event_time.rb +0 -199
- data/test/test_file_wrapper.rb +0 -53
- data/test/test_filter.rb +0 -121
- data/test/test_fluent_log_event_router.rb +0 -99
- data/test/test_formatter.rb +0 -369
- data/test/test_input.rb +0 -31
- data/test/test_log.rb +0 -1076
- data/test/test_match.rb +0 -148
- data/test/test_mixin.rb +0 -351
- data/test/test_msgpack_factory.rb +0 -50
- data/test/test_oj_options.rb +0 -55
- data/test/test_output.rb +0 -278
- data/test/test_plugin.rb +0 -251
- data/test/test_plugin_classes.rb +0 -370
- data/test/test_plugin_helper.rb +0 -81
- data/test/test_plugin_id.rb +0 -119
- data/test/test_process.rb +0 -14
- data/test/test_root_agent.rb +0 -951
- data/test/test_static_config_analysis.rb +0 -177
- data/test/test_supervisor.rb +0 -821
- data/test/test_test_drivers.rb +0 -136
- data/test/test_time_formatter.rb +0 -301
- data/test/test_time_parser.rb +0 -362
- data/test/test_tls.rb +0 -65
- data/test/test_unique_id.rb +0 -47
- data/test/test_variable_store.rb +0 -65
@@ -1,898 +0,0 @@
|
|
1
|
-
require_relative '../helper'
|
2
|
-
require 'fluent/plugin/buf_file_single'
|
3
|
-
require 'fluent/plugin/output'
|
4
|
-
require 'fluent/unique_id'
|
5
|
-
require 'fluent/system_config'
|
6
|
-
require 'fluent/env'
|
7
|
-
require 'fluent/test/driver/output'
|
8
|
-
|
9
|
-
require 'msgpack'
|
10
|
-
|
11
|
-
module FluentPluginFileSingleBufferTest
|
12
|
-
class DummyOutputPlugin < Fluent::Plugin::Output
|
13
|
-
Fluent::Plugin.register_output('buf_file_single_test', self)
|
14
|
-
config_section :buffer do
|
15
|
-
config_set_default :@type, 'file_single'
|
16
|
-
end
|
17
|
-
def multi_workers_ready?
|
18
|
-
true
|
19
|
-
end
|
20
|
-
def write(chunk)
|
21
|
-
# drop
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
class DummyOutputMPPlugin < Fluent::Plugin::Output
|
26
|
-
Fluent::Plugin.register_output('buf_file_single_mp_test', self)
|
27
|
-
config_section :buffer do
|
28
|
-
config_set_default :@type, 'file_single'
|
29
|
-
end
|
30
|
-
def formatted_to_msgpack_binary?
|
31
|
-
true
|
32
|
-
end
|
33
|
-
def multi_workers_ready?
|
34
|
-
true
|
35
|
-
end
|
36
|
-
def write(chunk)
|
37
|
-
# drop
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
class FileSingleBufferTest < Test::Unit::TestCase
|
43
|
-
def metadata(timekey: nil, tag: 'testing', variables: nil)
|
44
|
-
Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
|
45
|
-
end
|
46
|
-
|
47
|
-
PATH = File.expand_path('../../tmp/buffer_file_single_dir', __FILE__)
|
48
|
-
TAG_CONF = %[
|
49
|
-
<buffer tag>
|
50
|
-
@type file_single
|
51
|
-
path #{PATH}
|
52
|
-
</buffer>
|
53
|
-
]
|
54
|
-
FIELD_CONF = %[
|
55
|
-
<buffer k>
|
56
|
-
@type file_single
|
57
|
-
path #{PATH}
|
58
|
-
</buffer>
|
59
|
-
]
|
60
|
-
|
61
|
-
setup do
|
62
|
-
Fluent::Test.setup
|
63
|
-
|
64
|
-
@d = nil
|
65
|
-
@bufdir = PATH
|
66
|
-
FileUtils.rm_rf(@bufdir) rescue nil
|
67
|
-
FileUtils.mkdir_p(@bufdir)
|
68
|
-
end
|
69
|
-
|
70
|
-
teardown do
|
71
|
-
FileUtils.rm_rf(@bufdir) rescue nil
|
72
|
-
end
|
73
|
-
|
74
|
-
def create_driver(conf = TAG_CONF, klass = FluentPluginFileSingleBufferTest::DummyOutputPlugin)
|
75
|
-
Fluent::Test::Driver::Output.new(klass).configure(conf)
|
76
|
-
end
|
77
|
-
|
78
|
-
sub_test_case 'configuration' do
|
79
|
-
test 'path has "fsb" prefix and "buf" suffix by default' do
|
80
|
-
@d = create_driver
|
81
|
-
p = @d.instance.buffer
|
82
|
-
assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path
|
83
|
-
end
|
84
|
-
|
85
|
-
data('text based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputPlugin, :text],
|
86
|
-
'msgpack based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputMPPlugin, :msgpack])
|
87
|
-
test 'detect chunk_format' do |param|
|
88
|
-
klass, expected = param
|
89
|
-
@d = create_driver(TAG_CONF, klass)
|
90
|
-
p = @d.instance.buffer
|
91
|
-
assert_equal expected, p.chunk_format
|
92
|
-
end
|
93
|
-
|
94
|
-
test '"prefix.*.suffix" path will be replaced with default' do
|
95
|
-
@d = create_driver(%[
|
96
|
-
<buffer tag>
|
97
|
-
@type file_single
|
98
|
-
path #{@bufdir}/foo.*.bar
|
99
|
-
</buffer>
|
100
|
-
])
|
101
|
-
p = @d.instance.buffer
|
102
|
-
assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
sub_test_case 'buffer configurations and workers' do
|
107
|
-
setup do
|
108
|
-
@d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
109
|
-
end
|
110
|
-
|
111
|
-
test 'enables multi worker configuration with unexisting directory path' do
|
112
|
-
FileUtils.rm_rf(@bufdir)
|
113
|
-
buf_conf = config_element('buffer', '', {'path' => @bufdir})
|
114
|
-
assert_nothing_raised do
|
115
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
|
116
|
-
@d.configure(config_element('ROOT', '', {}, [buf_conf]))
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
test 'enables multi worker configuration with existing directory path' do
|
122
|
-
FileUtils.mkdir_p @bufdir
|
123
|
-
buf_conf = config_element('buffer', '', {'path' => @bufdir})
|
124
|
-
assert_nothing_raised do
|
125
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
|
126
|
-
@d.configure(config_element('ROOT', '', {}, [buf_conf]))
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
test 'enables multi worker configuration with root dir' do
|
132
|
-
buf_conf = config_element('buffer', '')
|
133
|
-
assert_nothing_raised do
|
134
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
|
135
|
-
@d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
140
|
-
|
141
|
-
test 'raise config error when using same file path' do
|
142
|
-
d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
143
|
-
d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
144
|
-
Fluent::SystemConfig.overwrite_system_config({}) do
|
145
|
-
d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))
|
146
|
-
end
|
147
|
-
|
148
|
-
any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|
|
149
|
-
stub(klass).called_in_test? { false }
|
150
|
-
end
|
151
|
-
|
152
|
-
err = assert_raise(Fluent::ConfigError) do
|
153
|
-
Fluent::SystemConfig.overwrite_system_config({}) do
|
154
|
-
d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))
|
155
|
-
end
|
156
|
-
end
|
157
|
-
assert_match(/plugin already uses same buffer path/, err.message)
|
158
|
-
end
|
159
|
-
|
160
|
-
sub_test_case 'buffer plugin configured only with path' do
|
161
|
-
setup do
|
162
|
-
@bufpath = File.join(@bufdir, 'testbuf.*.buf')
|
163
|
-
FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
|
164
|
-
|
165
|
-
@d = create_driver
|
166
|
-
@p = @d.instance.buffer
|
167
|
-
end
|
168
|
-
|
169
|
-
teardown do
|
170
|
-
if @p
|
171
|
-
@p.stop unless @p.stopped?
|
172
|
-
@p.before_shutdown unless @p.before_shutdown?
|
173
|
-
@p.shutdown unless @p.shutdown?
|
174
|
-
@p.after_shutdown unless @p.after_shutdown?
|
175
|
-
@p.close unless @p.closed?
|
176
|
-
@p.terminate unless @p.terminated?
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
180
|
-
test 'this is persistent plugin' do
|
181
|
-
assert @p.persistent?
|
182
|
-
end
|
183
|
-
|
184
|
-
test '#start creates directory for buffer chunks' do
|
185
|
-
@d = create_driver
|
186
|
-
@p = @d.instance.buffer
|
187
|
-
|
188
|
-
FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
|
189
|
-
assert !File.exist?(@bufdir)
|
190
|
-
|
191
|
-
@p.start
|
192
|
-
assert File.exist?(@bufdir)
|
193
|
-
assert { File.stat(@bufdir).mode.to_s(8).end_with?('755') }
|
194
|
-
end
|
195
|
-
|
196
|
-
test '#start creates directory for buffer chunks with specified permission' do
|
197
|
-
omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
|
198
|
-
|
199
|
-
@d = create_driver(%[
|
200
|
-
<buffer tag>
|
201
|
-
@type file_single
|
202
|
-
path #{PATH}
|
203
|
-
dir_permission 700
|
204
|
-
</buffer>
|
205
|
-
])
|
206
|
-
@p = @d.instance.buffer
|
207
|
-
|
208
|
-
FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
|
209
|
-
assert !File.exist?(@bufdir)
|
210
|
-
|
211
|
-
@p.start
|
212
|
-
assert File.exist?(@bufdir)
|
213
|
-
assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }
|
214
|
-
end
|
215
|
-
|
216
|
-
test '#start creates directory for buffer chunks with specified permission via system config' do
|
217
|
-
omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
|
218
|
-
|
219
|
-
sysconf = {'dir_permission' => '700'}
|
220
|
-
Fluent::SystemConfig.overwrite_system_config(sysconf) do
|
221
|
-
@d = create_driver
|
222
|
-
@p = @d.instance.buffer
|
223
|
-
|
224
|
-
FileUtils.rm_r @bufdir if File.exist?(@bufdir)
|
225
|
-
assert !File.exist?(@bufdir)
|
226
|
-
|
227
|
-
@p.start
|
228
|
-
assert File.exist?(@bufdir)
|
229
|
-
assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') }
|
230
|
-
end
|
231
|
-
end
|
232
|
-
|
233
|
-
test '#generate_chunk generates blank file chunk on path with unique_id and tag' do
|
234
|
-
FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)
|
235
|
-
|
236
|
-
m1 = metadata()
|
237
|
-
c1 = @p.generate_chunk(m1)
|
238
|
-
assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk
|
239
|
-
assert_equal m1, c1.metadata
|
240
|
-
assert c1.empty?
|
241
|
-
assert_equal :unstaged, c1.state
|
242
|
-
assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
|
243
|
-
assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
|
244
|
-
assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
|
245
|
-
|
246
|
-
c1.purge
|
247
|
-
end
|
248
|
-
|
249
|
-
test '#generate_chunk generates blank file chunk on path with unique_id and field key' do
|
250
|
-
FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir)
|
251
|
-
|
252
|
-
@d = create_driver(FIELD_CONF)
|
253
|
-
@p = @d.instance.buffer
|
254
|
-
|
255
|
-
m1 = metadata(tag: nil, variables: {:k => 'foo_bar'})
|
256
|
-
c1 = @p.generate_chunk(m1)
|
257
|
-
assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk
|
258
|
-
assert_equal m1, c1.metadata
|
259
|
-
assert c1.empty?
|
260
|
-
assert_equal :unstaged, c1.state
|
261
|
-
assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission
|
262
|
-
assert_equal File.join(@bufdir, "fsb.foo_bar.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path
|
263
|
-
|
264
|
-
c1.purge
|
265
|
-
end
|
266
|
-
|
267
|
-
test '#generate_chunk generates blank file chunk with specified permission' do
|
268
|
-
omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
|
269
|
-
|
270
|
-
@d = create_driver(%[
|
271
|
-
<buffer tag>
|
272
|
-
@type file_single
|
273
|
-
path #{PATH}
|
274
|
-
file_permission 600
|
275
|
-
</buffer>
|
276
|
-
])
|
277
|
-
@p = @d.instance.buffer
|
278
|
-
|
279
|
-
FileUtils.rm_r @bufdir if File.exist?(@bufdir)
|
280
|
-
assert !File.exist?(@bufdir)
|
281
|
-
|
282
|
-
@p.start
|
283
|
-
|
284
|
-
m = metadata()
|
285
|
-
c = @p.generate_chunk(m)
|
286
|
-
assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk
|
287
|
-
assert_equal m, c.metadata
|
288
|
-
assert c.empty?
|
289
|
-
assert_equal :unstaged, c.state
|
290
|
-
assert_equal 0600, c.permission
|
291
|
-
assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path
|
292
|
-
assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
|
293
|
-
|
294
|
-
c.purge
|
295
|
-
end
|
296
|
-
|
297
|
-
test '#generate_chunk generates blank file chunk with specified permission with system_config' do
|
298
|
-
omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
|
299
|
-
|
300
|
-
@d = create_driver(%[
|
301
|
-
<buffer tag>
|
302
|
-
@type file_single
|
303
|
-
path #{PATH}
|
304
|
-
</buffer>
|
305
|
-
])
|
306
|
-
@p = @d.instance.buffer
|
307
|
-
|
308
|
-
FileUtils.rm_r @bufdir if File.exist?(@bufdir)
|
309
|
-
assert !File.exist?(@bufdir)
|
310
|
-
|
311
|
-
@p.start
|
312
|
-
|
313
|
-
m = metadata()
|
314
|
-
c = nil
|
315
|
-
Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do
|
316
|
-
c = @p.generate_chunk(m)
|
317
|
-
end
|
318
|
-
assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk
|
319
|
-
assert_equal m, c.metadata
|
320
|
-
assert c.empty?
|
321
|
-
assert_equal :unstaged, c.state
|
322
|
-
assert_equal 0700, c.permission
|
323
|
-
assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path
|
324
|
-
assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }
|
325
|
-
|
326
|
-
c.purge
|
327
|
-
end
|
328
|
-
end
|
329
|
-
|
330
|
-
sub_test_case 'configured with system root directory and plugin @id' do
|
331
|
-
setup do
|
332
|
-
@root_dir = File.expand_path('../../tmp/buffer_file_single_root', __FILE__)
|
333
|
-
FileUtils.rm_rf(@root_dir)
|
334
|
-
|
335
|
-
@d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
336
|
-
@p = nil
|
337
|
-
end
|
338
|
-
|
339
|
-
teardown do
|
340
|
-
if @p
|
341
|
-
@p.stop unless @p.stopped?
|
342
|
-
@p.before_shutdown unless @p.before_shutdown?
|
343
|
-
@p.shutdown unless @p.shutdown?
|
344
|
-
@p.after_shutdown unless @p.after_shutdown?
|
345
|
-
@p.close unless @p.closed?
|
346
|
-
@p.terminate unless @p.terminated?
|
347
|
-
end
|
348
|
-
end
|
349
|
-
|
350
|
-
test '#start creates directory for buffer chunks' do
|
351
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
|
352
|
-
@d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [config_element('buffer', '', {})]))
|
353
|
-
@p = @d.buffer
|
354
|
-
end
|
355
|
-
|
356
|
-
expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "fsb.*.buf")
|
357
|
-
expected_buffer_dir = File.dirname(expected_buffer_path)
|
358
|
-
assert_equal expected_buffer_path, @d.buffer.path
|
359
|
-
assert_false Dir.exist?(expected_buffer_dir)
|
360
|
-
|
361
|
-
@p.start
|
362
|
-
assert Dir.exist?(expected_buffer_dir)
|
363
|
-
end
|
364
|
-
end
|
365
|
-
|
366
|
-
sub_test_case 'buffer plugin configuration errors' do
|
367
|
-
data('tag and key' => 'tag,key',
|
368
|
-
'multiple keys' => 'key1,key2')
|
369
|
-
test 'invalid chunk keys' do |param|
|
370
|
-
assert_raise Fluent::ConfigError do
|
371
|
-
@d = create_driver(%[
|
372
|
-
<buffer #{param}>
|
373
|
-
@type file_single
|
374
|
-
path #{PATH}
|
375
|
-
calc_num_records false
|
376
|
-
</buffer>
|
377
|
-
])
|
378
|
-
end
|
379
|
-
end
|
380
|
-
|
381
|
-
test 'path is not specified' do
|
382
|
-
assert_raise Fluent::ConfigError do
|
383
|
-
@d = create_driver(%[
|
384
|
-
<buffer tag>
|
385
|
-
@type file_single
|
386
|
-
</buffer>
|
387
|
-
])
|
388
|
-
end
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
sub_test_case 'there are no existing file chunks' do
|
393
|
-
setup do
|
394
|
-
FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir)
|
395
|
-
|
396
|
-
@d = create_driver
|
397
|
-
@p = @d.instance.buffer
|
398
|
-
@p.start
|
399
|
-
end
|
400
|
-
teardown do
|
401
|
-
if @p
|
402
|
-
@p.stop unless @p.stopped?
|
403
|
-
@p.before_shutdown unless @p.before_shutdown?
|
404
|
-
@p.shutdown unless @p.shutdown?
|
405
|
-
@p.after_shutdown unless @p.after_shutdown?
|
406
|
-
@p.close unless @p.closed?
|
407
|
-
@p.terminate unless @p.terminated?
|
408
|
-
end
|
409
|
-
if @bufdir
|
410
|
-
Dir.glob(File.join(@bufdir, '*')).each do |path|
|
411
|
-
next if ['.', '..'].include?(File.basename(path))
|
412
|
-
File.delete(path)
|
413
|
-
end
|
414
|
-
end
|
415
|
-
end
|
416
|
-
|
417
|
-
test '#resume returns empty buffer state' do
|
418
|
-
ary = @p.resume
|
419
|
-
assert_equal({}, ary[0])
|
420
|
-
assert_equal([], ary[1])
|
421
|
-
end
|
422
|
-
end
|
423
|
-
|
424
|
-
sub_test_case 'there are some existing file chunks' do
|
425
|
-
setup do
|
426
|
-
@c1id = Fluent::UniqueId.generate
|
427
|
-
p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
|
428
|
-
File.open(p1, 'wb') do |f|
|
429
|
-
f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
430
|
-
f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
431
|
-
f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
432
|
-
f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
433
|
-
end
|
434
|
-
t = Time.now - 50000
|
435
|
-
File.utime(t, t, p1)
|
436
|
-
|
437
|
-
@c2id = Fluent::UniqueId.generate
|
438
|
-
p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf")
|
439
|
-
File.open(p2, 'wb') do |f|
|
440
|
-
f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
441
|
-
f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
442
|
-
f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
443
|
-
end
|
444
|
-
t = Time.now - 40000
|
445
|
-
File.utime(t, t, p2)
|
446
|
-
|
447
|
-
@c3id = Fluent::UniqueId.generate
|
448
|
-
p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf")
|
449
|
-
File.open(p3, 'wb') do |f|
|
450
|
-
f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
451
|
-
f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
452
|
-
f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
453
|
-
f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
454
|
-
end
|
455
|
-
|
456
|
-
@c4id = Fluent::UniqueId.generate
|
457
|
-
p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf")
|
458
|
-
File.open(p4, 'wb') do |f|
|
459
|
-
f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
460
|
-
f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
461
|
-
f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
462
|
-
end
|
463
|
-
end
|
464
|
-
|
465
|
-
teardown do
|
466
|
-
if @p
|
467
|
-
@p.stop unless @p.stopped?
|
468
|
-
@p.before_shutdown unless @p.before_shutdown?
|
469
|
-
@p.shutdown unless @p.shutdown?
|
470
|
-
@p.after_shutdown unless @p.after_shutdown?
|
471
|
-
@p.close unless @p.closed?
|
472
|
-
@p.terminate unless @p.terminated?
|
473
|
-
end
|
474
|
-
if @bufdir
|
475
|
-
Dir.glob(File.join(@bufdir, '*')).each do |path|
|
476
|
-
next if ['.', '..'].include?(File.basename(path))
|
477
|
-
File.delete(path)
|
478
|
-
end
|
479
|
-
end
|
480
|
-
end
|
481
|
-
|
482
|
-
test '#resume returns staged/queued chunks with metadata' do
|
483
|
-
@d = create_driver
|
484
|
-
@p = @d.instance.buffer
|
485
|
-
@p.start
|
486
|
-
|
487
|
-
assert_equal 2, @p.stage.size
|
488
|
-
assert_equal 2, @p.queue.size
|
489
|
-
|
490
|
-
stage = @p.stage
|
491
|
-
|
492
|
-
m3 = metadata()
|
493
|
-
assert_equal @c3id, stage[m3].unique_id
|
494
|
-
assert_equal 4, stage[m3].size
|
495
|
-
assert_equal :staged, stage[m3].state
|
496
|
-
|
497
|
-
m4 = metadata(tag: 'foo')
|
498
|
-
assert_equal @c4id, stage[m4].unique_id
|
499
|
-
assert_equal 3, stage[m4].size
|
500
|
-
assert_equal :staged, stage[m4].state
|
501
|
-
end
|
502
|
-
|
503
|
-
test '#resume returns queued chunks ordered by last modified time (FIFO)' do
|
504
|
-
@d = create_driver
|
505
|
-
@p = @d.instance.buffer
|
506
|
-
@p.start
|
507
|
-
|
508
|
-
assert_equal 2, @p.stage.size
|
509
|
-
assert_equal 2, @p.queue.size
|
510
|
-
|
511
|
-
queue = @p.queue
|
512
|
-
|
513
|
-
assert{ queue[0].modified_at <= queue[1].modified_at }
|
514
|
-
|
515
|
-
assert_equal @c1id, queue[0].unique_id
|
516
|
-
assert_equal :queued, queue[0].state
|
517
|
-
assert_equal 'testing', queue[0].metadata.tag
|
518
|
-
assert_nil queue[0].metadata.variables
|
519
|
-
assert_equal 4, queue[0].size
|
520
|
-
|
521
|
-
assert_equal @c2id, queue[1].unique_id
|
522
|
-
assert_equal :queued, queue[1].state
|
523
|
-
assert_equal 'testing', queue[1].metadata.tag
|
524
|
-
assert_nil queue[1].metadata.variables
|
525
|
-
assert_equal 3, queue[1].size
|
526
|
-
end
|
527
|
-
|
528
|
-
test '#resume returns staged/queued chunks but skips size calculation by calc_num_records' do
|
529
|
-
@d = create_driver(%[
|
530
|
-
<buffer tag>
|
531
|
-
@type file_single
|
532
|
-
path #{PATH}
|
533
|
-
calc_num_records false
|
534
|
-
</buffer>
|
535
|
-
])
|
536
|
-
@p = @d.instance.buffer
|
537
|
-
@p.start
|
538
|
-
|
539
|
-
assert_equal 2, @p.stage.size
|
540
|
-
assert_equal 2, @p.queue.size
|
541
|
-
|
542
|
-
stage = @p.stage
|
543
|
-
|
544
|
-
m3 = metadata()
|
545
|
-
assert_equal @c3id, stage[m3].unique_id
|
546
|
-
assert_equal 0, stage[m3].size
|
547
|
-
assert_equal :staged, stage[m3].state
|
548
|
-
|
549
|
-
m4 = metadata(tag: 'foo')
|
550
|
-
assert_equal @c4id, stage[m4].unique_id
|
551
|
-
assert_equal 0, stage[m4].size
|
552
|
-
assert_equal :staged, stage[m4].state
|
553
|
-
end
|
554
|
-
end
|
555
|
-
|
556
|
-
sub_test_case 'there are some existing file chunks with placeholders path' do
|
557
|
-
setup do
|
558
|
-
@buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)
|
559
|
-
FileUtils.rm_rf(@buf_ph_dir)
|
560
|
-
FileUtils.mkdir_p(@buf_ph_dir)
|
561
|
-
|
562
|
-
@c1id = Fluent::UniqueId.generate
|
563
|
-
p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
|
564
|
-
File.open(p1, 'wb') do |f|
|
565
|
-
f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
566
|
-
end
|
567
|
-
t = Time.now - 50000
|
568
|
-
File.utime(t, t, p1)
|
569
|
-
|
570
|
-
@c2id = Fluent::UniqueId.generate
|
571
|
-
p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf")
|
572
|
-
File.open(p2, 'wb') do |f|
|
573
|
-
f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
574
|
-
end
|
575
|
-
end
|
576
|
-
|
577
|
-
teardown do
|
578
|
-
if @p
|
579
|
-
@p.stop unless @p.stopped?
|
580
|
-
@p.before_shutdown unless @p.before_shutdown?
|
581
|
-
@p.shutdown unless @p.shutdown?
|
582
|
-
@p.after_shutdown unless @p.after_shutdown?
|
583
|
-
@p.close unless @p.closed?
|
584
|
-
@p.terminate unless @p.terminated?
|
585
|
-
end
|
586
|
-
FileUtils.rm_rf(@buf_ph_dir)
|
587
|
-
end
|
588
|
-
|
589
|
-
test '#resume returns staged/queued chunks with metadata' do
|
590
|
-
@d = create_driver(%[
|
591
|
-
<buffer tag>
|
592
|
-
@type file_single
|
593
|
-
path #{@buf_ph_dir}
|
594
|
-
</buffer>
|
595
|
-
])
|
596
|
-
@p = @d.instance.buffer
|
597
|
-
@p.start
|
598
|
-
|
599
|
-
assert_equal 1, @p.stage.size
|
600
|
-
assert_equal 1, @p.queue.size
|
601
|
-
end
|
602
|
-
end
|
603
|
-
|
604
|
-
sub_test_case 'there are some existing msgpack file chunks' do
|
605
|
-
setup do
|
606
|
-
packer = Fluent::MessagePackFactory.packer
|
607
|
-
@c1id = Fluent::UniqueId.generate
|
608
|
-
p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
|
609
|
-
File.open(p1, 'wb') do |f|
|
610
|
-
packer.write(["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}])
|
611
|
-
packer.write(["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}])
|
612
|
-
packer.write(["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}])
|
613
|
-
packer.write(["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}])
|
614
|
-
f.write packer.full_pack
|
615
|
-
end
|
616
|
-
t = Time.now - 50000
|
617
|
-
File.utime(t, t, p1)
|
618
|
-
|
619
|
-
@c2id = Fluent::UniqueId.generate
|
620
|
-
p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf")
|
621
|
-
File.open(p2, 'wb') do |f|
|
622
|
-
packer.write(["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}])
|
623
|
-
packer.write(["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}])
|
624
|
-
packer.write(["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}])
|
625
|
-
f.write packer.full_pack
|
626
|
-
end
|
627
|
-
t = Time.now - 40000
|
628
|
-
File.utime(t, t, p2)
|
629
|
-
|
630
|
-
@c3id = Fluent::UniqueId.generate
|
631
|
-
p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf")
|
632
|
-
File.open(p3, 'wb') do |f|
|
633
|
-
packer.write(["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}])
|
634
|
-
packer.write(["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}])
|
635
|
-
packer.write(["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}])
|
636
|
-
packer.write(["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}])
|
637
|
-
f.write packer.full_pack
|
638
|
-
end
|
639
|
-
|
640
|
-
@c4id = Fluent::UniqueId.generate
|
641
|
-
p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf")
|
642
|
-
File.open(p4, 'wb') do |f|
|
643
|
-
packer.write(["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}])
|
644
|
-
packer.write(["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}])
|
645
|
-
packer.write(["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}])
|
646
|
-
f.write packer.full_pack
|
647
|
-
end
|
648
|
-
end
|
649
|
-
|
650
|
-
teardown do
|
651
|
-
if @p
|
652
|
-
@p.stop unless @p.stopped?
|
653
|
-
@p.before_shutdown unless @p.before_shutdown?
|
654
|
-
@p.shutdown unless @p.shutdown?
|
655
|
-
@p.after_shutdown unless @p.after_shutdown?
|
656
|
-
@p.close unless @p.closed?
|
657
|
-
@p.terminate unless @p.terminated?
|
658
|
-
end
|
659
|
-
if @bufdir
|
660
|
-
Dir.glob(File.join(@bufdir, '*')).each do |path|
|
661
|
-
next if ['.', '..'].include?(File.basename(path))
|
662
|
-
File.delete(path)
|
663
|
-
end
|
664
|
-
end
|
665
|
-
end
|
666
|
-
|
667
|
-
test '#resume returns staged/queued chunks with msgpack format' do
|
668
|
-
@d = create_driver(%[
|
669
|
-
<buffer tag>
|
670
|
-
@type file_single
|
671
|
-
path #{PATH}
|
672
|
-
chunk_format msgpack
|
673
|
-
</buffer>
|
674
|
-
])
|
675
|
-
@p = @d.instance.buffer
|
676
|
-
@p.start
|
677
|
-
|
678
|
-
assert_equal 2, @p.stage.size
|
679
|
-
assert_equal 2, @p.queue.size
|
680
|
-
|
681
|
-
stage = @p.stage
|
682
|
-
|
683
|
-
m3 = metadata()
|
684
|
-
assert_equal @c3id, stage[m3].unique_id
|
685
|
-
assert_equal 4, stage[m3].size
|
686
|
-
assert_equal :staged, stage[m3].state
|
687
|
-
|
688
|
-
m4 = metadata(tag: 'foo')
|
689
|
-
assert_equal @c4id, stage[m4].unique_id
|
690
|
-
assert_equal 3, stage[m4].size
|
691
|
-
assert_equal :staged, stage[m4].state
|
692
|
-
end
|
693
|
-
end
|
694
|
-
|
695
|
-
sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
|
696
|
-
setup do
|
697
|
-
@worker0_dir = File.join(@bufdir, "worker0")
|
698
|
-
@worker1_dir = File.join(@bufdir, "worker1")
|
699
|
-
FileUtils.rm_rf(@bufdir)
|
700
|
-
FileUtils.mkdir_p(@worker0_dir)
|
701
|
-
FileUtils.mkdir_p(@worker1_dir)
|
702
|
-
|
703
|
-
@bufdir_chunk_1 = Fluent::UniqueId.generate
|
704
|
-
bc1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.buf")
|
705
|
-
File.open(bc1, 'wb') do |f|
|
706
|
-
f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
707
|
-
f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
708
|
-
f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
709
|
-
f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
710
|
-
end
|
711
|
-
|
712
|
-
@bufdir_chunk_2 = Fluent::UniqueId.generate
|
713
|
-
bc2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.buf")
|
714
|
-
File.open(bc2, 'wb') do |f|
|
715
|
-
f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
716
|
-
f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
717
|
-
f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
718
|
-
f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
719
|
-
end
|
720
|
-
|
721
|
-
@worker_dir_chunk_1 = Fluent::UniqueId.generate
|
722
|
-
wc0_1 = File.join(@worker0_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf")
|
723
|
-
wc1_1 = File.join(@worker1_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf")
|
724
|
-
[wc0_1, wc1_1].each do |chunk_path|
|
725
|
-
File.open(chunk_path, 'wb') do |f|
|
726
|
-
f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
727
|
-
f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
728
|
-
f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
729
|
-
end
|
730
|
-
end
|
731
|
-
|
732
|
-
@worker_dir_chunk_2 = Fluent::UniqueId.generate
|
733
|
-
wc0_2 = File.join(@worker0_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf")
|
734
|
-
wc1_2 = File.join(@worker1_dir, "fsb.foo.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf")
|
735
|
-
[wc0_2, wc1_2].each do |chunk_path|
|
736
|
-
File.open(chunk_path, 'wb') do |f|
|
737
|
-
f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
738
|
-
f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
739
|
-
f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
740
|
-
f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
741
|
-
end
|
742
|
-
end
|
743
|
-
|
744
|
-
@worker_dir_chunk_3 = Fluent::UniqueId.generate
|
745
|
-
wc0_3 = File.join(@worker0_dir, "fsb.bar.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf")
|
746
|
-
wc1_3 = File.join(@worker1_dir, "fsb.baz.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf")
|
747
|
-
[wc0_3, wc1_3].each do |chunk_path|
|
748
|
-
File.open(chunk_path, 'wb') do |f|
|
749
|
-
f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
750
|
-
f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
751
|
-
f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
|
752
|
-
end
|
753
|
-
end
|
754
|
-
end
|
755
|
-
|
756
|
-
teardown do
|
757
|
-
if @p
|
758
|
-
@p.stop unless @p.stopped?
|
759
|
-
@p.before_shutdown unless @p.before_shutdown?
|
760
|
-
@p.shutdown unless @p.shutdown?
|
761
|
-
@p.after_shutdown unless @p.after_shutdown?
|
762
|
-
@p.close unless @p.closed?
|
763
|
-
@p.terminate unless @p.terminated?
|
764
|
-
end
|
765
|
-
end
|
766
|
-
|
767
|
-
test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do
|
768
|
-
ENV['SERVERENGINE_WORKER_ID'] = '0'
|
769
|
-
|
770
|
-
buf_conf = config_element('buffer', '', {'path' => @bufdir})
|
771
|
-
@d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
772
|
-
with_worker_config(workers: 2, worker_id: 0) do
|
773
|
-
@d.configure(config_element('output', '', {}, [buf_conf]))
|
774
|
-
end
|
775
|
-
|
776
|
-
@d.start
|
777
|
-
@p = @d.buffer
|
778
|
-
|
779
|
-
assert_equal 2, @p.stage.size
|
780
|
-
assert_equal 3, @p.queue.size
|
781
|
-
|
782
|
-
stage = @p.stage
|
783
|
-
|
784
|
-
m1 = metadata(tag: 'testing')
|
785
|
-
assert_equal @worker_dir_chunk_2, stage[m1].unique_id
|
786
|
-
assert_equal 4, stage[m1].size
|
787
|
-
assert_equal :staged, stage[m1].state
|
788
|
-
|
789
|
-
m2 = metadata(tag: 'bar')
|
790
|
-
assert_equal @worker_dir_chunk_3, stage[m2].unique_id
|
791
|
-
assert_equal 3, stage[m2].size
|
792
|
-
assert_equal :staged, stage[m2].state
|
793
|
-
|
794
|
-
queue = @p.queue
|
795
|
-
|
796
|
-
assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort
|
797
|
-
assert_equal [3, 4, 4], queue.map(&:size).sort
|
798
|
-
assert_equal [:queued, :queued, :queued], queue.map(&:state)
|
799
|
-
end
|
800
|
-
|
801
|
-
test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do
|
802
|
-
buf_conf = config_element('buffer', '', {'path' => @bufdir})
|
803
|
-
@d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
|
804
|
-
with_worker_config(workers: 2, worker_id: 1) do
|
805
|
-
@d.configure(config_element('output', '', {}, [buf_conf]))
|
806
|
-
end
|
807
|
-
|
808
|
-
@d.start
|
809
|
-
@p = @d.buffer
|
810
|
-
|
811
|
-
assert_equal 2, @p.stage.size
|
812
|
-
assert_equal 1, @p.queue.size
|
813
|
-
|
814
|
-
stage = @p.stage
|
815
|
-
|
816
|
-
m1 = metadata(tag: 'foo')
|
817
|
-
assert_equal @worker_dir_chunk_2, stage[m1].unique_id
|
818
|
-
assert_equal 4, stage[m1].size
|
819
|
-
assert_equal :staged, stage[m1].state
|
820
|
-
|
821
|
-
m2 = metadata(tag: 'baz')
|
822
|
-
assert_equal @worker_dir_chunk_3, stage[m2].unique_id
|
823
|
-
assert_equal 3, stage[m2].size
|
824
|
-
assert_equal :staged, stage[m2].state
|
825
|
-
|
826
|
-
queue = @p.queue
|
827
|
-
|
828
|
-
assert_equal @worker_dir_chunk_1, queue[0].unique_id
|
829
|
-
assert_equal 3, queue[0].size
|
830
|
-
assert_equal :queued, queue[0].state
|
831
|
-
end
|
832
|
-
end
|
833
|
-
|
834
|
-
sub_test_case 'there are existing broken file chunks' do
|
835
|
-
setup do
|
836
|
-
FileUtils.rm_rf(@bufdir) rescue nil
|
837
|
-
FileUtils.mkdir_p(@bufdir)
|
838
|
-
end
|
839
|
-
|
840
|
-
teardown do
|
841
|
-
return unless @p
|
842
|
-
|
843
|
-
@p.stop unless @p.stopped?
|
844
|
-
@p.before_shutdown unless @p.before_shutdown?
|
845
|
-
@p.shutdown unless @p.shutdown?
|
846
|
-
@p.after_shutdown unless @p.after_shutdown?
|
847
|
-
@p.close unless @p.closed?
|
848
|
-
@p.terminate unless @p.terminated?
|
849
|
-
end
|
850
|
-
|
851
|
-
test '#resume backups empty chunk' do
|
852
|
-
id_output = 'backup_test'
|
853
|
-
@d = create_driver(%[
|
854
|
-
@id #{id_output}
|
855
|
-
<buffer tag>
|
856
|
-
@type file_single
|
857
|
-
path #{PATH}
|
858
|
-
</buffer>
|
859
|
-
])
|
860
|
-
@p = @d.instance.buffer
|
861
|
-
|
862
|
-
c1id = Fluent::UniqueId.generate
|
863
|
-
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
864
|
-
File.open(p1, 'wb') { |f| } # create empty chunk file
|
865
|
-
|
866
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
867
|
-
@p.start
|
868
|
-
end
|
869
|
-
|
870
|
-
assert { not File.exist?(p1) }
|
871
|
-
assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
872
|
-
end
|
873
|
-
|
874
|
-
test '#resume throws away broken chunk with disable_chunk_backup' do
|
875
|
-
id_output = 'backup_test'
|
876
|
-
@d = create_driver(%[
|
877
|
-
@id #{id_output}
|
878
|
-
<buffer tag>
|
879
|
-
@type file_single
|
880
|
-
path #{PATH}
|
881
|
-
disable_chunk_backup true
|
882
|
-
</buffer>
|
883
|
-
])
|
884
|
-
@p = @d.instance.buffer
|
885
|
-
|
886
|
-
c1id = Fluent::UniqueId.generate
|
887
|
-
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
888
|
-
File.open(p1, 'wb') { |f| } # create empty chunk file
|
889
|
-
|
890
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
891
|
-
@p.start
|
892
|
-
end
|
893
|
-
|
894
|
-
assert { not File.exist?(p1) }
|
895
|
-
assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
896
|
-
end
|
897
|
-
end
|
898
|
-
end
|