fluentd 0.12.43 → 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +2 -0
- data/.travis.yml +33 -21
- data/CONTRIBUTING.md +1 -0
- data/ChangeLog +1239 -0
- data/README.md +0 -25
- data/Rakefile +2 -1
- data/Vagrantfile +17 -0
- data/appveyor.yml +35 -0
- data/example/filter_stdout.conf +5 -5
- data/example/in_forward.conf +2 -2
- data/example/in_http.conf +2 -2
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +2 -2
- data/example/in_tail.conf +2 -2
- data/example/in_tcp.conf +2 -2
- data/example/in_udp.conf +2 -2
- data/example/out_copy.conf +4 -4
- data/example/out_file.conf +2 -2
- data/example/out_forward.conf +2 -2
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +8 -8
- data/fluent.conf +29 -0
- data/fluentd.gemspec +18 -11
- data/lib/fluent/agent.rb +60 -58
- data/lib/fluent/command/cat.rb +1 -1
- data/lib/fluent/command/debug.rb +7 -5
- data/lib/fluent/command/fluentd.rb +97 -2
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/filter.rb +50 -0
- data/lib/fluent/compat/formatter.rb +109 -0
- data/lib/fluent/compat/input.rb +50 -0
- data/lib/fluent/compat/output.rb +617 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +163 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/config.rb +23 -20
- data/lib/fluent/config/configure_proxy.rb +119 -70
- data/lib/fluent/config/dsl.rb +5 -18
- data/lib/fluent/config/element.rb +72 -8
- data/lib/fluent/config/error.rb +0 -3
- data/lib/fluent/config/literal_parser.rb +0 -2
- data/lib/fluent/config/parser.rb +4 -4
- data/lib/fluent/config/section.rb +39 -28
- data/lib/fluent/config/types.rb +2 -13
- data/lib/fluent/config/v1_parser.rb +1 -3
- data/lib/fluent/configurable.rb +48 -16
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +26 -52
- data/lib/fluent/env.rb +6 -4
- data/lib/fluent/event.rb +58 -11
- data/lib/fluent/event_router.rb +5 -5
- data/lib/fluent/filter.rb +2 -50
- data/lib/fluent/formatter.rb +4 -293
- data/lib/fluent/input.rb +2 -32
- data/lib/fluent/label.rb +2 -2
- data/lib/fluent/load.rb +3 -2
- data/lib/fluent/log.rb +107 -38
- data/lib/fluent/match.rb +0 -36
- data/lib/fluent/mixin.rb +117 -7
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +7 -612
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +4 -800
- data/lib/fluent/plugin.rb +100 -121
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +121 -0
- data/lib/fluent/plugin/buf_file.rb +101 -182
- data/lib/fluent/plugin/buf_memory.rb +9 -92
- data/lib/fluent/plugin/buffer.rb +473 -0
- data/lib/fluent/plugin/buffer/chunk.rb +135 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +339 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +100 -0
- data/lib/fluent/plugin/exec_util.rb +80 -75
- data/lib/fluent/plugin/file_util.rb +33 -28
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +51 -0
- data/lib/fluent/plugin/filter_grep.rb +13 -40
- data/lib/fluent/plugin/filter_record_transformer.rb +22 -18
- data/lib/fluent/plugin/formatter.rb +93 -0
- data/lib/fluent/plugin/formatter_csv.rb +48 -0
- data/lib/fluent/plugin/formatter_hash.rb +32 -0
- data/lib/fluent/plugin/formatter_json.rb +47 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +32 -0
- data/lib/fluent/plugin/formatter_out_file.rb +45 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +39 -0
- data/lib/fluent/plugin/in_debug_agent.rb +4 -0
- data/lib/fluent/plugin/in_dummy.rb +22 -18
- data/lib/fluent/plugin/in_exec.rb +18 -8
- data/lib/fluent/plugin/in_forward.rb +36 -79
- data/lib/fluent/plugin/in_gc_stat.rb +4 -0
- data/lib/fluent/plugin/in_http.rb +21 -18
- data/lib/fluent/plugin/in_monitor_agent.rb +15 -48
- data/lib/fluent/plugin/in_object_space.rb +6 -1
- data/lib/fluent/plugin/in_stream.rb +7 -3
- data/lib/fluent/plugin/in_syslog.rb +46 -95
- data/lib/fluent/plugin/in_tail.rb +58 -640
- data/lib/fluent/plugin/in_tcp.rb +8 -1
- data/lib/fluent/plugin/in_udp.rb +8 -18
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_copy.rb +11 -7
- data/lib/fluent/plugin/out_exec.rb +15 -11
- data/lib/fluent/plugin/out_exec_filter.rb +18 -10
- data/lib/fluent/plugin/out_file.rb +34 -5
- data/lib/fluent/plugin/out_forward.rb +25 -19
- data/lib/fluent/plugin/out_null.rb +0 -14
- data/lib/fluent/plugin/out_roundrobin.rb +11 -7
- data/lib/fluent/plugin/out_stdout.rb +5 -7
- data/lib/fluent/plugin/out_stream.rb +3 -1
- data/lib/fluent/plugin/output.rb +979 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +244 -0
- data/lib/fluent/plugin/parser_apache.rb +24 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +21 -0
- data/lib/fluent/plugin/parser_csv.rb +31 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +102 -0
- data/lib/fluent/plugin/parser_nginx.rb +24 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +119 -117
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +116 -0
- data/lib/fluent/plugin/string_util.rb +16 -13
- data/lib/fluent/plugin_helper.rb +39 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +99 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +308 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +85 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +21 -30
- data/lib/fluent/registry.rb +21 -9
- data/lib/fluent/root_agent.rb +115 -40
- data/lib/fluent/supervisor.rb +330 -320
- data/lib/fluent/system_config.rb +42 -18
- data/lib/fluent/test.rb +6 -1
- data/lib/fluent/test/base.rb +23 -3
- data/lib/fluent/test/driver/base.rb +247 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +35 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/output.rb +78 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +0 -1
- data/lib/fluent/test/formatter_test.rb +2 -1
- data/lib/fluent/test/input_test.rb +23 -17
- data/lib/fluent/test/output_test.rb +28 -39
- data/lib/fluent/test/parser_test.rb +1 -1
- data/lib/fluent/time.rb +104 -1
- data/lib/fluent/{status.rb → unique_id.rb} +15 -24
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +72 -0
- data/test/compat/test_calls_super.rb +164 -0
- data/test/config/test_config_parser.rb +83 -0
- data/test/config/test_configurable.rb +547 -274
- data/test/config/test_configure_proxy.rb +146 -29
- data/test/config/test_dsl.rb +3 -181
- data/test/config/test_element.rb +274 -0
- data/test/config/test_literal_parser.rb +1 -1
- data/test/config/test_section.rb +79 -7
- data/test/config/test_system_config.rb +21 -0
- data/test/config/test_types.rb +3 -26
- data/test/helper.rb +78 -8
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +420 -521
- data/test/plugin/test_buf_memory.rb +32 -194
- data/test/plugin/test_buffer.rb +981 -0
- data/test/plugin/test_buffer_chunk.rb +110 -0
- data/test/plugin/test_buffer_file_chunk.rb +770 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_filter.rb +255 -0
- data/test/plugin/test_filter_grep.rb +2 -73
- data/test/plugin/test_filter_record_transformer.rb +24 -68
- data/test/plugin/test_filter_stdout.rb +6 -6
- data/test/plugin/test_in_debug_agent.rb +2 -0
- data/test/plugin/test_in_dummy.rb +11 -17
- data/test/plugin/test_in_exec.rb +6 -25
- data/test/plugin/test_in_forward.rb +112 -151
- data/test/plugin/test_in_gc_stat.rb +2 -0
- data/test/plugin/test_in_http.rb +106 -157
- data/test/plugin/test_in_object_space.rb +21 -5
- data/test/plugin/test_in_stream.rb +14 -13
- data/test/plugin/test_in_syslog.rb +30 -275
- data/test/plugin/test_in_tail.rb +95 -282
- data/test/plugin/test_in_tcp.rb +14 -0
- data/test/plugin/test_in_udp.rb +21 -67
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_copy.rb +15 -2
- data/test/plugin/test_out_exec.rb +75 -25
- data/test/plugin/test_out_exec_filter.rb +74 -8
- data/test/plugin/test_out_file.rb +61 -7
- data/test/plugin/test_out_forward.rb +92 -15
- data/test/plugin/test_out_roundrobin.rb +1 -0
- data/test/plugin/test_out_stdout.rb +22 -13
- data/test/plugin/test_out_stream.rb +18 -0
- data/test/plugin/test_output.rb +515 -0
- data/test/plugin/test_output_as_buffered.rb +1540 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +247 -0
- data/test/plugin/test_output_as_buffered_retries.rb +808 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +776 -0
- data/test/plugin/test_output_as_standard.rb +362 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin_helper/test_child_process.rb +599 -0
- data/test/plugin_helper/test_compat_parameters.rb +175 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +411 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +100 -0
- data/test/scripts/exec_script.rb +0 -6
- data/test/scripts/fluent/plugin/out_test.rb +3 -0
- data/test/test_config.rb +13 -4
- data/test/test_event.rb +24 -13
- data/test/test_event_router.rb +8 -7
- data/test/test_event_time.rb +187 -0
- data/test/test_formatter.rb +13 -51
- data/test/test_input.rb +1 -1
- data/test/test_log.rb +239 -16
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +53 -66
- data/test/test_parser.rb +105 -323
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_root_agent.rb +4 -52
- data/test/test_supervisor.rb +272 -0
- data/test/test_unique_id.rb +47 -0
- metadata +181 -55
- data/CHANGELOG.md +0 -710
- data/lib/fluent/buffer.rb +0 -365
- data/lib/fluent/plugin/filter_parser.rb +0 -107
- data/lib/fluent/plugin/in_status.rb +0 -76
- data/lib/fluent/test/helpers.rb +0 -86
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/test_filter_parser.rb +0 -744
- data/test/plugin/test_in_status.rb +0 -38
- data/test/test_buffer.rb +0 -624
@@ -0,0 +1,362 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/output'
|
3
|
+
require 'fluent/plugin/buffer'
|
4
|
+
require 'fluent/msgpack_factory'
|
5
|
+
require 'fluent/event'
|
6
|
+
|
7
|
+
require 'json'
|
8
|
+
require 'time'
|
9
|
+
require 'timeout'
|
10
|
+
|
11
|
+
require 'flexmock/test_unit'
|
12
|
+
|
13
|
+
module FluentPluginStandardBufferedOutputTest
|
14
|
+
class DummyBareOutput < Fluent::Plugin::Output
|
15
|
+
def register(name, &block)
|
16
|
+
instance_variable_set("@#{name}", block)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
class DummyAsyncOutput < DummyBareOutput
|
20
|
+
def format(tag, time, record)
|
21
|
+
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
22
|
+
end
|
23
|
+
def write(chunk)
|
24
|
+
@write ? @write.call(chunk) : nil
|
25
|
+
end
|
26
|
+
end
|
27
|
+
class DummyAsyncStandardOutput < DummyBareOutput
|
28
|
+
def write(chunk)
|
29
|
+
@write ? @write.call(chunk) : nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class StandardBufferedOutputTest < Test::Unit::TestCase
|
35
|
+
def create_output(type=:full)
|
36
|
+
case type
|
37
|
+
when :bare then FluentPluginStandardBufferedOutputTest::DummyBareOutput.new
|
38
|
+
when :buffered then FluentPluginStandardBufferedOutputTest::DummyAsyncOutput.new
|
39
|
+
when :standard then FluentPluginStandardBufferedOutputTest::DummyAsyncStandardOutput.new
|
40
|
+
else
|
41
|
+
raise ArgumentError, "unknown type: #{type}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
def create_metadata(timekey: nil, tag: nil, variables: nil)
|
45
|
+
Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
|
46
|
+
end
|
47
|
+
def waiting(seconds)
|
48
|
+
begin
|
49
|
+
Timeout.timeout(seconds) do
|
50
|
+
yield
|
51
|
+
end
|
52
|
+
rescue Timeout::Error
|
53
|
+
STDERR.print(*@i.log.out.logs)
|
54
|
+
raise
|
55
|
+
end
|
56
|
+
end
|
57
|
+
def test_event_stream
|
58
|
+
es = Fluent::MultiEventStream.new
|
59
|
+
es.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
60
|
+
es.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
61
|
+
es.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
62
|
+
es.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
63
|
+
es.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
64
|
+
es.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
65
|
+
es
|
66
|
+
end
|
67
|
+
|
68
|
+
setup do
|
69
|
+
@i = nil
|
70
|
+
end
|
71
|
+
|
72
|
+
teardown do
|
73
|
+
if @i
|
74
|
+
@i.stop unless @i.stopped?
|
75
|
+
@i.before_shutdown unless @i.before_shutdown?
|
76
|
+
@i.shutdown unless @i.shutdown?
|
77
|
+
@i.after_shutdown unless @i.after_shutdown?
|
78
|
+
@i.close unless @i.closed?
|
79
|
+
@i.terminate unless @i.terminated?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
sub_test_case 'standard buffered without any chunk keys' do
|
84
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
|
85
|
+
@i = create_output(:standard)
|
86
|
+
@i.configure(config_element())
|
87
|
+
@i.start
|
88
|
+
|
89
|
+
m = create_metadata()
|
90
|
+
es = test_event_stream
|
91
|
+
|
92
|
+
buffer_mock = flexmock(@i.buffer)
|
93
|
+
buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
|
94
|
+
|
95
|
+
@i.execute_chunking("mytag.test", es)
|
96
|
+
end
|
97
|
+
|
98
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
|
99
|
+
@i = create_output(:standard)
|
100
|
+
@i.configure(config_element('ROOT','',{"time_as_integer"=>"true"}))
|
101
|
+
@i.start
|
102
|
+
|
103
|
+
m = create_metadata()
|
104
|
+
es = test_event_stream
|
105
|
+
|
106
|
+
buffer_mock = flexmock(@i.buffer)
|
107
|
+
buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
|
108
|
+
|
109
|
+
@i.execute_chunking("mytag.test", es)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
sub_test_case 'standard buffered with tag chunk key' do
|
114
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
|
115
|
+
@i = create_output(:standard)
|
116
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
|
117
|
+
@i.start
|
118
|
+
|
119
|
+
m = create_metadata(tag: "mytag.test")
|
120
|
+
es = test_event_stream
|
121
|
+
|
122
|
+
buffer_mock = flexmock(@i.buffer)
|
123
|
+
buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
|
124
|
+
|
125
|
+
@i.execute_chunking("mytag.test", es)
|
126
|
+
end
|
127
|
+
|
128
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
|
129
|
+
@i = create_output(:standard)
|
130
|
+
@i.configure(config_element('ROOT','',{"time_as_integer"=>"true"},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
|
131
|
+
@i.start
|
132
|
+
|
133
|
+
m = create_metadata(tag: "mytag.test")
|
134
|
+
es = test_event_stream
|
135
|
+
|
136
|
+
buffer_mock = flexmock(@i.buffer)
|
137
|
+
buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
|
138
|
+
|
139
|
+
@i.execute_chunking("mytag.test", es)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
sub_test_case 'standard buffered with time chunk key' do
|
144
|
+
test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
|
145
|
+
@i = create_output(:standard)
|
146
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
|
147
|
+
@i.start
|
148
|
+
|
149
|
+
m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
|
150
|
+
m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
|
151
|
+
m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
|
152
|
+
|
153
|
+
es1 = Fluent::MultiEventStream.new
|
154
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
155
|
+
es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
156
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
157
|
+
|
158
|
+
es2 = Fluent::MultiEventStream.new
|
159
|
+
es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
160
|
+
es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
161
|
+
|
162
|
+
es3 = Fluent::MultiEventStream.new
|
163
|
+
es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
164
|
+
|
165
|
+
buffer_mock = flexmock(@i.buffer)
|
166
|
+
buffer_mock.should_receive(:write).once.with({
|
167
|
+
m1 => [es1.to_msgpack_stream, 3],
|
168
|
+
m2 => [es2.to_msgpack_stream, 2],
|
169
|
+
m3 => [es3.to_msgpack_stream, 1],
|
170
|
+
}, bulk: true, enqueue: false)
|
171
|
+
|
172
|
+
es = test_event_stream
|
173
|
+
@i.execute_chunking("mytag.test", es)
|
174
|
+
end
|
175
|
+
|
176
|
+
test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format, but time will be int if time_as_integer specified' do
|
177
|
+
@i = create_output(:standard)
|
178
|
+
@i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
|
179
|
+
@i.start
|
180
|
+
|
181
|
+
m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
|
182
|
+
m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
|
183
|
+
m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
|
184
|
+
|
185
|
+
es1 = Fluent::MultiEventStream.new
|
186
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
187
|
+
es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
188
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
189
|
+
|
190
|
+
es2 = Fluent::MultiEventStream.new
|
191
|
+
es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
192
|
+
es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
193
|
+
|
194
|
+
es3 = Fluent::MultiEventStream.new
|
195
|
+
es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
196
|
+
|
197
|
+
buffer_mock = flexmock(@i.buffer)
|
198
|
+
buffer_mock.should_receive(:write).with({
|
199
|
+
m1 => [es1.to_msgpack_stream(time_int: true), 3],
|
200
|
+
m2 => [es2.to_msgpack_stream(time_int: true), 2],
|
201
|
+
m3 => [es3.to_msgpack_stream(time_int: true), 1],
|
202
|
+
}, bulk: true, enqueue: false)
|
203
|
+
|
204
|
+
es = test_event_stream
|
205
|
+
@i.execute_chunking("mytag.test", es)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
sub_test_case 'standard buffered with variable chunk keys' do
|
210
|
+
test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
|
211
|
+
@i = create_output(:standard)
|
212
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
|
213
|
+
@i.start
|
214
|
+
|
215
|
+
m1 = create_metadata(variables: {key: "my value", name: "moris1"})
|
216
|
+
es1 = Fluent::MultiEventStream.new
|
217
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
218
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
219
|
+
es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
220
|
+
es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
221
|
+
es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
222
|
+
|
223
|
+
m2 = create_metadata(variables: {key: "my value", name: "moris2"})
|
224
|
+
es2 = Fluent::MultiEventStream.new
|
225
|
+
es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
226
|
+
|
227
|
+
buffer_mock = flexmock(@i.buffer)
|
228
|
+
buffer_mock.should_receive(:write).with({
|
229
|
+
m1 => [es1.to_msgpack_stream, 5],
|
230
|
+
m2 => [es2.to_msgpack_stream, 1],
|
231
|
+
}, bulk: true, enqueue: false).once
|
232
|
+
|
233
|
+
es = test_event_stream
|
234
|
+
@i.execute_chunking("mytag.test", es)
|
235
|
+
end
|
236
|
+
|
237
|
+
test '#execute_chunking calls @buffer.write(bulk: true) in times of # of variable variations with predefined msgpack format, but time will be int if time_as_integer specified' do
|
238
|
+
@i = create_output(:standard)
|
239
|
+
@i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
|
240
|
+
@i.start
|
241
|
+
|
242
|
+
m1 = create_metadata(variables: {key: "my value", name: "moris1"})
|
243
|
+
es1 = Fluent::MultiEventStream.new
|
244
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
245
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
246
|
+
es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
247
|
+
es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
248
|
+
es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
249
|
+
|
250
|
+
m2 = create_metadata(variables: {key: "my value", name: "moris2"})
|
251
|
+
es2 = Fluent::MultiEventStream.new
|
252
|
+
es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
253
|
+
|
254
|
+
buffer_mock = flexmock(@i.buffer)
|
255
|
+
buffer_mock.should_receive(:write).with({
|
256
|
+
m1 => [es1.to_msgpack_stream(time_int: true), 5],
|
257
|
+
m2 => [es2.to_msgpack_stream(time_int: true), 1],
|
258
|
+
}, bulk: true, enqueue: false).once
|
259
|
+
|
260
|
+
es = test_event_stream
|
261
|
+
@i.execute_chunking("mytag.test", es)
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
sub_test_case 'custom format buffered without any chunk keys' do
|
266
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
|
267
|
+
@i = create_output(:buffered)
|
268
|
+
@i.register(:format){|tag, time, record| [time, record].to_json }
|
269
|
+
@i.configure(config_element())
|
270
|
+
@i.start
|
271
|
+
|
272
|
+
m = create_metadata()
|
273
|
+
es = test_event_stream
|
274
|
+
|
275
|
+
buffer_mock = flexmock(@i.buffer)
|
276
|
+
buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
|
277
|
+
|
278
|
+
@i.execute_chunking("mytag.test", es)
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
sub_test_case 'custom format buffered with tag chunk key' do
|
283
|
+
test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
|
284
|
+
@i = create_output(:buffered)
|
285
|
+
@i.register(:format){|tag, time, record| [time, record].to_json }
|
286
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
|
287
|
+
@i.start
|
288
|
+
|
289
|
+
m = create_metadata(tag: "mytag.test")
|
290
|
+
es = test_event_stream
|
291
|
+
|
292
|
+
buffer_mock = flexmock(@i.buffer)
|
293
|
+
buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
|
294
|
+
|
295
|
+
@i.execute_chunking("mytag.test", es)
|
296
|
+
end
|
297
|
+
end
|
298
|
+
sub_test_case 'custom format buffered with time chunk key' do
|
299
|
+
test '#execute_chunking calls @buffer.write with customized format' do
|
300
|
+
@i = create_output(:buffered)
|
301
|
+
@i.register(:format){|tag, time, record| [time, record].to_json }
|
302
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
|
303
|
+
@i.start
|
304
|
+
|
305
|
+
m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
|
306
|
+
m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
|
307
|
+
m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
|
308
|
+
|
309
|
+
es1 = Fluent::MultiEventStream.new
|
310
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
311
|
+
es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
312
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
313
|
+
|
314
|
+
es2 = Fluent::MultiEventStream.new
|
315
|
+
es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
316
|
+
es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
317
|
+
|
318
|
+
es3 = Fluent::MultiEventStream.new
|
319
|
+
es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
320
|
+
|
321
|
+
buffer_mock = flexmock(@i.buffer)
|
322
|
+
buffer_mock.should_receive(:write).with({
|
323
|
+
m1 => es1.map{|t,r| [t,r].to_json },
|
324
|
+
m2 => es2.map{|t,r| [t,r].to_json },
|
325
|
+
m3 => es3.map{|t,r| [t,r].to_json },
|
326
|
+
}, bulk: false, enqueue: false).once
|
327
|
+
|
328
|
+
es = test_event_stream
|
329
|
+
@i.execute_chunking("mytag.test", es)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
sub_test_case 'custom format buffered with variable chunk keys' do
|
334
|
+
test '#execute_chunking calls @buffer.write in times of # of variable variations with customized format' do
|
335
|
+
@i = create_output(:buffered)
|
336
|
+
@i.register(:format){|tag, time, record| [time, record].to_json }
|
337
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
|
338
|
+
@i.start
|
339
|
+
|
340
|
+
m1 = create_metadata(variables: {key: "my value", name: "moris1"})
|
341
|
+
es1 = Fluent::MultiEventStream.new
|
342
|
+
es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
343
|
+
es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
344
|
+
es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
345
|
+
es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
346
|
+
es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
|
347
|
+
|
348
|
+
m2 = create_metadata(variables: {key: "my value", name: "moris2"})
|
349
|
+
es2 = Fluent::MultiEventStream.new
|
350
|
+
es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
|
351
|
+
|
352
|
+
buffer_mock = flexmock(@i.buffer)
|
353
|
+
buffer_mock.should_receive(:write).with({
|
354
|
+
m1 => es1.map{|t,r| [t,r].to_json },
|
355
|
+
m2 => es2.map{|t,r| [t,r].to_json },
|
356
|
+
}, bulk: false, enqueue: false).once
|
357
|
+
|
358
|
+
es = test_event_stream
|
359
|
+
@i.execute_chunking("mytag.test", es)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/base'
|
3
|
+
require 'fluent/plugin/input'
|
4
|
+
require 'fluent/plugin/owned_by_mixin'
|
5
|
+
|
6
|
+
module OwnedByMixinTestEnv
|
7
|
+
class DummyParent < Fluent::Plugin::Input
|
8
|
+
Fluent::Plugin.register_input('dummy_parent', self)
|
9
|
+
end
|
10
|
+
class DummyChild < Fluent::Plugin::Base
|
11
|
+
include Fluent::Plugin::OwnedByMixin
|
12
|
+
Fluent::Plugin.register_parser('dummy_child', self)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
class OwnedByMixinTest < Test::Unit::TestCase
|
17
|
+
sub_test_case 'Owned plugins' do
|
18
|
+
setup do
|
19
|
+
Fluent::Test.setup
|
20
|
+
end
|
21
|
+
|
22
|
+
test 'inherits plugin id and logger from parent' do
|
23
|
+
parent = Fluent::Plugin.new_input('dummy_parent')
|
24
|
+
parent.configure(config_element('ROOT', '', {'@id' => 'my_parent_id', '@log_level' => 'trace'}))
|
25
|
+
child = Fluent::Plugin.new_parser('dummy_child', parent: parent)
|
26
|
+
|
27
|
+
assert_equal parent.object_id, child.owner.object_id
|
28
|
+
|
29
|
+
assert child.instance_eval{ @_plugin_id_configured }
|
30
|
+
assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
|
31
|
+
|
32
|
+
assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin/storage'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
|
5
|
+
class DummyPlugin < Fluent::Plugin::TestBase
|
6
|
+
end
|
7
|
+
|
8
|
+
class BareStorage < Fluent::Plugin::Storage
|
9
|
+
Fluent::Plugin.register_storage('bare', self)
|
10
|
+
end
|
11
|
+
|
12
|
+
class BasicStorage < Fluent::Plugin::Storage
|
13
|
+
Fluent::Plugin.register_storage('example', self)
|
14
|
+
|
15
|
+
attr_reader :data, :saved
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
super
|
19
|
+
@data = @saved = nil
|
20
|
+
end
|
21
|
+
def load
|
22
|
+
@data = {}
|
23
|
+
end
|
24
|
+
def save
|
25
|
+
@saved = @data.dup
|
26
|
+
end
|
27
|
+
def get(key)
|
28
|
+
@data[key]
|
29
|
+
end
|
30
|
+
def fetch(key, defval)
|
31
|
+
@data.fetch(key, defval)
|
32
|
+
end
|
33
|
+
def put(key, value)
|
34
|
+
@data[key] = value
|
35
|
+
end
|
36
|
+
def delete(key)
|
37
|
+
@data.delete(key)
|
38
|
+
end
|
39
|
+
def update(key, &block)
|
40
|
+
@data[key] = block.call(@data[key])
|
41
|
+
end
|
42
|
+
def close
|
43
|
+
@data = {}
|
44
|
+
super
|
45
|
+
end
|
46
|
+
def terminate
|
47
|
+
@saved = {}
|
48
|
+
super
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
class StorageTest < Test::Unit::TestCase
|
53
|
+
sub_test_case 'BareStorage' do
|
54
|
+
setup do
|
55
|
+
plugin = DummyPlugin.new
|
56
|
+
@s = BareStorage.new
|
57
|
+
@s.configure(config_element())
|
58
|
+
@s.owner = plugin
|
59
|
+
end
|
60
|
+
|
61
|
+
test 'is configured with plugin information and system config' do
|
62
|
+
plugin = DummyPlugin.new
|
63
|
+
plugin.system_config_override({'process_name' => 'mytest'})
|
64
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
|
65
|
+
s = BareStorage.new
|
66
|
+
s.configure(config_element())
|
67
|
+
s.owner = plugin
|
68
|
+
|
69
|
+
assert_equal 'mytest', s.owner.system_config.process_name
|
70
|
+
assert_equal '1', s.instance_eval{ @_plugin_id }
|
71
|
+
assert_equal true, s.instance_eval{ @_plugin_id_configured }
|
72
|
+
end
|
73
|
+
|
74
|
+
test 'does NOT have features for high-performance/high-consistent storages' do
|
75
|
+
assert_equal false, @s.persistent_always?
|
76
|
+
assert_equal false, @s.synchronized?
|
77
|
+
end
|
78
|
+
|
79
|
+
test 'does have default values which is conservative for almost all users' do
|
80
|
+
assert_equal false, @s.persistent
|
81
|
+
assert_equal true, @s.autosave
|
82
|
+
assert_equal 10, @s.autosave_interval
|
83
|
+
assert_equal true, @s.save_at_shutdown
|
84
|
+
end
|
85
|
+
|
86
|
+
test 'load/save doesn NOT anything: just as memory storage' do
|
87
|
+
assert_nothing_raised{ @s.load }
|
88
|
+
assert_nothing_raised{ @s.save }
|
89
|
+
end
|
90
|
+
|
91
|
+
test 'all operations are not defined yet' do
|
92
|
+
assert_raise NotImplementedError do
|
93
|
+
@s.get('key')
|
94
|
+
end
|
95
|
+
assert_raise NotImplementedError do
|
96
|
+
@s.fetch('key', 'value')
|
97
|
+
end
|
98
|
+
assert_raise NotImplementedError do
|
99
|
+
@s.put('key', 'value')
|
100
|
+
end
|
101
|
+
assert_raise NotImplementedError do
|
102
|
+
@s.delete('key')
|
103
|
+
end
|
104
|
+
assert_raise NotImplementedError do
|
105
|
+
@s.update('key'){ |v| v + '2' }
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
sub_test_case 'ExampleStorage' do
|
111
|
+
setup do
|
112
|
+
plugin = DummyPlugin.new
|
113
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
|
114
|
+
@s = BasicStorage.new
|
115
|
+
@s.configure(config_element())
|
116
|
+
@s.owner = plugin
|
117
|
+
end
|
118
|
+
|
119
|
+
test 'load/save works well as plugin internal state operations' do
|
120
|
+
plugin = DummyPlugin.new
|
121
|
+
plugin.configure(config_element('ROOT', '', {'@id' => '0'}))
|
122
|
+
s = BasicStorage.new
|
123
|
+
s.owner = plugin
|
124
|
+
|
125
|
+
assert_nothing_raised{ s.load }
|
126
|
+
assert s.data
|
127
|
+
assert_nil s.saved
|
128
|
+
|
129
|
+
assert_nothing_raised{ s.save }
|
130
|
+
assert s.saved
|
131
|
+
assert{ s.data == s.saved }
|
132
|
+
assert{ s.data.object_id != s.saved.object_id }
|
133
|
+
end
|
134
|
+
|
135
|
+
test 'all operations work well' do
|
136
|
+
@s.load
|
137
|
+
|
138
|
+
assert_nil @s.get('key')
|
139
|
+
assert_equal 'value', @s.fetch('key', 'value')
|
140
|
+
assert_nil @s.get('key')
|
141
|
+
|
142
|
+
assert_equal 'value', @s.put('key', 'value')
|
143
|
+
assert_equal 'value', @s.get('key')
|
144
|
+
|
145
|
+
assert_equal 'valuevalue', @s.update('key'){|v| v * 2 }
|
146
|
+
|
147
|
+
assert_equal 'valuevalue', @s.delete('key')
|
148
|
+
end
|
149
|
+
|
150
|
+
test 'close and terminate work to operate internal states' do
|
151
|
+
@s.load
|
152
|
+
@s.put('k1', 'v1')
|
153
|
+
@s.put('k2', 'v2')
|
154
|
+
assert_equal 2, @s.data.size
|
155
|
+
@s.save
|
156
|
+
assert_equal @s.data.size, @s.saved.size
|
157
|
+
|
158
|
+
assert_nothing_raised{ @s.close }
|
159
|
+
assert @s.data.empty?
|
160
|
+
assert !@s.saved.empty?
|
161
|
+
|
162
|
+
assert_nothing_raised{ @s.terminate }
|
163
|
+
assert @s.data.empty?
|
164
|
+
assert @s.saved.empty?
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|