fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +16 -5
- data/ADOPTERS.md +5 -0
- data/{ChangeLog → CHANGELOG.md} +495 -6
- data/CONTRIBUTING.md +5 -2
- data/GOVERNANCE.md +55 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +7 -5
- data/README.md +17 -10
- data/bin/fluent-ca-generate +6 -0
- data/example/counter.conf +18 -0
- data/example/secondary_file.conf +3 -2
- data/fluentd.gemspec +3 -3
- data/lib/fluent/agent.rb +1 -1
- data/lib/fluent/command/binlog_reader.rb +11 -2
- data/lib/fluent/command/ca_generate.rb +181 -0
- data/lib/fluent/command/cat.rb +28 -15
- data/lib/fluent/command/debug.rb +4 -4
- data/lib/fluent/command/fluentd.rb +2 -2
- data/lib/fluent/command/plugin_config_formatter.rb +24 -2
- data/lib/fluent/command/plugin_generator.rb +26 -8
- data/lib/fluent/config/configure_proxy.rb +7 -1
- data/lib/fluent/config/dsl.rb +8 -5
- data/lib/fluent/config/element.rb +5 -0
- data/lib/fluent/config/literal_parser.rb +7 -1
- data/lib/fluent/config/types.rb +28 -2
- data/lib/fluent/config/v1_parser.rb +1 -2
- data/lib/fluent/configurable.rb +1 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/counter/base_socket.rb +46 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/env.rb +1 -0
- data/lib/fluent/event_router.rb +1 -1
- data/lib/fluent/log.rb +119 -29
- data/lib/fluent/plugin/base.rb +12 -0
- data/lib/fluent/plugin/buf_file.rb +20 -16
- data/lib/fluent/plugin/buffer.rb +130 -32
- data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
- data/lib/fluent/plugin/compressable.rb +1 -1
- data/lib/fluent/plugin/filter_grep.rb +135 -21
- data/lib/fluent/plugin/filter_parser.rb +13 -2
- data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +5 -1
- data/lib/fluent/plugin/in_debug_agent.rb +8 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +84 -3
- data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
- data/lib/fluent/plugin/in_syslog.rb +31 -10
- data/lib/fluent/plugin/in_tail.rb +142 -53
- data/lib/fluent/plugin/in_tcp.rb +5 -6
- data/lib/fluent/plugin/in_udp.rb +6 -2
- data/lib/fluent/plugin/in_unix.rb +1 -1
- data/lib/fluent/plugin/multi_output.rb +1 -0
- data/lib/fluent/plugin/out_copy.rb +25 -2
- data/lib/fluent/plugin/out_file.rb +26 -7
- data/lib/fluent/plugin/out_forward.rb +81 -42
- data/lib/fluent/plugin/out_secondary_file.rb +2 -2
- data/lib/fluent/plugin/out_stdout.rb +0 -1
- data/lib/fluent/plugin/out_stream.rb +1 -1
- data/lib/fluent/plugin/output.rb +221 -57
- data/lib/fluent/plugin/parser_apache.rb +1 -1
- data/lib/fluent/plugin/parser_apache2.rb +5 -1
- data/lib/fluent/plugin/parser_apache_error.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +10 -3
- data/lib/fluent/plugin/parser_ltsv.rb +7 -0
- data/lib/fluent/plugin/parser_multiline.rb +2 -1
- data/lib/fluent/plugin/parser_nginx.rb +1 -1
- data/lib/fluent/plugin/parser_none.rb +1 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -14
- data/lib/fluent/plugin/parser_syslog.rb +9 -5
- data/lib/fluent/plugin_helper.rb +2 -0
- data/lib/fluent/plugin_helper/cert_option.rb +28 -9
- data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_loop.rb +9 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
- data/lib/fluent/plugin_helper/retry_state.rb +15 -7
- data/lib/fluent/plugin_helper/server.rb +87 -25
- data/lib/fluent/plugin_helper/socket_option.rb +5 -2
- data/lib/fluent/plugin_helper/timer.rb +8 -7
- data/lib/fluent/root_agent.rb +18 -9
- data/lib/fluent/supervisor.rb +63 -23
- data/lib/fluent/system_config.rb +30 -2
- data/lib/fluent/test/helpers.rb +1 -1
- data/lib/fluent/time.rb +15 -7
- data/lib/fluent/timezone.rb +26 -2
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/README.md.erb +2 -2
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_fluentd.rb +2 -2
- data/test/command/test_plugin_config_formatter.rb +8 -7
- data/test/command/test_plugin_generator.rb +65 -39
- data/test/config/test_config_parser.rb +7 -2
- data/test/config/test_configurable.rb +7 -2
- data/test/config/test_configure_proxy.rb +41 -3
- data/test/config/test_dsl.rb +10 -10
- data/test/config/test_element.rb +10 -0
- data/test/config/test_literal_parser.rb +8 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_system_config.rb +19 -1
- data/test/config/test_types.rb +37 -0
- data/test/counter/test_client.rb +559 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/plugin/test_buf_file.rb +124 -0
- data/test/plugin/test_buffer.rb +3 -2
- data/test/plugin/test_filter_grep.rb +580 -2
- data/test/plugin/test_filter_parser.rb +33 -2
- data/test/plugin/test_filter_record_transformer.rb +22 -1
- data/test/plugin/test_formatter_ltsv.rb +3 -0
- data/test/plugin/test_formatter_tsv.rb +68 -0
- data/test/plugin/test_in_debug_agent.rb +21 -0
- data/test/plugin/test_in_exec.rb +3 -5
- data/test/plugin/test_in_http.rb +178 -0
- data/test/plugin/test_in_monitor_agent.rb +1 -1
- data/test/plugin/test_in_syslog.rb +64 -0
- data/test/plugin/test_in_tail.rb +116 -6
- data/test/plugin/test_in_tcp.rb +21 -0
- data/test/plugin/test_in_udp.rb +78 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_out_copy.rb +31 -0
- data/test/plugin/test_out_file.rb +108 -2
- data/test/plugin/test_out_forward.rb +195 -2
- data/test/plugin/test_out_secondary_file.rb +14 -0
- data/test/plugin/test_output.rb +159 -45
- data/test/plugin/test_output_as_buffered.rb +19 -0
- data/test/plugin/test_output_as_buffered_backup.rb +307 -0
- data/test/plugin/test_output_as_buffered_retries.rb +70 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin/test_parser_apache2.rb +1 -0
- data/test/plugin/test_parser_labeled_tsv.rb +17 -0
- data/test/plugin/test_parser_nginx.rb +40 -0
- data/test/plugin/test_parser_regexp.rb +6 -7
- data/test/plugin/test_parser_syslog.rb +155 -5
- data/test/plugin_helper/test_child_process.rb +4 -4
- data/test/plugin_helper/test_compat_parameters.rb +22 -0
- data/test/plugin_helper/test_record_accessor.rb +197 -0
- data/test/plugin_helper/test_retry_state.rb +20 -0
- data/test/plugin_helper/test_server.rb +30 -2
- data/test/test_config.rb +3 -3
- data/test/test_configdsl.rb +2 -2
- data/test/test_log.rb +51 -1
- data/test/test_root_agent.rb +33 -0
- data/test/test_supervisor.rb +105 -0
- metadata +68 -8
- data/COPYING +0 -14
@@ -0,0 +1,44 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/counter/error'
|
3
|
+
|
4
|
+
class CounterErrorTest < ::Test::Unit::TestCase
|
5
|
+
setup do
|
6
|
+
@message = 'error message'
|
7
|
+
end
|
8
|
+
|
9
|
+
test 'invalid_params' do
|
10
|
+
error = Fluent::Counter::InvalidParams.new(@message)
|
11
|
+
expected = { 'code' => 'invalid_params', 'message' => @message }
|
12
|
+
assert_equal expected, error.to_hash
|
13
|
+
end
|
14
|
+
|
15
|
+
test 'unknown_key' do
|
16
|
+
error = Fluent::Counter::UnknownKey.new(@message)
|
17
|
+
expected = { 'code' => 'unknown_key', 'message' => @message }
|
18
|
+
assert_equal expected, error.to_hash
|
19
|
+
end
|
20
|
+
|
21
|
+
test 'parse_error' do
|
22
|
+
error = Fluent::Counter::ParseError.new(@message)
|
23
|
+
expected = { 'code' => 'parse_error', 'message' => @message }
|
24
|
+
assert_equal expected, error.to_hash
|
25
|
+
end
|
26
|
+
|
27
|
+
test 'method_not_found' do
|
28
|
+
error = Fluent::Counter::MethodNotFound.new(@message)
|
29
|
+
expected = { 'code' => 'method_not_found', 'message' => @message }
|
30
|
+
assert_equal expected, error.to_hash
|
31
|
+
end
|
32
|
+
|
33
|
+
test 'invalid_request' do
|
34
|
+
error = Fluent::Counter::InvalidRequest.new(@message)
|
35
|
+
expected = { 'code' => 'invalid_request', 'message' => @message }
|
36
|
+
assert_equal expected, error.to_hash
|
37
|
+
end
|
38
|
+
|
39
|
+
test 'internal_server_error' do
|
40
|
+
error = Fluent::Counter::InternalServerError.new(@message)
|
41
|
+
expected = { 'code' => 'internal_server_error', 'message' => @message }
|
42
|
+
assert_equal expected, error.to_hash
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/counter/mutex_hash'
|
3
|
+
require 'fluent/counter/store'
|
4
|
+
require 'flexmock/test_unit'
|
5
|
+
require 'timecop'
|
6
|
+
|
7
|
+
class MutexHashTest < ::Test::Unit::TestCase
|
8
|
+
setup do
|
9
|
+
@store = {}
|
10
|
+
@value = 'sample value'
|
11
|
+
@counter_store_mutex = Fluent::Counter::MutexHash.new(@store)
|
12
|
+
end
|
13
|
+
|
14
|
+
sub_test_case 'synchronize' do
|
15
|
+
test "create new mutex values if keys don't exist" do
|
16
|
+
keys = ['key', 'key1']
|
17
|
+
|
18
|
+
@counter_store_mutex.synchronize(*keys) do |store, k|
|
19
|
+
store[k] = @value
|
20
|
+
end
|
21
|
+
|
22
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
23
|
+
keys.each do |key|
|
24
|
+
assert_true mhash[key].is_a?(Mutex)
|
25
|
+
assert_equal @value, @store[key]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
test 'nothing to do when an empty array passed' do
|
30
|
+
@counter_store_mutex.synchronize(*[]) do |store, k|
|
31
|
+
store[k] = @value
|
32
|
+
end
|
33
|
+
|
34
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
35
|
+
assert_true mhash.empty?
|
36
|
+
assert_true @store.empty?
|
37
|
+
end
|
38
|
+
|
39
|
+
test 'use a one mutex value when the same key specified' do
|
40
|
+
key = 'key'
|
41
|
+
@counter_store_mutex.synchronize(key) do |store, k|
|
42
|
+
store[k] = @value
|
43
|
+
end
|
44
|
+
|
45
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
46
|
+
m = mhash[key]
|
47
|
+
assert_true m.is_a?(Mutex)
|
48
|
+
assert_equal @value, @store[key]
|
49
|
+
|
50
|
+
# access the same key once again
|
51
|
+
value2 = 'test value2'
|
52
|
+
@counter_store_mutex.synchronize(key) do |store, k|
|
53
|
+
store[k] = value2
|
54
|
+
end
|
55
|
+
|
56
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
57
|
+
m2 = mhash[key]
|
58
|
+
assert_true m2.is_a?(Mutex)
|
59
|
+
assert_equal value2, @store[key]
|
60
|
+
|
61
|
+
assert_equal m, m2
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
sub_test_case 'synchronize_key' do
|
66
|
+
test "create new mutex values if keys don't exist" do
|
67
|
+
keys = ['key', 'key1']
|
68
|
+
|
69
|
+
@counter_store_mutex.synchronize_keys(*keys) do |store, k|
|
70
|
+
store[k] = @value
|
71
|
+
end
|
72
|
+
|
73
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
74
|
+
keys.each do |key|
|
75
|
+
assert_true mhash[key].is_a?(Mutex)
|
76
|
+
assert_equal @value, @store[key]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
test 'nothing to do when an empty array passed' do
|
81
|
+
@counter_store_mutex.synchronize_keys(*[]) do |store, k|
|
82
|
+
store[k] = @value
|
83
|
+
end
|
84
|
+
|
85
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
86
|
+
assert_true mhash.empty?
|
87
|
+
assert_true @store.empty?
|
88
|
+
end
|
89
|
+
|
90
|
+
test 'use a one mutex value when the same key specified' do
|
91
|
+
key = 'key'
|
92
|
+
@counter_store_mutex.synchronize_keys(key) do |store, k|
|
93
|
+
store[k] = @value
|
94
|
+
end
|
95
|
+
|
96
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
97
|
+
m = mhash[key]
|
98
|
+
assert_true m.is_a?(Mutex)
|
99
|
+
assert_equal @value, @store[key]
|
100
|
+
|
101
|
+
# access the same key once again
|
102
|
+
value2 = 'test value2'
|
103
|
+
@counter_store_mutex.synchronize_keys(key) do |store, k|
|
104
|
+
store[k] = value2
|
105
|
+
end
|
106
|
+
|
107
|
+
mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash)
|
108
|
+
m2 = mhash[key]
|
109
|
+
assert_true m2.is_a?(Mutex)
|
110
|
+
assert_equal value2, @store[key]
|
111
|
+
|
112
|
+
assert_equal m, m2
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
class CleanupThreadTest < ::Test::Unit::TestCase
|
118
|
+
StoreValue = Struct.new(:last_modified_at)
|
119
|
+
|
120
|
+
setup do
|
121
|
+
# timecop isn't compatible with EventTime
|
122
|
+
t = Time.parse('2016-09-22 16:59:59 +0900')
|
123
|
+
Timecop.freeze(t)
|
124
|
+
|
125
|
+
@store = Fluent::Counter::Store.new
|
126
|
+
@mhash = Fluent::Counter::MutexHash.new(@store)
|
127
|
+
|
128
|
+
# stub sleep method to avoid waiting CLEANUP_INTERVAL
|
129
|
+
ct = @mhash.instance_variable_get(:@cleanup_thread)
|
130
|
+
flexstub(ct).should_receive(:sleep)
|
131
|
+
end
|
132
|
+
|
133
|
+
teardown do
|
134
|
+
@mhash.stop
|
135
|
+
Timecop.return
|
136
|
+
end
|
137
|
+
|
138
|
+
test 'clean up unused mutex' do
|
139
|
+
name = 'key1'
|
140
|
+
init_obj = { 'name' => name, 'reset_interval' => 2 }
|
141
|
+
|
142
|
+
@mhash.synchronize(init_obj['name']) do
|
143
|
+
@store.init(name, init_obj)
|
144
|
+
end
|
145
|
+
|
146
|
+
ct = @mhash.instance_variable_get(:@mutex_hash)
|
147
|
+
assert ct[name]
|
148
|
+
|
149
|
+
Timecop.travel(15 * 60 + 1) # 15 min
|
150
|
+
|
151
|
+
@mhash.start # start cleanup
|
152
|
+
sleep 1
|
153
|
+
|
154
|
+
ct = @mhash.instance_variable_get(:@mutex_hash)
|
155
|
+
assert_empty ct
|
156
|
+
|
157
|
+
@mhash.stop
|
158
|
+
end
|
159
|
+
|
160
|
+
test "don't remove when `last_modified_at` is greater than (Time.now - CLEANUP_INTERVAL)" do
|
161
|
+
name = 'key1'
|
162
|
+
init_obj = { 'name' => name, 'reset_interval' => 2 }
|
163
|
+
|
164
|
+
@mhash.synchronize(init_obj['name']) do
|
165
|
+
@store.init(name, init_obj)
|
166
|
+
end
|
167
|
+
|
168
|
+
ct = @mhash.instance_variable_get(:@mutex_hash)
|
169
|
+
assert ct[name]
|
170
|
+
|
171
|
+
@mhash.start # start cleanup
|
172
|
+
sleep 1
|
173
|
+
|
174
|
+
ct = @mhash.instance_variable_get(:@mutex_hash)
|
175
|
+
assert ct[name]
|
176
|
+
|
177
|
+
@mhash.stop
|
178
|
+
end
|
179
|
+
end
|
@@ -0,0 +1,589 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/counter/server'
|
3
|
+
require 'fluent/counter/store'
|
4
|
+
require 'fluent/time'
|
5
|
+
require 'flexmock/test_unit'
|
6
|
+
require 'timecop'
|
7
|
+
|
8
|
+
class CounterServerTest < ::Test::Unit::TestCase
|
9
|
+
setup do
|
10
|
+
# timecop isn't compatible with EventTime
|
11
|
+
t = Time.parse('2016-09-22 16:59:59 +0900')
|
12
|
+
Timecop.freeze(t)
|
13
|
+
@now = Fluent::EventTime.now
|
14
|
+
|
15
|
+
@scope = "server\tworker\tplugin"
|
16
|
+
@server_name = 'server1'
|
17
|
+
@server = Fluent::Counter::Server.new(@server_name, opt: { log: $log })
|
18
|
+
@server.instance_eval { @server.close }
|
19
|
+
end
|
20
|
+
|
21
|
+
teardown do
|
22
|
+
Timecop.return
|
23
|
+
end
|
24
|
+
|
25
|
+
def extract_value_from_counter(counter, scope, name)
|
26
|
+
store = counter.instance_variable_get(:@store).instance_variable_get(:@storage).instance_variable_get(:@store)
|
27
|
+
key = Fluent::Counter::Store.gen_key(scope, name)
|
28
|
+
store[key]
|
29
|
+
end
|
30
|
+
|
31
|
+
def travel(sec)
|
32
|
+
# Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing
|
33
|
+
# rounding errors to Time.now, we need to use Timecop.freeze() instead.
|
34
|
+
Timecop.freeze(Time.now + sec)
|
35
|
+
end
|
36
|
+
|
37
|
+
test 'raise an error when server name is invalid' do
|
38
|
+
assert_raise do
|
39
|
+
Fluent::Counter::Server.new("\tinvalid_name")
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
sub_test_case 'on_message' do
|
44
|
+
data(
|
45
|
+
establish: 'establish',
|
46
|
+
init: 'init',
|
47
|
+
delete: 'delete',
|
48
|
+
inc: 'inc',
|
49
|
+
get: 'get',
|
50
|
+
reset: 'reset',
|
51
|
+
)
|
52
|
+
test 'call valid methods' do |method|
|
53
|
+
stub(@server).send do |_m, params, scope, options|
|
54
|
+
{ 'data' => [params, scope, options] }
|
55
|
+
end
|
56
|
+
|
57
|
+
request = { 'id' => 0, 'method' => method }
|
58
|
+
expected = { 'id' => 0, 'data' => [nil, nil, nil] }
|
59
|
+
assert_equal expected, @server.on_message(request)
|
60
|
+
end
|
61
|
+
|
62
|
+
data(
|
63
|
+
missing_id: [
|
64
|
+
{ 'method' => 'init' },
|
65
|
+
{ 'code' => 'invalid_request', 'message' => 'Request should include `id`' }
|
66
|
+
],
|
67
|
+
missing_method: [
|
68
|
+
{ 'id' => 0 },
|
69
|
+
{ 'code' => 'invalid_request', 'message' => 'Request should include `method`' }
|
70
|
+
],
|
71
|
+
invalid_method: [
|
72
|
+
{ 'id' => 0, 'method' => 'invalid_method' },
|
73
|
+
{ 'code' => 'method_not_found', 'message' => 'Unknown method name passed: invalid_method' }
|
74
|
+
]
|
75
|
+
)
|
76
|
+
test 'invalid request' do |(request, error)|
|
77
|
+
expected = {
|
78
|
+
'id' => request['id'],
|
79
|
+
'data' => [],
|
80
|
+
'errors' => [error],
|
81
|
+
}
|
82
|
+
|
83
|
+
assert_equal expected, @server.on_message(request)
|
84
|
+
end
|
85
|
+
|
86
|
+
test 'return an `internal_server_error` error object when an error raises in safe_run' do
|
87
|
+
stub(@server).send do |_m, _params, _scope, _options|
|
88
|
+
raise 'Error in safe_run'
|
89
|
+
end
|
90
|
+
|
91
|
+
request = { 'id' => 0, 'method' => 'init' }
|
92
|
+
expected = {
|
93
|
+
'id' => request['id'],
|
94
|
+
'data' => [],
|
95
|
+
'errors' => [
|
96
|
+
{ 'code' => 'internal_server_error', 'message' => 'Error in safe_run' }
|
97
|
+
]
|
98
|
+
}
|
99
|
+
assert_equal expected, @server.on_message(request)
|
100
|
+
end
|
101
|
+
|
102
|
+
test 'output an error log when passed data is not Hash' do
|
103
|
+
data = 'this is not a hash'
|
104
|
+
mock($log).error("Received data is not Hash: #{data}")
|
105
|
+
@server.on_message(data)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
sub_test_case 'establish' do
|
110
|
+
test 'establish a scope in a counter' do
|
111
|
+
result = @server.send('establish', ['key'], nil, nil)
|
112
|
+
expected = { 'data' => ["#{@server_name}\tkey"] }
|
113
|
+
assert_equal expected, result
|
114
|
+
end
|
115
|
+
|
116
|
+
data(
|
117
|
+
empty: [[], 'One or more `params` are required'],
|
118
|
+
empty_key: [[''], '`scope` is the invalid format'],
|
119
|
+
invalid_key: [['_key'], '`scope` is the invalid format'],
|
120
|
+
)
|
121
|
+
test 'raise an error: invalid_params' do |(params, msg)|
|
122
|
+
result = @server.send('establish', params, nil, nil)
|
123
|
+
expected = {
|
124
|
+
'data' => [],
|
125
|
+
'errors' => [{ 'code' => 'invalid_params', 'message' => msg }]
|
126
|
+
}
|
127
|
+
assert_equal expected, result
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
sub_test_case 'init' do
|
132
|
+
setup do
|
133
|
+
@name = 'key1'
|
134
|
+
@key = Fluent::Counter::Store.gen_key(@scope, @name)
|
135
|
+
end
|
136
|
+
|
137
|
+
test 'create new value in a counter' do
|
138
|
+
assert_nil extract_value_from_counter(@server, @scope, @name)
|
139
|
+
result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})
|
140
|
+
assert_nil result['errors']
|
141
|
+
|
142
|
+
counter = result['data'].first
|
143
|
+
assert_equal 'numeric', counter['type']
|
144
|
+
assert_equal @name, counter['name']
|
145
|
+
assert_equal 0, counter['current']
|
146
|
+
assert_equal 0, counter['total']
|
147
|
+
assert_equal @now, counter['last_reset_at']
|
148
|
+
assert extract_value_from_counter(@server, @scope, @name)
|
149
|
+
end
|
150
|
+
|
151
|
+
data(
|
152
|
+
numeric: 'numeric',
|
153
|
+
integer: 'integer',
|
154
|
+
float: 'float'
|
155
|
+
)
|
156
|
+
test 'set the type of a counter value' do |type|
|
157
|
+
result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1, 'type' => type }], @scope, {})
|
158
|
+
counter = result['data'].first
|
159
|
+
assert_equal type, counter['type']
|
160
|
+
|
161
|
+
v = extract_value_from_counter(@server, @scope, @name)
|
162
|
+
assert_equal type, v['type']
|
163
|
+
end
|
164
|
+
|
165
|
+
data(
|
166
|
+
empty: [[], 'One or more `params` are required'],
|
167
|
+
missing_name: [
|
168
|
+
[{ 'rest_interval' => 20 }],
|
169
|
+
'`name` is required'
|
170
|
+
],
|
171
|
+
invalid_name: [
|
172
|
+
[{ 'name' => '_test', 'reset_interval' => 20 }],
|
173
|
+
'`name` is the invalid format'
|
174
|
+
],
|
175
|
+
missing_interval: [
|
176
|
+
[{ 'name' => 'test' }],
|
177
|
+
'`reset_interval` is required'
|
178
|
+
],
|
179
|
+
minus_interval: [
|
180
|
+
[{ 'name' => 'test', 'reset_interval' => -1 }],
|
181
|
+
'`reset_interval` should be a positive number'
|
182
|
+
],
|
183
|
+
invalid_type: [
|
184
|
+
[{ 'name' => 'test', 'reset_interval' => 1, 'type' => 'invalid_type' }],
|
185
|
+
'`type` should be integer, float, or numeric'
|
186
|
+
]
|
187
|
+
)
|
188
|
+
test 'return an error object: invalid_params' do |(params, msg)|
|
189
|
+
result = @server.send('init', params, @scope, {})
|
190
|
+
assert_empty result['data']
|
191
|
+
error = result['errors'].first
|
192
|
+
assert_equal 'invalid_params', error['code']
|
193
|
+
assert_equal msg, error['message']
|
194
|
+
end
|
195
|
+
|
196
|
+
test 'return error objects when passed key already exists' do
|
197
|
+
@server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})
|
198
|
+
|
199
|
+
# call `init` to same key twice
|
200
|
+
result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})
|
201
|
+
assert_empty result['data']
|
202
|
+
error = result['errors'].first
|
203
|
+
|
204
|
+
expected = { 'code' => 'invalid_params', 'message' => "#{@key} already exists in counter" }
|
205
|
+
assert_equal expected, error
|
206
|
+
end
|
207
|
+
|
208
|
+
test 'return existing value when passed key already exists and ignore option is true' do
|
209
|
+
v1 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})
|
210
|
+
|
211
|
+
# call `init` to same key twice
|
212
|
+
v2 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'ignore' => true })
|
213
|
+
assert_nil v2['errors']
|
214
|
+
assert_equal v1['data'], v2['data']
|
215
|
+
end
|
216
|
+
|
217
|
+
test 'call `synchronize_keys` when random option is true' do
|
218
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
219
|
+
mock(mhash).synchronize(@key).once
|
220
|
+
@server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {})
|
221
|
+
|
222
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
223
|
+
mock(mhash).synchronize_keys(@key).once
|
224
|
+
@server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'random' => true })
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
sub_test_case 'delete' do
|
229
|
+
setup do
|
230
|
+
@name = 'key1'
|
231
|
+
@key = Fluent::Counter::Store.gen_key(@scope, @name)
|
232
|
+
@server.send('init', [{ 'name' => @name, 'reset_interval' => 20 }], @scope, {})
|
233
|
+
end
|
234
|
+
|
235
|
+
test 'delete a value from a counter' do
|
236
|
+
assert extract_value_from_counter(@server, @scope, @name)
|
237
|
+
|
238
|
+
result = @server.send('delete', [@name], @scope, {})
|
239
|
+
assert_nil result['errors']
|
240
|
+
|
241
|
+
counter = result['data'].first
|
242
|
+
assert_equal 0, counter['current']
|
243
|
+
assert_equal 0, counter['total']
|
244
|
+
assert_equal 'numeric', counter['type']
|
245
|
+
assert_equal @name, counter['name']
|
246
|
+
assert_equal @now, counter['last_reset_at']
|
247
|
+
|
248
|
+
assert_nil extract_value_from_counter(@server, @scope, @name)
|
249
|
+
end
|
250
|
+
|
251
|
+
data(
|
252
|
+
empty: [[], 'One or more `params` are required'],
|
253
|
+
empty_key: [[''], '`key` is the invalid format'],
|
254
|
+
invalid_key: [['_key'], '`key` is the invalid format'],
|
255
|
+
)
|
256
|
+
test 'return an error object: invalid_params' do |(params, msg)|
|
257
|
+
result = @server.send('delete', params, @scope, {})
|
258
|
+
|
259
|
+
assert_empty result['data']
|
260
|
+
error = result['errors'].first
|
261
|
+
assert_equal 'invalid_params', error['code']
|
262
|
+
assert_equal msg, error['message']
|
263
|
+
end
|
264
|
+
|
265
|
+
test 'return an error object: unknown_key' do
|
266
|
+
unknown_key = 'unknown_key'
|
267
|
+
result = @server.send('delete', [unknown_key], @scope, {})
|
268
|
+
|
269
|
+
assert_empty result['data']
|
270
|
+
error = result['errors'].first
|
271
|
+
assert_equal unknown_key, error['code']
|
272
|
+
assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message']
|
273
|
+
end
|
274
|
+
|
275
|
+
test 'call `synchronize_keys` when random option is true' do
|
276
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
277
|
+
mock(mhash).synchronize(@key).once
|
278
|
+
@server.send('delete', [@name], @scope, {})
|
279
|
+
|
280
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
281
|
+
mock(mhash).synchronize_keys(@key).once
|
282
|
+
@server.send('delete', [@name], @scope, { 'random' => true })
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
sub_test_case 'inc' do
|
287
|
+
setup do
|
288
|
+
@name1 = 'key1'
|
289
|
+
@name2 = 'key2'
|
290
|
+
@key1 = Fluent::Counter::Store.gen_key(@scope, @name1)
|
291
|
+
inc_objects = [
|
292
|
+
{ 'name' => @name1, 'reset_interval' => 20 },
|
293
|
+
{ 'name' => @name2, 'type' => 'integer', 'reset_interval' => 20 }
|
294
|
+
]
|
295
|
+
@server.send('init', inc_objects, @scope, {})
|
296
|
+
end
|
297
|
+
|
298
|
+
test 'increment or decrement a value in counter' do
|
299
|
+
result = @server.send('inc', [{ 'name' => @name1, 'value' => 10 }], @scope, {})
|
300
|
+
assert_nil result['errors']
|
301
|
+
|
302
|
+
counter = result['data'].first
|
303
|
+
assert_equal 10, counter['current']
|
304
|
+
assert_equal 10, counter['total']
|
305
|
+
assert_equal 'numeric', counter['type']
|
306
|
+
assert_equal @name1, counter['name']
|
307
|
+
assert_equal @now, counter['last_reset_at']
|
308
|
+
|
309
|
+
c = extract_value_from_counter(@server, @scope, @name1)
|
310
|
+
assert_equal 10, c['current']
|
311
|
+
assert_equal 10, c['total']
|
312
|
+
assert_equal @now, Fluent::EventTime.new(*c['last_reset_at'])
|
313
|
+
assert_equal @now, Fluent::EventTime.new(*c['last_modified_at'])
|
314
|
+
end
|
315
|
+
|
316
|
+
test 'create new value and increment/decrement its value when `force` option is true' do
|
317
|
+
new_name = 'new_key'
|
318
|
+
assert_nil extract_value_from_counter(@server, @scope, new_name)
|
319
|
+
|
320
|
+
v1 = @server.send('inc', [{ 'name' => new_name, 'value' => 10 }], @scope, {})
|
321
|
+
assert_empty v1['data']
|
322
|
+
error = v1['errors'].first
|
323
|
+
assert_equal 'unknown_key', error['code']
|
324
|
+
|
325
|
+
assert_nil extract_value_from_counter(@server, @scope, new_name)
|
326
|
+
|
327
|
+
v2 = @server.send(
|
328
|
+
'inc',
|
329
|
+
[{ 'name' => new_name, 'value' => 10, 'reset_interval' => 20 }],
|
330
|
+
@scope,
|
331
|
+
{ 'force' => true }
|
332
|
+
)
|
333
|
+
|
334
|
+
assert_nil v2['errors']
|
335
|
+
|
336
|
+
counter = v2['data'].first
|
337
|
+
assert_equal 10, counter['current']
|
338
|
+
assert_equal 10, counter['total']
|
339
|
+
assert_equal 'numeric', counter['type']
|
340
|
+
assert_equal new_name, counter['name']
|
341
|
+
assert_equal @now, counter['last_reset_at']
|
342
|
+
|
343
|
+
assert extract_value_from_counter(@server, @scope, new_name)
|
344
|
+
end
|
345
|
+
|
346
|
+
data(
|
347
|
+
empty: [[], 'One or more `params` are required', {}],
|
348
|
+
missing_name: [
|
349
|
+
[{ 'value' => 10 }],
|
350
|
+
'`name` is required', {}
|
351
|
+
],
|
352
|
+
missing_value: [
|
353
|
+
[{ 'name' => 'key1' }],
|
354
|
+
'`value` is required', {}
|
355
|
+
],
|
356
|
+
invalid_type: [
|
357
|
+
[{ 'name' => 'key2', 'value' => 10.0 }],
|
358
|
+
'`type` is integer. You should pass integer value as a `value`', {}
|
359
|
+
],
|
360
|
+
missing_reset_interval: [
|
361
|
+
[{ 'name' => 'key1', 'value' => 1 }],
|
362
|
+
'`reset_interval` is required',
|
363
|
+
{ 'force' => true }
|
364
|
+
]
|
365
|
+
)
|
366
|
+
test 'return an error object: invalid_params' do |(params, msg, opt)|
|
367
|
+
result = @server.send('inc', params, @scope, opt)
|
368
|
+
assert_empty result['data']
|
369
|
+
|
370
|
+
error = result['errors'].first
|
371
|
+
assert_equal 'invalid_params', error['code']
|
372
|
+
assert_equal msg, error['message']
|
373
|
+
end
|
374
|
+
|
375
|
+
test 'call `synchronize_keys` when random option is true' do
|
376
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
377
|
+
mock(mhash).synchronize(@key1).once
|
378
|
+
params = [{ 'name' => @name1, 'value' => 1 }]
|
379
|
+
@server.send('inc', params, @scope, {})
|
380
|
+
|
381
|
+
mhash = @server.instance_variable_get(:@mutex_hash)
|
382
|
+
mock(mhash).synchronize_keys(@key1).once
|
383
|
+
@server.send('inc', params, @scope, { 'random' => true })
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
sub_test_case 'reset' do
|
388
|
+
setup do
|
389
|
+
@name = 'key'
|
390
|
+
@travel_sec = 10
|
391
|
+
@server.send('init', [{ 'name' => @name, 'reset_interval' => 10 }], @scope, {})
|
392
|
+
@server.send('inc', [{ 'name' => @name, 'value' => 10 }], @scope, {})
|
393
|
+
end
|
394
|
+
|
395
|
+
test 'reset a value in the counter' do
|
396
|
+
travel(@travel_sec)
|
397
|
+
|
398
|
+
result = @server.send('reset', [@name], @scope, {})
|
399
|
+
assert_nil result['errors']
|
400
|
+
|
401
|
+
data = result['data'].first
|
402
|
+
assert_true data['success']
|
403
|
+
assert_equal @travel_sec, data['elapsed_time']
|
404
|
+
|
405
|
+
counter = data['counter_data']
|
406
|
+
assert_equal 10, counter['current']
|
407
|
+
assert_equal 10, counter['total']
|
408
|
+
assert_equal 'numeric', counter['type']
|
409
|
+
assert_equal @name, counter['name']
|
410
|
+
assert_equal @now, counter['last_reset_at']
|
411
|
+
|
412
|
+
v = extract_value_from_counter(@server, @scope, @name)
|
413
|
+
assert_equal 0, v['current']
|
414
|
+
assert_equal 10, v['total']
|
415
|
+
assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_reset_at'])
|
416
|
+
assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_modified_at'])
|
417
|
+
end
|
418
|
+
|
419
|
+
test 'reset a value after `reset_interval` passed' do
|
420
|
+
first_travel_sec = 5
|
421
|
+
travel(first_travel_sec) # jump time less than reset_interval
|
422
|
+
result = @server.send('reset', [@name], @scope, {})
|
423
|
+
v = result['data'].first
|
424
|
+
|
425
|
+
assert_equal false, v['success']
|
426
|
+
assert_equal first_travel_sec, v['elapsed_time']
|
427
|
+
|
428
|
+
store = extract_value_from_counter(@server, @scope, @name)
|
429
|
+
assert_equal 10, store['current']
|
430
|
+
assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])
|
431
|
+
|
432
|
+
# time is passed greater than reset_interval
|
433
|
+
travel(@travel_sec)
|
434
|
+
result = @server.send('reset', [@name], @scope, {})
|
435
|
+
v = result['data'].first
|
436
|
+
|
437
|
+
assert_true v['success']
|
438
|
+
assert_equal @travel_sec + first_travel_sec, v['elapsed_time']
|
439
|
+
|
440
|
+
v1 = extract_value_from_counter(@server, @scope, @name)
|
441
|
+
assert_equal 0, v1['current']
|
442
|
+
assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
|
443
|
+
assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
|
444
|
+
end
|
445
|
+
|
446
|
+
data(
|
447
|
+
empty: [[], 'One or more `params` are required'],
|
448
|
+
empty_key: [[''], '`key` is the invalid format'],
|
449
|
+
invalid_key: [['_key'], '`key` is the invalid format'],
|
450
|
+
)
|
451
|
+
test 'return an error object: invalid_params' do |(params, msg)|
|
452
|
+
result = @server.send('reset', params, @scope, {})
|
453
|
+
assert_empty result['data']
|
454
|
+
assert_equal 'invalid_params', result['errors'].first['code']
|
455
|
+
assert_equal msg, result['errors'].first['message']
|
456
|
+
end
|
457
|
+
|
458
|
+
test 'return an error object: unknown_key' do
|
459
|
+
unknown_key = 'unknown_key'
|
460
|
+
result = @server.send('reset', [unknown_key], @scope, {})
|
461
|
+
|
462
|
+
assert_empty result['data']
|
463
|
+
error = result['errors'].first
|
464
|
+
assert_equal unknown_key, error['code']
|
465
|
+
assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message']
|
466
|
+
end
|
467
|
+
end
|
468
|
+
|
469
|
+
sub_test_case 'get' do
|
470
|
+
setup do
|
471
|
+
@name1 = 'key1'
|
472
|
+
@name2 = 'key2'
|
473
|
+
init_objects = [
|
474
|
+
{ 'name' => @name1, 'reset_interval' => 0 },
|
475
|
+
{ 'name' => @name2, 'reset_interval' => 0 },
|
476
|
+
]
|
477
|
+
@server.send('init', init_objects, @scope, {})
|
478
|
+
end
|
479
|
+
|
480
|
+
test 'get a counter value' do
|
481
|
+
key = @name1
|
482
|
+
result = @server.send('get', [key], @scope, {})
|
483
|
+
assert_nil result['errors']
|
484
|
+
|
485
|
+
counter = result['data'].first
|
486
|
+
assert_equal 0, counter['current']
|
487
|
+
assert_equal 0, counter['total']
|
488
|
+
assert_equal 'numeric', counter['type']
|
489
|
+
assert_equal key, counter['name']
|
490
|
+
end
|
491
|
+
|
492
|
+
test 'get counter values' do
|
493
|
+
result = @server.send('get', [@name1, @name2], @scope, {})
|
494
|
+
assert_nil result['errors']
|
495
|
+
|
496
|
+
counter1 = result['data'][0]
|
497
|
+
assert_equal 0, counter1['current']
|
498
|
+
assert_equal 0, counter1['total']
|
499
|
+
assert_equal 'numeric', counter1['type']
|
500
|
+
assert_equal @name1, counter1['name']
|
501
|
+
|
502
|
+
counter2 = result['data'][1]
|
503
|
+
assert_equal 0, counter2['current']
|
504
|
+
assert_equal 0, counter2['total']
|
505
|
+
assert_equal 'numeric', counter2['type']
|
506
|
+
assert_equal @name2, counter2['name']
|
507
|
+
end
|
508
|
+
|
509
|
+
data(
|
510
|
+
empty: [[], 'One or more `params` are required'],
|
511
|
+
empty_key: [[''], '`key` is the invalid format'],
|
512
|
+
invalid_key: [['_key'], '`key` is the invalid format'],
|
513
|
+
)
|
514
|
+
test 'return an error object: invalid_params' do |(params, msg)|
|
515
|
+
result = @server.send('get', params, @scope, {})
|
516
|
+
assert_empty result['data']
|
517
|
+
assert_equal 'invalid_params', result['errors'].first['code']
|
518
|
+
assert_equal msg, result['errors'].first['message']
|
519
|
+
end
|
520
|
+
|
521
|
+
test 'return an error object: unknown_key' do
|
522
|
+
unknown_key = 'unknown_key'
|
523
|
+
result = @server.send('get', [unknown_key], @scope, {})
|
524
|
+
|
525
|
+
assert_empty result['data']
|
526
|
+
error = result['errors'].first
|
527
|
+
assert_equal unknown_key, error['code']
|
528
|
+
assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message']
|
529
|
+
end
|
530
|
+
end
|
531
|
+
end
|
532
|
+
|
533
|
+
class CounterCounterResponseTest < ::Test::Unit::TestCase
|
534
|
+
setup do
|
535
|
+
@response = Fluent::Counter::Server::Response.new
|
536
|
+
@errors = [
|
537
|
+
StandardError.new('standard error'),
|
538
|
+
Fluent::Counter::InternalServerError.new('internal server error')
|
539
|
+
]
|
540
|
+
@now = Fluent::EventTime.now
|
541
|
+
value = {
|
542
|
+
'name' => 'name',
|
543
|
+
'total' => 100,
|
544
|
+
'current' => 11,
|
545
|
+
'type' => 'numeric',
|
546
|
+
'reset_interval' => 10,
|
547
|
+
'last_reset_at' => @now,
|
548
|
+
}
|
549
|
+
@values = [value, 'test']
|
550
|
+
end
|
551
|
+
|
552
|
+
test 'push_error' do
|
553
|
+
@errors.each do |e|
|
554
|
+
@response.push_error(e)
|
555
|
+
end
|
556
|
+
|
557
|
+
v = @response.instance_variable_get(:@errors)
|
558
|
+
assert_equal @errors, v
|
559
|
+
end
|
560
|
+
|
561
|
+
test 'push_data' do
|
562
|
+
@values.each do |v|
|
563
|
+
@response.push_data v
|
564
|
+
end
|
565
|
+
|
566
|
+
data = @response.instance_variable_get(:@data)
|
567
|
+
assert_equal @values, data
|
568
|
+
end
|
569
|
+
|
570
|
+
test 'to_hash' do
|
571
|
+
@errors.each do |e|
|
572
|
+
@response.push_error(e)
|
573
|
+
end
|
574
|
+
|
575
|
+
@values.each do |v|
|
576
|
+
@response.push_data v
|
577
|
+
end
|
578
|
+
|
579
|
+
expected_errors = [
|
580
|
+
{ 'code' => 'internal_server_error', 'message' => 'standard error' },
|
581
|
+
{ 'code' => 'internal_server_error', 'message' => 'internal server error' }
|
582
|
+
]
|
583
|
+
expected_data = @values
|
584
|
+
|
585
|
+
hash = @response.to_hash
|
586
|
+
assert_equal expected_errors, hash['errors']
|
587
|
+
assert_equal expected_data, hash['data']
|
588
|
+
end
|
589
|
+
end
|