fluent-plugin-cloudwatch-logs 0.7.3 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +8 -0
- data/README.md +1 -0
- data/Rakefile +1 -0
- data/fluent-plugin-cloudwatch-logs.gemspec +2 -1
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +16 -12
- data/test/plugin/test_in_cloudwatch_logs.rb +444 -161
- data/test/plugin/test_out_cloudwatch_logs.rb +575 -567
- data/test/test_helper.rb +2 -1
- metadata +21 -6
@@ -3,6 +3,7 @@ require 'test_helper'
|
|
3
3
|
require 'fileutils'
|
4
4
|
require 'fluent/test/driver/output'
|
5
5
|
require 'fluent/test/helpers'
|
6
|
+
require 'fluent/plugin/out_cloudwatch_logs'
|
6
7
|
|
7
8
|
class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
8
9
|
include CloudwatchLogsTestHelper
|
@@ -10,698 +11,705 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
10
11
|
|
11
12
|
def setup
|
12
13
|
Fluent::Test.setup
|
13
|
-
require 'fluent/plugin/out_cloudwatch_logs'
|
14
14
|
end
|
15
15
|
|
16
|
-
|
17
|
-
|
16
|
+
sub_test_case "configure" do
|
17
|
+
def test_configure
|
18
|
+
d = create_driver(<<-EOC)
|
19
|
+
@type cloudwatch_logs
|
20
|
+
aws_key_id test_id
|
21
|
+
aws_sec_key test_key
|
22
|
+
region us-east-1
|
23
|
+
log_group_name test_group
|
24
|
+
log_stream_name test_stream
|
25
|
+
auto_create_stream false
|
26
|
+
log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"}
|
27
|
+
retention_in_days 5
|
28
|
+
message_keys fluentd, aws, cloudwatch
|
29
|
+
EOC
|
30
|
+
|
31
|
+
assert_equal('test_id', d.instance.aws_key_id)
|
32
|
+
assert_equal('test_key', d.instance.aws_sec_key)
|
33
|
+
assert_equal('us-east-1', d.instance.region)
|
34
|
+
assert_equal('test_group', d.instance.log_group_name)
|
35
|
+
assert_equal('test_stream', d.instance.log_stream_name)
|
36
|
+
assert_equal(false, d.instance.auto_create_stream)
|
37
|
+
assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
|
38
|
+
assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
|
39
|
+
assert_equal(5, d.instance.retention_in_days)
|
40
|
+
assert_equal(:yajl, d.instance.json_handler)
|
41
|
+
assert_equal(["fluentd","aws","cloudwatch"], d.instance.message_keys)
|
42
|
+
end
|
18
43
|
end
|
19
44
|
|
45
|
+
sub_test_case "real world" do
|
46
|
+
def setup
|
47
|
+
omit if ENV["CI"] == "true"
|
48
|
+
end
|
49
|
+
|
50
|
+
def teardown
|
51
|
+
return if ENV["CI"] == "true"
|
52
|
+
clear_log_group
|
53
|
+
end
|
20
54
|
|
21
|
-
|
22
|
-
|
23
|
-
@type cloudwatch_logs
|
24
|
-
aws_key_id test_id
|
25
|
-
aws_sec_key test_key
|
26
|
-
region us-east-1
|
27
|
-
log_group_name test_group
|
28
|
-
log_stream_name test_stream
|
29
|
-
auto_create_stream false
|
30
|
-
log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"}
|
31
|
-
retention_in_days 5
|
32
|
-
message_keys fluentd, aws, cloudwatch
|
33
|
-
EOC
|
55
|
+
def test_write
|
56
|
+
new_log_stream
|
34
57
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
|
43
|
-
assert_equal(5, d.instance.retention_in_days)
|
44
|
-
assert_equal(:yajl, d.instance.json_handler)
|
45
|
-
assert_equal(["fluentd","aws","cloudwatch"], d.instance.message_keys)
|
46
|
-
end
|
58
|
+
d = create_driver
|
59
|
+
time = event_time
|
60
|
+
d.run(default_tag: fluentd_tag, flush: true) do
|
61
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
62
|
+
# Addition converts EventTime to seconds
|
63
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
64
|
+
end
|
47
65
|
|
48
|
-
|
49
|
-
new_log_stream
|
66
|
+
sleep 10
|
50
67
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
68
|
+
logs = d.logs
|
69
|
+
events = get_log_events
|
70
|
+
assert_equal(2, events.size)
|
71
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
72
|
+
assert_equal('{"cloudwatch":"logs1"}', events[0].message)
|
73
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
74
|
+
assert_equal('{"cloudwatch":"logs2"}', events[1].message)
|
58
75
|
|
59
|
-
|
76
|
+
assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
|
77
|
+
end
|
60
78
|
|
61
|
-
|
62
|
-
|
63
|
-
assert_equal(2, events.size)
|
64
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
65
|
-
assert_equal('{"cloudwatch":"logs1"}', events[0].message)
|
66
|
-
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
67
|
-
assert_equal('{"cloudwatch":"logs2"}', events[1].message)
|
79
|
+
def test_write_utf8
|
80
|
+
new_log_stream
|
68
81
|
|
69
|
-
|
70
|
-
|
82
|
+
d = create_driver
|
83
|
+
time = event_time
|
84
|
+
d.run(default_tag: fluentd_tag) do
|
85
|
+
d.feed(time, { 'cloudwatch' => 'これは日本語です'.force_encoding('UTF-8')})
|
86
|
+
end
|
71
87
|
|
72
|
-
|
73
|
-
new_log_stream
|
88
|
+
sleep 10
|
74
89
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
90
|
+
events = get_log_events
|
91
|
+
assert_equal(1, events.size)
|
92
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
93
|
+
assert_equal('{"cloudwatch":"これは日本語です"}', events[0].message)
|
79
94
|
end
|
80
95
|
|
81
|
-
|
96
|
+
def test_write_24h_apart
|
97
|
+
new_log_stream
|
82
98
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
99
|
+
d = create_driver(<<-EOC)
|
100
|
+
#{default_config}
|
101
|
+
log_group_name #{log_group_name}
|
102
|
+
log_stream_name #{log_stream_name}
|
103
|
+
utc
|
104
|
+
EOC
|
105
|
+
time = event_time
|
106
|
+
d.run(default_tag: fluentd_tag) do
|
107
|
+
d.feed(time - 60 * 60 * 25, {'cloudwatch' => 'logs0'})
|
108
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
109
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
110
|
+
end
|
88
111
|
|
89
|
-
|
90
|
-
new_log_stream
|
112
|
+
sleep 10
|
91
113
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
d.feed(time - 60 * 60 * 25, {'cloudwatch' => 'logs0'})
|
101
|
-
d.feed(time, {'cloudwatch' => 'logs1'})
|
102
|
-
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
114
|
+
events = get_log_events
|
115
|
+
assert_equal(3, events.size)
|
116
|
+
assert_equal((time.to_i - 60 * 60 * 25) * 1000, events[0].timestamp)
|
117
|
+
assert_equal('{"cloudwatch":"logs0"}', events[0].message)
|
118
|
+
assert_equal((time.to_f * 1000).floor, events[1].timestamp)
|
119
|
+
assert_equal('{"cloudwatch":"logs1"}', events[1].message)
|
120
|
+
assert_equal((time.to_i + 1) * 1000, events[2].timestamp)
|
121
|
+
assert_equal('{"cloudwatch":"logs2"}', events[2].message)
|
103
122
|
end
|
104
123
|
|
105
|
-
|
124
|
+
def test_write_with_message_keys
|
125
|
+
new_log_stream
|
106
126
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
assert_equal((time.to_i + 1) * 1000, events[2].timestamp)
|
114
|
-
assert_equal('{"cloudwatch":"logs2"}', events[2].message)
|
115
|
-
end
|
127
|
+
d = create_driver(<<-EOC)
|
128
|
+
#{default_config}
|
129
|
+
message_keys message,cloudwatch
|
130
|
+
log_group_name #{log_group_name}
|
131
|
+
log_stream_name #{log_stream_name}
|
132
|
+
EOC
|
116
133
|
|
117
|
-
|
118
|
-
|
134
|
+
time = event_time
|
135
|
+
d.run(default_tag: fluentd_tag) do
|
136
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
137
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
138
|
+
end
|
119
139
|
|
120
|
-
|
121
|
-
#{default_config}
|
122
|
-
message_keys message,cloudwatch
|
123
|
-
log_group_name #{log_group_name}
|
124
|
-
log_stream_name #{log_stream_name}
|
125
|
-
EOC
|
140
|
+
sleep 10
|
126
141
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
142
|
+
events = get_log_events
|
143
|
+
assert_equal(2, events.size)
|
144
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
145
|
+
assert_equal('message1 logs1', events[0].message)
|
146
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
147
|
+
assert_equal('message2 logs2', events[1].message)
|
131
148
|
end
|
132
149
|
|
133
|
-
|
150
|
+
def test_write_with_max_message_length
|
151
|
+
new_log_stream
|
134
152
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
153
|
+
d = create_driver(<<-EOC)
|
154
|
+
#{default_config}
|
155
|
+
message_keys message,cloudwatch
|
156
|
+
max_message_length 10
|
157
|
+
log_group_name #{log_group_name}
|
158
|
+
log_stream_name #{log_stream_name}
|
159
|
+
EOC
|
142
160
|
|
143
|
-
|
144
|
-
|
161
|
+
time = event_time
|
162
|
+
d.run(default_tag: fluentd_tag) do
|
163
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
164
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
165
|
+
end
|
145
166
|
|
146
|
-
|
147
|
-
#{default_config}
|
148
|
-
message_keys message,cloudwatch
|
149
|
-
max_message_length 10
|
150
|
-
log_group_name #{log_group_name}
|
151
|
-
log_stream_name #{log_stream_name}
|
152
|
-
EOC
|
167
|
+
sleep 10
|
153
168
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
169
|
+
events = get_log_events
|
170
|
+
assert_equal(2, events.size)
|
171
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
172
|
+
assert_equal('message1 l', events[0].message)
|
173
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
174
|
+
assert_equal('message2 l', events[1].message)
|
158
175
|
end
|
159
176
|
|
160
|
-
|
177
|
+
def test_write_use_tag_as_group
|
178
|
+
new_log_stream
|
161
179
|
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
end
|
180
|
+
d = create_driver(<<-EOC)
|
181
|
+
#{default_config}
|
182
|
+
message_keys message,cloudwatch
|
183
|
+
use_tag_as_group true
|
184
|
+
log_stream_name #{log_stream_name}
|
185
|
+
EOC
|
169
186
|
|
170
|
-
|
171
|
-
|
187
|
+
time = event_time
|
188
|
+
d.run(default_tag: fluentd_tag) do
|
189
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
190
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
191
|
+
end
|
172
192
|
|
173
|
-
|
174
|
-
#{default_config}
|
175
|
-
message_keys message,cloudwatch
|
176
|
-
use_tag_as_group true
|
177
|
-
log_stream_name #{log_stream_name}
|
178
|
-
EOC
|
193
|
+
sleep 10
|
179
194
|
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
195
|
+
events = get_log_events(fluentd_tag)
|
196
|
+
assert_equal(2, events.size)
|
197
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
198
|
+
assert_equal('message1 logs1', events[0].message)
|
199
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
200
|
+
assert_equal('message2 logs2', events[1].message)
|
184
201
|
end
|
185
202
|
|
186
|
-
|
203
|
+
def test_write_use_tag_as_stream
|
204
|
+
new_log_stream
|
187
205
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
end
|
206
|
+
d = create_driver(<<-EOC)
|
207
|
+
#{default_config}
|
208
|
+
message_keys message,cloudwatch
|
209
|
+
use_tag_as_stream true
|
210
|
+
log_group_name #{log_group_name}
|
211
|
+
EOC
|
195
212
|
|
196
|
-
|
197
|
-
|
213
|
+
time = event_time
|
214
|
+
d.run(default_tag: fluentd_tag) do
|
215
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
216
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
217
|
+
end
|
198
218
|
|
199
|
-
|
200
|
-
#{default_config}
|
201
|
-
message_keys message,cloudwatch
|
202
|
-
use_tag_as_stream true
|
203
|
-
log_group_name #{log_group_name}
|
204
|
-
EOC
|
219
|
+
sleep 10
|
205
220
|
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
221
|
+
events = get_log_events(log_group_name, fluentd_tag)
|
222
|
+
assert_equal(2, events.size)
|
223
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
224
|
+
assert_equal('message1 logs1', events[0].message)
|
225
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
226
|
+
assert_equal('message2 logs2', events[1].message)
|
210
227
|
end
|
211
228
|
|
212
|
-
|
229
|
+
def test_write_use_placeholders
|
230
|
+
new_log_stream
|
231
|
+
|
232
|
+
config = {'@type' => 'cloudwatch_logs',
|
233
|
+
'auto_create_stream' => true,
|
234
|
+
'message_keys' => ["message","cloudwatch"],
|
235
|
+
'log_stream_name' => "${tag}",
|
236
|
+
'log_group_name' => log_group_name}
|
237
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
238
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
239
|
+
config.merge!(config_elementify(region)) if region
|
240
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
241
|
+
|
242
|
+
d = create_driver(
|
243
|
+
Fluent::Config::Element.new('ROOT', '', config,[
|
244
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
245
|
+
'@type' => 'memory',
|
246
|
+
'timekey' => 3600
|
247
|
+
}, [])
|
248
|
+
])
|
249
|
+
)
|
250
|
+
|
251
|
+
time = event_time
|
252
|
+
d.run(default_tag: fluentd_tag) do
|
253
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
254
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
255
|
+
end
|
213
256
|
|
214
|
-
|
215
|
-
assert_equal(2, events.size)
|
216
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
217
|
-
assert_equal('message1 logs1', events[0].message)
|
218
|
-
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
219
|
-
assert_equal('message2 logs2', events[1].message)
|
220
|
-
end
|
257
|
+
sleep 10
|
221
258
|
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
'log_stream_name' => "${tag}",
|
229
|
-
'log_group_name' => log_group_name}
|
230
|
-
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
231
|
-
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
232
|
-
config.merge!(config_elementify(region)) if region
|
233
|
-
config.merge!(config_elementify(endpoint)) if endpoint
|
234
|
-
|
235
|
-
d = create_driver(
|
236
|
-
Fluent::Config::Element.new('ROOT', '', config,[
|
237
|
-
Fluent::Config::Element.new('buffer', 'tag, time', {
|
238
|
-
'@type' => 'memory',
|
239
|
-
'timekey' => 3600
|
240
|
-
}, [])
|
241
|
-
])
|
242
|
-
)
|
243
|
-
|
244
|
-
time = event_time
|
245
|
-
d.run(default_tag: fluentd_tag) do
|
246
|
-
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
247
|
-
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
259
|
+
events = get_log_events(log_group_name, fluentd_tag)
|
260
|
+
assert_equal(2, events.size)
|
261
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
262
|
+
assert_equal('message1 logs1', events[0].message)
|
263
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
264
|
+
assert_equal('message2 logs2', events[1].message)
|
248
265
|
end
|
249
266
|
|
250
|
-
|
267
|
+
def test_write_use_placeholders_parts
|
268
|
+
new_log_stream
|
269
|
+
|
270
|
+
config = {'@type' => 'cloudwatch_logs',
|
271
|
+
'auto_create_stream' => true,
|
272
|
+
'message_keys' => ["message","cloudwatch"],
|
273
|
+
'log_stream_name' => "${tag[0]}-${tag[1]}-${tag[2]}-${tag[3]}",
|
274
|
+
'log_group_name' => log_group_name}
|
275
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
276
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
277
|
+
config.merge!(config_elementify(region)) if region
|
278
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
279
|
+
|
280
|
+
d = create_driver(
|
281
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
282
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
283
|
+
'@type' => 'memory',
|
284
|
+
'timekey' => 3600
|
285
|
+
}, [])
|
286
|
+
])
|
287
|
+
)
|
288
|
+
|
289
|
+
time = event_time
|
290
|
+
d.run(default_tag: fluentd_tag) do
|
291
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
292
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
293
|
+
end
|
251
294
|
|
252
|
-
|
253
|
-
assert_equal(2, events.size)
|
254
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
255
|
-
assert_equal('message1 logs1', events[0].message)
|
256
|
-
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
257
|
-
assert_equal('message2 logs2', events[1].message)
|
258
|
-
end
|
295
|
+
sleep 10
|
259
296
|
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
'log_stream_name' => "${tag[0]}-${tag[1]}-${tag[2]}-${tag[3]}",
|
267
|
-
'log_group_name' => log_group_name}
|
268
|
-
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
269
|
-
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
270
|
-
config.merge!(config_elementify(region)) if region
|
271
|
-
config.merge!(config_elementify(endpoint)) if endpoint
|
272
|
-
|
273
|
-
d = create_driver(
|
274
|
-
Fluent::Config::Element.new('ROOT', '', config,[
|
275
|
-
Fluent::Config::Element.new('buffer', 'tag, time', {
|
276
|
-
'@type' => 'memory',
|
277
|
-
'timekey' => 3600
|
278
|
-
}, [])
|
279
|
-
])
|
280
|
-
)
|
281
|
-
|
282
|
-
time = event_time
|
283
|
-
d.run(default_tag: fluentd_tag) do
|
284
|
-
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
285
|
-
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
297
|
+
events = get_log_events(log_group_name, 'fluent-plugin-cloudwatch-test')
|
298
|
+
assert_equal(2, events.size)
|
299
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
300
|
+
assert_equal('message1 logs1', events[0].message)
|
301
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
302
|
+
assert_equal('message2 logs2', events[1].message)
|
286
303
|
end
|
287
304
|
|
288
|
-
|
305
|
+
def test_write_use_time_placeholders
|
306
|
+
new_log_stream
|
307
|
+
|
308
|
+
config = {'@type' => 'cloudwatch_logs',
|
309
|
+
'auto_create_stream' => true,
|
310
|
+
'message_keys' => ["message","cloudwatch"],
|
311
|
+
'log_stream_name' => "fluent-plugin-cloudwatch-test-%Y%m%d",
|
312
|
+
'log_group_name' => log_group_name}
|
313
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
314
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
315
|
+
config.merge!(config_elementify(region)) if region
|
316
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
317
|
+
|
318
|
+
d = create_driver(
|
319
|
+
Fluent::Config::Element.new('ROOT', '', config,[
|
320
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
321
|
+
'@type' => 'memory',
|
322
|
+
'timekey' => 3600
|
323
|
+
}, [])
|
324
|
+
])
|
325
|
+
)
|
326
|
+
|
327
|
+
time = event_time
|
328
|
+
d.run(default_tag: fluentd_tag) do
|
329
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
330
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
331
|
+
end
|
289
332
|
|
290
|
-
|
291
|
-
assert_equal(2, events.size)
|
292
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
293
|
-
assert_equal('message1 logs1', events[0].message)
|
294
|
-
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
295
|
-
assert_equal('message2 logs2', events[1].message)
|
296
|
-
end
|
333
|
+
sleep 10
|
297
334
|
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
'log_stream_name' => "fluent-plugin-cloudwatch-test-%Y%m%d",
|
305
|
-
'log_group_name' => log_group_name}
|
306
|
-
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
307
|
-
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
308
|
-
config.merge!(config_elementify(region)) if region
|
309
|
-
config.merge!(config_elementify(endpoint)) if endpoint
|
310
|
-
|
311
|
-
d = create_driver(
|
312
|
-
Fluent::Config::Element.new('ROOT', '', config,[
|
313
|
-
Fluent::Config::Element.new('buffer', 'tag, time', {
|
314
|
-
'@type' => 'memory',
|
315
|
-
'timekey' => 3600
|
316
|
-
}, [])
|
317
|
-
])
|
318
|
-
)
|
319
|
-
|
320
|
-
time = event_time
|
321
|
-
d.run(default_tag: fluentd_tag) do
|
322
|
-
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
|
323
|
-
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
|
335
|
+
events = get_log_events(log_group_name, "fluent-plugin-cloudwatch-test-#{Time.at(time).strftime("%Y%m%d")}")
|
336
|
+
assert_equal(2, events.size)
|
337
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
338
|
+
assert_equal('message1 logs1', events[0].message)
|
339
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
340
|
+
assert_equal('message2 logs2', events[1].message)
|
324
341
|
end
|
325
342
|
|
326
|
-
|
343
|
+
def test_include_time_key
|
344
|
+
new_log_stream
|
327
345
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
346
|
+
d = create_driver(<<-EOC)
|
347
|
+
#{default_config}
|
348
|
+
include_time_key true
|
349
|
+
log_group_name #{log_group_name}
|
350
|
+
log_stream_name #{log_stream_name}
|
351
|
+
utc
|
352
|
+
EOC
|
335
353
|
|
336
|
-
|
337
|
-
|
354
|
+
time = event_time
|
355
|
+
d.run(default_tag: fluentd_tag) do
|
356
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
357
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
358
|
+
end
|
338
359
|
|
339
|
-
|
340
|
-
#{default_config}
|
341
|
-
include_time_key true
|
342
|
-
log_group_name #{log_group_name}
|
343
|
-
log_stream_name #{log_stream_name}
|
344
|
-
utc
|
345
|
-
EOC
|
360
|
+
sleep 10
|
346
361
|
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
362
|
+
events = get_log_events
|
363
|
+
assert_equal(2, events.size)
|
364
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
365
|
+
assert_equal("{\"cloudwatch\":\"logs1\",\"time\":\"#{Time.at(time.to_r).utc.strftime("%Y-%m-%dT%H:%M:%SZ")}\"}", events[0].message)
|
366
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
367
|
+
assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{Time.at((time+1).to_r).utc.strftime("%Y-%m-%dT%H:%M:%SZ")}\"}", events[1].message)
|
351
368
|
end
|
352
369
|
|
353
|
-
|
370
|
+
def test_include_time_key_localtime
|
371
|
+
new_log_stream
|
354
372
|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
373
|
+
d = create_driver(<<-EOC)
|
374
|
+
#{default_config}
|
375
|
+
include_time_key true
|
376
|
+
localtime true
|
377
|
+
log_group_name #{log_group_name}
|
378
|
+
log_stream_name #{log_stream_name}
|
379
|
+
EOC
|
362
380
|
|
363
|
-
|
364
|
-
|
381
|
+
time = event_time
|
382
|
+
d.run(default_tag: fluentd_tag) do
|
383
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
384
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
385
|
+
end
|
365
386
|
|
366
|
-
|
367
|
-
#{default_config}
|
368
|
-
include_time_key true
|
369
|
-
localtime true
|
370
|
-
log_group_name #{log_group_name}
|
371
|
-
log_stream_name #{log_stream_name}
|
372
|
-
EOC
|
387
|
+
sleep 10
|
373
388
|
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
389
|
+
events = get_log_events
|
390
|
+
assert_equal(2, events.size)
|
391
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
392
|
+
assert_equal("{\"cloudwatch\":\"logs1\",\"time\":\"#{Time.at(time.to_r).strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[0].message)
|
393
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
394
|
+
assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{Time.at((time+1).to_r).to_time.strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[1].message)
|
378
395
|
end
|
379
396
|
|
380
|
-
|
381
|
-
|
382
|
-
events = get_log_events
|
383
|
-
assert_equal(2, events.size)
|
384
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
385
|
-
assert_equal("{\"cloudwatch\":\"logs1\",\"time\":\"#{Time.at(time.to_r).strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[0].message)
|
386
|
-
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
387
|
-
assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{Time.at((time+1).to_r).to_time.strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[1].message)
|
388
|
-
end
|
389
|
-
|
390
|
-
def test_log_group_name_key_and_log_stream_name_key
|
391
|
-
new_log_stream
|
397
|
+
def test_log_group_name_key_and_log_stream_name_key
|
398
|
+
new_log_stream
|
392
399
|
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
400
|
+
d = create_driver(<<-EOC)
|
401
|
+
#{default_config}
|
402
|
+
log_group_name_key group_name_key
|
403
|
+
log_stream_name_key stream_name_key
|
404
|
+
@log_level debug
|
405
|
+
EOC
|
399
406
|
|
400
|
-
|
401
|
-
|
407
|
+
stream1 = new_log_stream
|
408
|
+
stream2 = new_log_stream
|
402
409
|
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
410
|
+
records = [
|
411
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => stream1},
|
412
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => stream2},
|
413
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => stream1},
|
414
|
+
]
|
408
415
|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
416
|
+
time = event_time
|
417
|
+
d.run(default_tag: fluentd_tag) do
|
418
|
+
records.each_with_index do |record, i|
|
419
|
+
d.feed(time + i, record)
|
420
|
+
end
|
413
421
|
end
|
414
|
-
end
|
415
422
|
|
416
|
-
|
417
|
-
|
418
|
-
|
423
|
+
logs = d.logs
|
424
|
+
# Call API once for each stream
|
425
|
+
assert_equal(2, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
419
426
|
|
420
|
-
|
427
|
+
sleep 10
|
421
428
|
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
429
|
+
events = get_log_events(log_group_name, stream1)
|
430
|
+
assert_equal(2, events.size)
|
431
|
+
assert_equal(time.to_i * 1000, events[0].timestamp)
|
432
|
+
assert_equal((time.to_i + 2) * 1000, events[1].timestamp)
|
433
|
+
assert_equal(records[0], JSON.parse(events[0].message))
|
434
|
+
assert_equal(records[2], JSON.parse(events[1].message))
|
428
435
|
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
436
|
+
events = get_log_events(log_group_name, stream2)
|
437
|
+
assert_equal(1, events.size)
|
438
|
+
assert_equal((time.to_i + 1) * 1000, events[0].timestamp)
|
439
|
+
assert_equal(records[1], JSON.parse(events[0].message))
|
440
|
+
end
|
434
441
|
|
435
|
-
|
436
|
-
|
442
|
+
def test_remove_log_group_name_key_and_remove_log_stream_name_key
|
443
|
+
new_log_stream
|
437
444
|
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
time = event_time
|
447
|
-
d.run(default_tag: fluentd_tag) do
|
448
|
-
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name})
|
449
|
-
end
|
445
|
+
d = create_driver(<<-EOC)
|
446
|
+
#{default_config}
|
447
|
+
log_group_name_key group_name_key
|
448
|
+
log_stream_name_key stream_name_key
|
449
|
+
remove_log_group_name_key true
|
450
|
+
remove_log_stream_name_key true
|
451
|
+
EOC
|
450
452
|
|
451
|
-
|
453
|
+
time = event_time
|
454
|
+
d.run(default_tag: fluentd_tag) do
|
455
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name})
|
456
|
+
end
|
452
457
|
|
453
|
-
|
454
|
-
assert_equal(1, events.size)
|
455
|
-
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
456
|
-
assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
|
457
|
-
end
|
458
|
+
sleep 10
|
458
459
|
|
459
|
-
|
460
|
-
|
460
|
+
events = get_log_events(log_group_name, log_stream_name)
|
461
|
+
assert_equal(1, events.size)
|
462
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
463
|
+
assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
|
464
|
+
end
|
461
465
|
|
462
|
-
|
463
|
-
|
464
|
-
auto_create_stream true
|
465
|
-
use_tag_as_stream true
|
466
|
-
log_group_name_key group_name_key
|
467
|
-
log_group_aws_tags {"tag1": "value1", "tag2": "value2"}
|
468
|
-
EOC
|
466
|
+
def test_log_group_aws_tags
|
467
|
+
clear_log_group
|
469
468
|
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
469
|
+
d = create_driver(<<-EOC)
|
470
|
+
#{default_config}
|
471
|
+
auto_create_stream true
|
472
|
+
use_tag_as_stream true
|
473
|
+
log_group_name_key group_name_key
|
474
|
+
log_group_aws_tags {"tag1": "value1", "tag2": "value2"}
|
475
|
+
EOC
|
475
476
|
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
477
|
+
records = [
|
478
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
|
479
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
|
480
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
|
481
|
+
]
|
482
|
+
|
483
|
+
time = Time.now
|
484
|
+
d.run(default_tag: fluentd_tag) do
|
485
|
+
records.each_with_index do |record, i|
|
486
|
+
d.feed(time.to_i + i, record)
|
487
|
+
end
|
480
488
|
end
|
481
|
-
end
|
482
|
-
|
483
|
-
awstags = get_log_group_tags
|
484
|
-
assert_equal("value1", awstags.fetch("tag1"))
|
485
|
-
assert_equal("value2", awstags.fetch("tag2"))
|
486
|
-
end
|
487
489
|
|
488
|
-
|
489
|
-
|
490
|
+
awstags = get_log_group_tags
|
491
|
+
assert_equal("value1", awstags.fetch("tag1"))
|
492
|
+
assert_equal("value2", awstags.fetch("tag2"))
|
493
|
+
end
|
490
494
|
|
491
|
-
|
492
|
-
|
493
|
-
auto_create_stream true
|
494
|
-
use_tag_as_stream true
|
495
|
-
log_group_name_key group_name_key
|
496
|
-
retention_in_days 7
|
497
|
-
EOC
|
495
|
+
def test_retention_in_days
|
496
|
+
clear_log_group
|
498
497
|
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
498
|
+
d = create_driver(<<-EOC)
|
499
|
+
#{default_config}
|
500
|
+
auto_create_stream true
|
501
|
+
use_tag_as_stream true
|
502
|
+
log_group_name_key group_name_key
|
503
|
+
retention_in_days 7
|
504
|
+
EOC
|
504
505
|
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
506
|
+
records = [
|
507
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
|
508
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
|
509
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
|
510
|
+
]
|
511
|
+
|
512
|
+
time = Time.now
|
513
|
+
d.run(default_tag: fluentd_tag) do
|
514
|
+
records.each_with_index do |record, i|
|
515
|
+
d.feed(time.to_i + i, record)
|
516
|
+
end
|
509
517
|
end
|
510
|
-
end
|
511
518
|
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
def test_invalid_retention_in_days
|
517
|
-
clear_log_group
|
519
|
+
retention = get_log_group_retention_days
|
520
|
+
assert_equal(d.instance.retention_in_days, retention)
|
521
|
+
end
|
518
522
|
|
519
|
-
|
520
|
-
|
521
|
-
auto_create_stream true
|
522
|
-
use_tag_as_stream true
|
523
|
-
log_group_name_key group_name_key
|
524
|
-
retention_in_days 4
|
525
|
-
EOC
|
523
|
+
def test_invalid_retention_in_days
|
524
|
+
clear_log_group
|
526
525
|
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
526
|
+
d = create_driver(<<-EOC)
|
527
|
+
#{default_config}
|
528
|
+
auto_create_stream true
|
529
|
+
use_tag_as_stream true
|
530
|
+
log_group_name_key group_name_key
|
531
|
+
retention_in_days 4
|
532
|
+
EOC
|
532
533
|
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
534
|
+
records = [
|
535
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
|
536
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
|
537
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
|
538
|
+
]
|
539
|
+
|
540
|
+
time = Time.now
|
541
|
+
d.run(default_tag: fluentd_tag) do
|
542
|
+
records.each_with_index do |record, i|
|
543
|
+
d.feed(time.to_i + i, record)
|
544
|
+
end
|
537
545
|
end
|
538
|
-
end
|
539
|
-
|
540
|
-
assert_match(/failed to set retention policy for Log group/, d.logs[0])
|
541
|
-
end
|
542
546
|
|
543
|
-
|
544
|
-
|
547
|
+
assert_match(/failed to set retention policy for Log group/, d.logs[0])
|
548
|
+
end
|
545
549
|
|
546
|
-
|
547
|
-
|
548
|
-
auto_create_stream true
|
549
|
-
use_tag_as_stream true
|
550
|
-
log_group_name_key group_name_key
|
551
|
-
log_group_aws_tags_key aws_tags
|
552
|
-
EOC
|
550
|
+
def test_log_group_aws_tags_key
|
551
|
+
clear_log_group
|
553
552
|
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
553
|
+
d = create_driver(<<-EOC)
|
554
|
+
#{default_config}
|
555
|
+
auto_create_stream true
|
556
|
+
use_tag_as_stream true
|
557
|
+
log_group_name_key group_name_key
|
558
|
+
log_group_aws_tags_key aws_tags
|
559
|
+
EOC
|
559
560
|
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
561
|
+
records = [
|
562
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
|
563
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
|
564
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}}
|
565
|
+
]
|
566
|
+
|
567
|
+
time = Time.now
|
568
|
+
d.run(default_tag: fluentd_tag) do
|
569
|
+
records.each_with_index do |record, i|
|
570
|
+
d.feed(time.to_i + i, record)
|
571
|
+
end
|
564
572
|
end
|
565
|
-
end
|
566
573
|
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
574
|
+
awstags = get_log_group_tags
|
575
|
+
assert_equal("value1", awstags.fetch("tag1"))
|
576
|
+
assert_equal("value2", awstags.fetch("tag2"))
|
577
|
+
end
|
571
578
|
|
572
|
-
|
573
|
-
|
579
|
+
def test_log_group_aws_tags_key_same_group_diff_tags
|
580
|
+
clear_log_group
|
574
581
|
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
+
d = create_driver(<<-EOC)
|
583
|
+
#{default_config}
|
584
|
+
auto_create_stream true
|
585
|
+
use_tag_as_stream true
|
586
|
+
log_group_name_key group_name_key
|
587
|
+
log_group_aws_tags_key aws_tags
|
588
|
+
EOC
|
582
589
|
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
590
|
+
records = [
|
591
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
|
592
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag3" => "value3", "tag4" => "value4"}}
|
593
|
+
]
|
587
594
|
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
595
|
+
time = Time.now
|
596
|
+
d.run(default_tag: fluentd_tag) do
|
597
|
+
records.each_with_index do |record, i|
|
598
|
+
d.feed(time.to_i + i, record)
|
599
|
+
end
|
592
600
|
end
|
593
|
-
end
|
594
601
|
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
602
|
+
awstags = get_log_group_tags
|
603
|
+
assert_equal("value1", awstags.fetch("tag1"))
|
604
|
+
assert_equal("value2", awstags.fetch("tag2"))
|
605
|
+
assert_raise KeyError do
|
606
|
+
awstags.fetch("tag3")
|
607
|
+
end
|
608
|
+
assert_raise KeyError do
|
609
|
+
awstags.fetch("tag4")
|
610
|
+
end
|
603
611
|
end
|
604
|
-
end
|
605
|
-
|
606
|
-
def test_log_group_aws_tags_key_no_tags
|
607
|
-
clear_log_group
|
608
|
-
|
609
|
-
d = create_driver(<<-EOC)
|
610
|
-
#{default_config}
|
611
|
-
auto_create_stream true
|
612
|
-
log_group_name_key group_name_key
|
613
|
-
log_stream_name_key stream_name_key
|
614
|
-
remove_log_group_name_key true
|
615
|
-
remove_log_stream_name_key true
|
616
|
-
log_group_aws_tags_key aws_tags
|
617
|
-
EOC
|
618
612
|
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
613
|
+
def test_log_group_aws_tags_key_no_tags
|
614
|
+
clear_log_group
|
615
|
+
|
616
|
+
d = create_driver(<<-EOC)
|
617
|
+
#{default_config}
|
618
|
+
auto_create_stream true
|
619
|
+
log_group_name_key group_name_key
|
620
|
+
log_stream_name_key stream_name_key
|
621
|
+
remove_log_group_name_key true
|
622
|
+
remove_log_stream_name_key true
|
623
|
+
log_group_aws_tags_key aws_tags
|
624
|
+
EOC
|
624
625
|
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
626
|
+
stream = log_stream_name
|
627
|
+
records = [
|
628
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => stream},
|
629
|
+
{'cloudwatch' => 'logs2', 'message' => 'message2', 'group_name_key' => log_group_name, 'stream_name_key' => stream}
|
630
|
+
]
|
631
|
+
|
632
|
+
time = Time.now
|
633
|
+
d.run(default_tag: fluentd_tag) do
|
634
|
+
records.each_with_index do |record, i|
|
635
|
+
d.feed(time.to_i + i, record)
|
636
|
+
end
|
629
637
|
end
|
630
|
-
end
|
631
638
|
|
632
|
-
|
639
|
+
sleep 10
|
633
640
|
|
634
|
-
|
641
|
+
awstags = get_log_group_tags
|
635
642
|
|
636
|
-
|
637
|
-
|
638
|
-
|
643
|
+
assert_raise KeyError do
|
644
|
+
awstags.fetch("tag1")
|
645
|
+
end
|
639
646
|
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
647
|
+
events = get_log_events(log_group_name, stream)
|
648
|
+
assert_equal(2, events.size)
|
649
|
+
assert_equal(time.to_i * 1000, events[0].timestamp)
|
650
|
+
assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
|
651
|
+
assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
|
652
|
+
end
|
646
653
|
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
+
def test_retrying_on_throttling_exception
|
655
|
+
resp = mock()
|
656
|
+
resp.expects(:rejected_log_events_info)
|
657
|
+
resp.expects(:next_sequence_token)
|
658
|
+
client = Aws::CloudWatchLogs::Client.new
|
659
|
+
client.stubs(:put_log_events).
|
660
|
+
raises(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")).then.returns(resp)
|
661
|
+
|
662
|
+
d = create_driver
|
663
|
+
time = event_time
|
664
|
+
d.instance.instance_variable_set(:@logs, client)
|
665
|
+
d.run(default_tag: fluentd_tag) do
|
666
|
+
d.feed(time, {'message' => 'message1'})
|
667
|
+
end
|
654
668
|
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
d.feed(time, {'message' => 'message1'})
|
669
|
+
logs = d.logs
|
670
|
+
assert_equal(1, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
671
|
+
assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
|
672
|
+
assert_equal(1, logs.select {|l| l =~ /retry succeeded/ }.size)
|
660
673
|
end
|
661
674
|
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
675
|
+
def test_retrying_on_throttling_exception_and_throw_away
|
676
|
+
client = Aws::CloudWatchLogs::Client.new
|
677
|
+
client.stubs(:put_log_events).
|
678
|
+
raises(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
|
679
|
+
time = Fluent::Engine.now
|
680
|
+
d = create_driver(<<-EOC)
|
681
|
+
#{default_config}
|
682
|
+
log_group_name #{log_group_name}
|
683
|
+
log_stream_name #{log_stream_name}
|
684
|
+
put_log_events_retry_limit 1
|
685
|
+
@log_level debug
|
686
|
+
EOC
|
687
|
+
d.instance.instance_variable_set(:@logs, client)
|
688
|
+
d.run(default_tag: fluentd_tag) do
|
689
|
+
d.feed(time, {'message' => 'message1'})
|
690
|
+
end
|
667
691
|
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
time = Fluent::Engine.now
|
673
|
-
d = create_driver(<<-EOC)
|
674
|
-
#{default_config}
|
675
|
-
log_group_name #{log_group_name}
|
676
|
-
log_stream_name #{log_stream_name}
|
677
|
-
put_log_events_retry_limit 1
|
678
|
-
@log_level debug
|
679
|
-
EOC
|
680
|
-
d.instance.instance_variable_set(:@logs, client)
|
681
|
-
d.run(default_tag: fluentd_tag) do
|
682
|
-
d.feed(time, {'message' => 'message1'})
|
692
|
+
logs = d.logs
|
693
|
+
assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
694
|
+
assert_equal(3, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
|
695
|
+
assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents and discard logs/ }.size)
|
683
696
|
end
|
684
697
|
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
698
|
+
def test_too_large_event
|
699
|
+
time = Fluent::Engine.now
|
700
|
+
d = create_driver(<<-EOC)
|
701
|
+
#{default_config}
|
702
|
+
log_group_name #{log_group_name}
|
703
|
+
log_stream_name #{log_stream_name}
|
704
|
+
@log_level debug
|
705
|
+
EOC
|
706
|
+
d.run(default_tag: fluentd_tag) do
|
707
|
+
d.feed(time, {'message' => '*' * 256 * 1024})
|
708
|
+
end
|
690
709
|
|
691
|
-
|
692
|
-
|
693
|
-
d = create_driver(<<-EOC)
|
694
|
-
#{default_config}
|
695
|
-
log_group_name #{log_group_name}
|
696
|
-
log_stream_name #{log_stream_name}
|
697
|
-
@log_level debug
|
698
|
-
EOC
|
699
|
-
d.run(default_tag: fluentd_tag) do
|
700
|
-
d.feed(time, {'message' => '*' * 256 * 1024})
|
710
|
+
logs = d.logs
|
711
|
+
assert(logs.any?{|log| log.include?("Log event is discarded because it is too large: 262184 bytes exceeds limit of 262144")})
|
701
712
|
end
|
702
|
-
|
703
|
-
logs = d.logs
|
704
|
-
assert(logs.any?{|log| log.include?("Log event is discarded because it is too large: 262184 bytes exceeds limit of 262144")})
|
705
713
|
end
|
706
714
|
|
707
715
|
def test_scrub_record
|