fluent-plugin-cloudwatch-logs 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +8 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +47 -6
- data/test/plugin/test_out_cloudwatch_logs.rb +78 -17
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f11742cedd1eb409768c54c02aca68f71bf7eaaf
|
4
|
+
data.tar.gz: 49ce80a5464dc1dd99c847e6f39f914b314d83a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b43fdfa3ee97e844180a4b775e238d7c9cac3eaa786d9ed26d897956f5d0b1a3071c5f3e65ad89b99e120a5c464a5c7114c0ef8376df563815b4d2e9a1f1b604
|
7
|
+
data.tar.gz: 862017735509e6e2340ecd135432d85d5edf4d5e8143ed0ef3cc9a3d08386a657ddb153e04333178838402844b3b5044010f5480ddfcb76f7137dbc6ea2919c4
|
data/README.md
CHANGED
@@ -75,6 +75,10 @@ Fetch sample log from CloudWatch Logs:
|
|
75
75
|
#use_tag_as_stream false
|
76
76
|
#include_time_key true
|
77
77
|
#localtime true
|
78
|
+
#log_group_name_key group_name_key
|
79
|
+
#log_stream_name_key stream_name_key
|
80
|
+
#remove_log_group_name_key true
|
81
|
+
#remove_log_stream_name_key true
|
78
82
|
</match>
|
79
83
|
```
|
80
84
|
|
@@ -88,6 +92,10 @@ Fetch sample log from CloudWatch Logs:
|
|
88
92
|
* `use_tag_as_stream`: to use tag as a stream name
|
89
93
|
* `include_time_key`: include time key as part of the log entry (defaults to UTC)
|
90
94
|
* `localtime`: use localtime timezone for `include_time_key` output (overrides UTC default)
|
95
|
+
* `log_group_name_key`: use specified field of records as log group name
|
96
|
+
* `log_stream_name_key`: use specified field of records as log stream name
|
97
|
+
* `remove_log_group_name_key`: remove field specified by `log_group_name_key`
|
98
|
+
* `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
|
91
99
|
|
92
100
|
### in_cloudwatch_logs
|
93
101
|
|
@@ -13,8 +13,12 @@ module Fluent
|
|
13
13
|
config_param :message_keys, :string, :default => nil
|
14
14
|
config_param :max_message_length, :integer, :default => nil
|
15
15
|
config_param :max_events_per_batch, :integer, :default => 10000
|
16
|
-
config_param :use_tag_as_group, :bool, :default => false
|
17
|
-
config_param :use_tag_as_stream, :bool, :default => false
|
16
|
+
config_param :use_tag_as_group, :bool, :default => false # TODO: Rename to use_tag_as_group_name ?
|
17
|
+
config_param :use_tag_as_stream, :bool, :default => false # TODO: Rename to use_tag_as_stream_name ?
|
18
|
+
config_param :log_group_name_key, :string, :default => nil
|
19
|
+
config_param :log_stream_name_key, :string, :default => nil
|
20
|
+
config_param :remove_log_group_name_key, :bool, :default => false
|
21
|
+
config_param :remove_log_stream_name_key, :bool, :default => false
|
18
22
|
config_param :http_proxy, :string, default: nil
|
19
23
|
|
20
24
|
MAX_EVENTS_SIZE = 1_048_576
|
@@ -29,6 +33,18 @@ module Fluent
|
|
29
33
|
require 'aws-sdk-core'
|
30
34
|
end
|
31
35
|
|
36
|
+
def configure(conf)
|
37
|
+
super
|
38
|
+
|
39
|
+
unless [conf['log_group_name'], conf['use_tag_as_group'], conf['log_group_name_key']].compact.size == 1
|
40
|
+
raise ConfigError, "Set only one of log_group_name, use_tag_as_group and log_group_name_key"
|
41
|
+
end
|
42
|
+
|
43
|
+
unless [conf['log_stream_name'], conf['use_tag_as_stream'], conf['log_stream_name_key']].compact.size == 1
|
44
|
+
raise ConfigError, "Set only one of log_stream_name, use_tag_as_stream and log_stream_name_key"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
32
48
|
def start
|
33
49
|
super
|
34
50
|
|
@@ -47,10 +63,35 @@ module Fluent
|
|
47
63
|
def write(chunk)
|
48
64
|
events = []
|
49
65
|
chunk.enum_for(:msgpack_each).chunk {|tag, time, record|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
66
|
+
group = case
|
67
|
+
when @use_tag_as_group
|
68
|
+
tag
|
69
|
+
when @log_group_name_key
|
70
|
+
if @remove_log_group_name_key
|
71
|
+
record.delete(@log_group_name_key)
|
72
|
+
else
|
73
|
+
record[@log_group_name_key]
|
74
|
+
end
|
75
|
+
else
|
76
|
+
@log_group_name
|
77
|
+
end
|
78
|
+
|
79
|
+
stream = case
|
80
|
+
when @use_tag_as_stream
|
81
|
+
tag
|
82
|
+
when @log_stream_name_key
|
83
|
+
if @remove_log_stream_name_key
|
84
|
+
record.delete(@log_stream_name_key)
|
85
|
+
else
|
86
|
+
record[@log_stream_name_key]
|
87
|
+
end
|
88
|
+
else
|
89
|
+
@log_stream_name
|
90
|
+
end
|
91
|
+
|
92
|
+
[group, stream]
|
93
|
+
}.each {|group_stream, rs|
|
94
|
+
group_name, stream_name = group_stream
|
54
95
|
|
55
96
|
unless log_group_exists?(group_name)
|
56
97
|
if @auto_create_stream
|
@@ -43,7 +43,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
43
43
|
d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
|
44
44
|
d.run
|
45
45
|
|
46
|
-
sleep
|
46
|
+
sleep 10
|
47
47
|
|
48
48
|
events = get_log_events
|
49
49
|
assert_equal(2, events.size)
|
@@ -61,7 +61,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
61
61
|
d.emit({'cloudwatch' => 'これは日本語です'.force_encoding('UTF-8')}, time.to_i)
|
62
62
|
d.run
|
63
63
|
|
64
|
-
sleep
|
64
|
+
sleep 10
|
65
65
|
|
66
66
|
events = get_log_events
|
67
67
|
assert_equal(1, events.size)
|
@@ -79,7 +79,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
79
79
|
d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
|
80
80
|
d.run
|
81
81
|
|
82
|
-
sleep
|
82
|
+
sleep 10
|
83
83
|
|
84
84
|
events = get_log_events
|
85
85
|
assert_equal(3, events.size)
|
@@ -97,6 +97,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
97
97
|
d = create_driver(<<-EOC)
|
98
98
|
#{default_config}
|
99
99
|
message_keys message,cloudwatch
|
100
|
+
log_group_name #{log_group_name}
|
101
|
+
log_stream_name #{log_stream_name}
|
100
102
|
EOC
|
101
103
|
|
102
104
|
time = Time.now
|
@@ -104,7 +106,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
104
106
|
d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
|
105
107
|
d.run
|
106
108
|
|
107
|
-
sleep
|
109
|
+
sleep 10
|
108
110
|
|
109
111
|
events = get_log_events
|
110
112
|
assert_equal(2, events.size)
|
@@ -121,6 +123,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
121
123
|
#{default_config}
|
122
124
|
message_keys message,cloudwatch
|
123
125
|
max_message_length 10
|
126
|
+
log_group_name #{log_group_name}
|
127
|
+
log_stream_name #{log_stream_name}
|
124
128
|
EOC
|
125
129
|
|
126
130
|
time = Time.now
|
@@ -128,7 +132,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
128
132
|
d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
|
129
133
|
d.run
|
130
134
|
|
131
|
-
sleep
|
135
|
+
sleep 10
|
132
136
|
|
133
137
|
events = get_log_events
|
134
138
|
assert_equal(2, events.size)
|
@@ -145,6 +149,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
145
149
|
#{default_config}
|
146
150
|
message_keys message,cloudwatch
|
147
151
|
use_tag_as_group true
|
152
|
+
log_stream_name #{log_stream_name}
|
148
153
|
EOC
|
149
154
|
|
150
155
|
time = Time.now
|
@@ -152,7 +157,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
152
157
|
d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
|
153
158
|
d.run
|
154
159
|
|
155
|
-
sleep
|
160
|
+
sleep 10
|
156
161
|
|
157
162
|
events = get_log_events(fluentd_tag)
|
158
163
|
assert_equal(2, events.size)
|
@@ -169,6 +174,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
169
174
|
#{default_config}
|
170
175
|
message_keys message,cloudwatch
|
171
176
|
use_tag_as_stream true
|
177
|
+
log_group_name #{log_group_name}
|
172
178
|
EOC
|
173
179
|
|
174
180
|
time = Time.now
|
@@ -176,7 +182,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
176
182
|
d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
|
177
183
|
d.run
|
178
184
|
|
179
|
-
sleep
|
185
|
+
sleep 10
|
180
186
|
|
181
187
|
events = get_log_events(log_group_name, fluentd_tag)
|
182
188
|
assert_equal(2, events.size)
|
@@ -192,6 +198,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
192
198
|
d = create_driver(<<-EOC)
|
193
199
|
#{default_config}
|
194
200
|
include_time_key true
|
201
|
+
log_group_name #{log_group_name}
|
202
|
+
log_stream_name #{log_stream_name}
|
195
203
|
EOC
|
196
204
|
|
197
205
|
time = Time.now
|
@@ -199,7 +207,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
199
207
|
d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
|
200
208
|
d.run
|
201
209
|
|
202
|
-
sleep
|
210
|
+
sleep 10
|
203
211
|
|
204
212
|
events = get_log_events
|
205
213
|
assert_equal(2, events.size)
|
@@ -216,6 +224,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
216
224
|
#{default_config}
|
217
225
|
include_time_key true
|
218
226
|
localtime true
|
227
|
+
log_group_name #{log_group_name}
|
228
|
+
log_stream_name #{log_stream_name}
|
219
229
|
EOC
|
220
230
|
|
221
231
|
time = Time.now
|
@@ -223,7 +233,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
223
233
|
d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
|
224
234
|
d.run
|
225
235
|
|
226
|
-
sleep
|
236
|
+
sleep 10
|
227
237
|
|
228
238
|
events = get_log_events
|
229
239
|
assert_equal(2, events.size)
|
@@ -233,20 +243,71 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
233
243
|
assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{(time+1).strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[1].message)
|
234
244
|
end
|
235
245
|
|
246
|
+
def test_log_group_name_key_and_log_stream_name_key
|
247
|
+
new_log_stream
|
248
|
+
|
249
|
+
d = create_driver(<<-EOC)
|
250
|
+
#{default_config}
|
251
|
+
log_group_name_key group_name_key
|
252
|
+
log_stream_name_key stream_name_key
|
253
|
+
EOC
|
254
|
+
|
255
|
+
record = {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name}
|
256
|
+
|
257
|
+
time = Time.now
|
258
|
+
d.emit(record, time.to_i)
|
259
|
+
d.run
|
260
|
+
|
261
|
+
sleep 10
|
262
|
+
|
263
|
+
events = get_log_events(log_group_name, log_stream_name)
|
264
|
+
assert_equal(1, events.size)
|
265
|
+
assert_equal(time.to_i * 1000, events[0].timestamp)
|
266
|
+
assert_equal(record, JSON.parse(events[0].message))
|
267
|
+
end
|
268
|
+
|
269
|
+
def test_remove_log_group_name_key_and_remove_log_stream_name_key
|
270
|
+
new_log_stream
|
271
|
+
|
272
|
+
d = create_driver(<<-EOC)
|
273
|
+
#{default_config}
|
274
|
+
log_group_name_key group_name_key
|
275
|
+
log_stream_name_key stream_name_key
|
276
|
+
remove_log_group_name_key true
|
277
|
+
remove_log_stream_name_key true
|
278
|
+
EOC
|
279
|
+
|
280
|
+
time = Time.now
|
281
|
+
d.emit({'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name}, time.to_i)
|
282
|
+
d.run
|
283
|
+
|
284
|
+
sleep 10
|
285
|
+
|
286
|
+
events = get_log_events(log_group_name, log_stream_name)
|
287
|
+
assert_equal(1, events.size)
|
288
|
+
assert_equal(time.to_i * 1000, events[0].timestamp)
|
289
|
+
assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
|
290
|
+
end
|
291
|
+
|
236
292
|
private
|
237
293
|
def default_config
|
238
294
|
<<-EOC
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
#{aws_sec_key}
|
245
|
-
#{region}
|
295
|
+
type cloudwatch_logs
|
296
|
+
auto_create_stream true
|
297
|
+
#{aws_key_id}
|
298
|
+
#{aws_sec_key}
|
299
|
+
#{region}
|
246
300
|
EOC
|
247
301
|
end
|
248
302
|
|
249
|
-
def create_driver(conf =
|
303
|
+
def create_driver(conf = nil)
|
304
|
+
unless conf
|
305
|
+
conf = <<-EOC
|
306
|
+
#{default_config}
|
307
|
+
log_group_name #{log_group_name}
|
308
|
+
log_stream_name #{log_stream_name}
|
309
|
+
EOC
|
310
|
+
end
|
250
311
|
Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput, fluentd_tag).configure(conf)
|
251
312
|
end
|
252
313
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
version: '0'
|
122
122
|
requirements: []
|
123
123
|
rubyforge_project:
|
124
|
-
rubygems_version: 2.
|
124
|
+
rubygems_version: 2.5.1
|
125
125
|
signing_key:
|
126
126
|
specification_version: 4
|
127
127
|
summary: CloudWatch Logs Plugin for Fluentd
|