fluent-plugin-cloudwatch-logs 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +21 -6
- data/test/plugin/test_out_cloudwatch_logs.rb +137 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff7c8514095015603f6197f312b3e73439485ad9fd8b296522605919f5c53930
|
4
|
+
data.tar.gz: b49f0d62c6033f22409c1046ebd87d884a1f693725e4f372119487225c7b0d5f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a3520d9d89be225d7fce95a4cc5acc0064b6fdf316de8c85b0a0dda8e7bcc95c3a8290e106459312e9c14aaf2639300aed693169ac200f56c795b04b38d91f6
|
7
|
+
data.tar.gz: 2f3ab8a2a8eb9e98d013f184bdb9cfc7cc2607ba88a1bce37ed8ea56552d1211c1b1b565bebe7aa00791d2c5579d3348507ce612e1070d7bb8abe9dfa3b509ad
|
data/README.md
CHANGED
@@ -165,6 +165,9 @@ Fetch sample log from CloudWatch Logs:
|
|
165
165
|
# role_session_name ROLE_SESSION_NAME
|
166
166
|
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
167
167
|
#</web_identity_credentials>
|
168
|
+
#<format>
|
169
|
+
# @type ltsv
|
170
|
+
#</format>
|
168
171
|
</match>
|
169
172
|
```
|
170
173
|
|
@@ -207,6 +210,7 @@ Fetch sample log from CloudWatch Logs:
|
|
207
210
|
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
208
211
|
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
209
212
|
is set to 3600 seconds (1 hour). (default `nil`)
|
213
|
+
* `<format>`: For specifying records format. See [formatter overview](https://docs.fluentd.org/formatter) and [formatter section overview](https://docs.fluentd.org/configuration/format-section) on the official documentation.
|
210
214
|
|
211
215
|
**NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
|
212
216
|
Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
|
@@ -9,7 +9,7 @@ module Fluent::Plugin
|
|
9
9
|
|
10
10
|
class TooLargeEventError < Fluent::UnrecoverableError; end
|
11
11
|
|
12
|
-
helpers :compat_parameters, :inject
|
12
|
+
helpers :compat_parameters, :inject, :formatter
|
13
13
|
|
14
14
|
DEFAULT_BUFFER_TYPE = "memory"
|
15
15
|
|
@@ -58,6 +58,9 @@ module Fluent::Plugin
|
|
58
58
|
config_section :buffer do
|
59
59
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
60
60
|
end
|
61
|
+
config_section :format do
|
62
|
+
config_set_default :@type, 'json'
|
63
|
+
end
|
61
64
|
|
62
65
|
MAX_EVENTS_SIZE = 1_048_576
|
63
66
|
MAX_EVENT_SIZE = 256 * 1024
|
@@ -88,6 +91,22 @@ module Fluent::Plugin
|
|
88
91
|
if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
|
89
92
|
raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
|
90
93
|
end
|
94
|
+
|
95
|
+
formatter_conf = conf.elements('format').first
|
96
|
+
@formatter_proc = unless formatter_conf
|
97
|
+
unless @message_keys.empty?
|
98
|
+
Proc.new { |tag, time, record|
|
99
|
+
@message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
|
100
|
+
}
|
101
|
+
else
|
102
|
+
Proc.new { |tag, time, record|
|
103
|
+
@json_handler.dump(record)
|
104
|
+
}
|
105
|
+
end
|
106
|
+
else
|
107
|
+
formatter = formatter_create(usage: 'cloudwatch-logs-plugin', conf: formatter_conf)
|
108
|
+
formatter.method(:format)
|
109
|
+
end
|
91
110
|
end
|
92
111
|
|
93
112
|
def start
|
@@ -264,11 +283,7 @@ module Fluent::Plugin
|
|
264
283
|
time_ms = (time.to_f * 1000).floor
|
265
284
|
|
266
285
|
scrub_record!(record)
|
267
|
-
|
268
|
-
message = @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
|
269
|
-
else
|
270
|
-
message = @json_handler.dump(record)
|
271
|
-
end
|
286
|
+
message = @formatter_proc.call(t, time, record)
|
272
287
|
|
273
288
|
if message.empty?
|
274
289
|
log.warn "Within specified message_key(s): (#{@message_keys.join(',')}) do not have non-empty record. Skip."
|
@@ -76,6 +76,143 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
76
76
|
assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
|
77
77
|
end
|
78
78
|
|
79
|
+
sub_test_case "formatter" do
|
80
|
+
test "csv" do
|
81
|
+
new_log_stream
|
82
|
+
|
83
|
+
config = {'@type' => 'cloudwatch_logs',
|
84
|
+
'auto_create_stream' => true,
|
85
|
+
'log_stream_name' => log_stream_name,
|
86
|
+
'log_group_name' => log_group_name,
|
87
|
+
'@log_level' => 'debug'}
|
88
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
89
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
90
|
+
config.merge!(config_elementify(region)) if region
|
91
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
92
|
+
|
93
|
+
d = create_driver(
|
94
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
95
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
96
|
+
'@type' => 'memory',
|
97
|
+
'timekey' => 3600
|
98
|
+
}, []),
|
99
|
+
Fluent::Config::Element.new('format', '', {
|
100
|
+
'@type' => 'csv',
|
101
|
+
'fields' => ["message","cloudwatch"],
|
102
|
+
}, []),
|
103
|
+
]))
|
104
|
+
|
105
|
+
time = event_time
|
106
|
+
d.run(default_tag: fluentd_tag, flush: true) do
|
107
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
108
|
+
# Addition converts EventTime to seconds
|
109
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
110
|
+
end
|
111
|
+
|
112
|
+
sleep 10
|
113
|
+
|
114
|
+
logs = d.logs
|
115
|
+
events = get_log_events
|
116
|
+
assert_equal(2, events.size)
|
117
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
118
|
+
assert_equal('"","logs1"', events[0].message.strip)
|
119
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
120
|
+
assert_equal('"","logs2"', events[1].message.strip)
|
121
|
+
|
122
|
+
assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
|
123
|
+
end
|
124
|
+
|
125
|
+
test "ltsv" do
|
126
|
+
new_log_stream
|
127
|
+
|
128
|
+
config = {'@type' => 'cloudwatch_logs',
|
129
|
+
'auto_create_stream' => true,
|
130
|
+
'log_stream_name' => log_stream_name,
|
131
|
+
'log_group_name' => log_group_name,
|
132
|
+
'@log_level' => 'debug'}
|
133
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
134
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
135
|
+
config.merge!(config_elementify(region)) if region
|
136
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
137
|
+
|
138
|
+
d = create_driver(
|
139
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
140
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
141
|
+
'@type' => 'memory',
|
142
|
+
'timekey' => 3600
|
143
|
+
}, []),
|
144
|
+
Fluent::Config::Element.new('format', '', {
|
145
|
+
'@type' => 'ltsv',
|
146
|
+
'fields' => ["message","cloudwatch"],
|
147
|
+
}, []),
|
148
|
+
]))
|
149
|
+
|
150
|
+
time = event_time
|
151
|
+
d.run(default_tag: fluentd_tag, flush: true) do
|
152
|
+
d.feed(time, {'cloudwatch' => 'logs1'})
|
153
|
+
# Addition converts EventTime to seconds
|
154
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2'})
|
155
|
+
end
|
156
|
+
|
157
|
+
sleep 10
|
158
|
+
|
159
|
+
logs = d.logs
|
160
|
+
events = get_log_events
|
161
|
+
assert_equal(2, events.size)
|
162
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
163
|
+
assert_equal('cloudwatch:logs1', events[0].message.strip)
|
164
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
165
|
+
assert_equal('cloudwatch:logs2', events[1].message.strip)
|
166
|
+
|
167
|
+
assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
|
168
|
+
end
|
169
|
+
|
170
|
+
test "single_value" do
|
171
|
+
new_log_stream
|
172
|
+
|
173
|
+
config = {'@type' => 'cloudwatch_logs',
|
174
|
+
'auto_create_stream' => true,
|
175
|
+
'log_stream_name' => log_stream_name,
|
176
|
+
'log_group_name' => log_group_name,
|
177
|
+
'@log_level' => 'debug'}
|
178
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
179
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
180
|
+
config.merge!(config_elementify(region)) if region
|
181
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
182
|
+
|
183
|
+
d = create_driver(
|
184
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
185
|
+
Fluent::Config::Element.new('buffer', 'tag, time', {
|
186
|
+
'@type' => 'memory',
|
187
|
+
'timekey' => 3600
|
188
|
+
}, []),
|
189
|
+
Fluent::Config::Element.new('format', '', {
|
190
|
+
'@type' => 'single_value',
|
191
|
+
'message_key' => "cloudwatch",
|
192
|
+
}, []),
|
193
|
+
]))
|
194
|
+
|
195
|
+
time = event_time
|
196
|
+
d.run(default_tag: fluentd_tag, flush: true) do
|
197
|
+
d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'Hi!'})
|
198
|
+
# Addition converts EventTime to seconds
|
199
|
+
d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'Hi!'})
|
200
|
+
end
|
201
|
+
|
202
|
+
sleep 10
|
203
|
+
|
204
|
+
logs = d.logs
|
205
|
+
events = get_log_events
|
206
|
+
assert_equal(2, events.size)
|
207
|
+
assert_equal((time.to_f * 1000).floor, events[0].timestamp)
|
208
|
+
assert_equal('logs1', events[0].message.strip)
|
209
|
+
assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
|
210
|
+
assert_equal('logs2', events[1].message.strip)
|
211
|
+
|
212
|
+
assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
79
216
|
def test_write_utf8
|
80
217
|
new_log_stream
|
81
218
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-11-
|
11
|
+
date: 2020-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|