fluent-plugin-cloudwatch-logs 0.9.4 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/README.md +72 -1
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +113 -45
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +34 -6
- data/test/plugin/test_in_cloudwatch_logs.rb +110 -7
- data/test/plugin/test_out_cloudwatch_logs.rb +67 -0
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1dc48c250b022126a1de2b125bfa8ad3320daaa5eca5613f51ba7e6571a0b9a9
|
4
|
+
data.tar.gz: 23993ce51cac3aacfbe6937c1f928a00a61fbd94f64fb4ccf8c38ac8e4656787
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 84fd2ea44c0b498364a13da89d422d39b6ea18abdb38add8fbacbc9f0c7b04b6ed18498f26e85920ffe8a7c80e5c14dce8f191c6ecc1a2f1c36809ce67e6961b
|
7
|
+
data.tar.gz: e16ab191ba87408d82e1ffa73564aec909a0795cca65c0ab506b2bb538f4c1cd0ad61641035c6b42056ccd4459e4ba677cb35a28d8af74edc3d1d0bd04422db1
|
@@ -0,0 +1,12 @@
|
|
1
|
+
name: Autocloser
|
2
|
+
on: [issues]
|
3
|
+
jobs:
|
4
|
+
autoclose:
|
5
|
+
runs-on: ubuntu-latest
|
6
|
+
steps:
|
7
|
+
- name: Autoclose issues that did not follow issue template
|
8
|
+
uses: roots/issue-closer-action@v1.1
|
9
|
+
with:
|
10
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
11
|
+
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
12
|
+
issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"
|
data/README.md
CHANGED
@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
|
|
43
43
|
}
|
44
44
|
```
|
45
45
|
|
46
|
+
More restricted IAM policy for `out_cloudwatch_logs` is:
|
47
|
+
|
48
|
+
```json
|
49
|
+
{
|
50
|
+
"Version": "2012-10-17",
|
51
|
+
"Statement": [
|
52
|
+
{
|
53
|
+
"Action": [
|
54
|
+
"logs:PutLogEvents",
|
55
|
+
"logs:CreateLogGroup",
|
56
|
+
"logs:PutRetentionPolicy",
|
57
|
+
"logs:CreateLogStream",
|
58
|
+
"logs:DescribeLogGroups",
|
59
|
+
"logs:DescribeLogStreams"
|
60
|
+
],
|
61
|
+
"Effect": "Allow",
|
62
|
+
"Resource": "*"
|
63
|
+
}
|
64
|
+
]
|
65
|
+
}
|
66
|
+
```
|
67
|
+
|
68
|
+
Also, more restricted IAM policy for `in_cloudwatch_logs` is:
|
69
|
+
|
70
|
+
```json
|
71
|
+
{
|
72
|
+
"Version": "2012-10-17",
|
73
|
+
"Statement": [
|
74
|
+
{
|
75
|
+
"Action": [
|
76
|
+
"logs:GetLogEvents",
|
77
|
+
"logs:DescribeLogStreams"
|
78
|
+
],
|
79
|
+
"Effect": "Allow",
|
80
|
+
"Resource": "*"
|
81
|
+
}
|
82
|
+
]
|
83
|
+
}
|
84
|
+
```
|
85
|
+
|
46
86
|
## Authentication
|
47
87
|
|
48
88
|
There are several methods to provide authentication credentials. Be aware that there are various tradeoffs for these methods,
|
@@ -120,6 +160,11 @@ Fetch sample log from CloudWatch Logs:
|
|
120
160
|
#endpoint http://localhost:5000/
|
121
161
|
#json_handler json
|
122
162
|
#log_rejected_request true
|
163
|
+
#<web_identity_credentials>
|
164
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
165
|
+
# role_session_name ROLE_SESSION_NAME
|
166
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
167
|
+
#</web_identity_credentials>
|
123
168
|
</match>
|
124
169
|
```
|
125
170
|
|
@@ -154,6 +199,14 @@ Fetch sample log from CloudWatch Logs:
|
|
154
199
|
* `retention_in_days_key`: use specified field of records as retention period
|
155
200
|
* `use_tag_as_group`: to use tag as a group name
|
156
201
|
* `use_tag_as_stream`: to use tag as a stream name
|
202
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
203
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
204
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
205
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
206
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
207
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
208
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
209
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
157
210
|
|
158
211
|
**NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
|
159
212
|
Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
|
@@ -178,6 +231,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
178
231
|
#<parse>
|
179
232
|
# @type none # or csv, tsv, regexp etc.
|
180
233
|
#</parse>
|
234
|
+
#<storage>
|
235
|
+
# @type local # or redis, memcached, etc.
|
236
|
+
#</storage>
|
237
|
+
#<web_identity_credentials>
|
238
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
239
|
+
# role_session_name ROLE_SESSION_NAME
|
240
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
241
|
+
#</web_identity_credentials>
|
181
242
|
</source>
|
182
243
|
```
|
183
244
|
|
@@ -194,7 +255,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
194
255
|
* `log_stream_name`: name of log stream to fetch logs
|
195
256
|
* `region`: AWS Region. See [Authentication](#authentication) for more information.
|
196
257
|
* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
|
197
|
-
* `
|
258
|
+
* `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
|
259
|
+
* `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
|
198
260
|
* `tag`: fluentd tag
|
199
261
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
200
262
|
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
@@ -204,6 +266,15 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
204
266
|
* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
|
205
267
|
* `format`: specify CloudWatchLogs' log format. (default `nil`)
|
206
268
|
* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
|
269
|
+
* `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
|
270
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
271
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
272
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
273
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
274
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
275
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
276
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
277
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
207
278
|
|
208
279
|
## Test
|
209
280
|
|
@@ -8,7 +8,9 @@ module Fluent::Plugin
|
|
8
8
|
class CloudwatchLogsInput < Input
|
9
9
|
Fluent::Plugin.register_input('cloudwatch_logs', self)
|
10
10
|
|
11
|
-
helpers :parser, :thread, :compat_parameters
|
11
|
+
helpers :parser, :thread, :compat_parameters, :storage
|
12
|
+
|
13
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
12
14
|
|
13
15
|
config_param :aws_key_id, :string, default: nil, secret: true
|
14
16
|
config_param :aws_sec_key, :string, default: nil, secret: true
|
@@ -21,7 +23,8 @@ module Fluent::Plugin
|
|
21
23
|
config_param :log_group_name, :string
|
22
24
|
config_param :log_stream_name, :string, default: nil
|
23
25
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
24
|
-
config_param :state_file, :string
|
26
|
+
config_param :state_file, :string, default: nil,
|
27
|
+
deprecated: "Use <stroage> instead."
|
25
28
|
config_param :fetch_interval, :time, default: 60
|
26
29
|
config_param :http_proxy, :string, default: nil
|
27
30
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
@@ -31,11 +34,25 @@ module Fluent::Plugin
|
|
31
34
|
config_param :end_time, :string, default: nil
|
32
35
|
config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
|
33
36
|
config_param :throttling_retry_seconds, :time, default: nil
|
37
|
+
config_param :include_metadata, :bool, default: false
|
38
|
+
config_section :web_identity_credentials, multi: false do
|
39
|
+
config_param :role_arn, :string
|
40
|
+
config_param :role_session_name, :string
|
41
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
42
|
+
config_param :policy, :string, default: nil
|
43
|
+
config_param :duration_seconds, :time, default: nil
|
44
|
+
end
|
34
45
|
|
35
46
|
config_section :parse do
|
36
47
|
config_set_default :@type, 'none'
|
37
48
|
end
|
38
49
|
|
50
|
+
config_section :storage do
|
51
|
+
config_set_default :usage, 'store_next_tokens'
|
52
|
+
config_set_default :@type, DEFAULT_STORAGE_TYPE
|
53
|
+
config_set_default :persistent, false
|
54
|
+
end
|
55
|
+
|
39
56
|
def initialize
|
40
57
|
super
|
41
58
|
|
@@ -53,6 +70,7 @@ module Fluent::Plugin
|
|
53
70
|
if @start_time && @end_time && (@end_time < @start_time)
|
54
71
|
raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
|
55
72
|
end
|
73
|
+
@next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
56
74
|
end
|
57
75
|
|
58
76
|
def start
|
@@ -68,6 +86,18 @@ module Fluent::Plugin
|
|
68
86
|
role_arn: @aws_sts_role_arn,
|
69
87
|
role_session_name: @aws_sts_session_name
|
70
88
|
)
|
89
|
+
elsif @web_identity_credentials
|
90
|
+
c = @web_identity_credentials
|
91
|
+
credentials_options = {}
|
92
|
+
credentials_options[:role_arn] = c.role_arn
|
93
|
+
credentials_options[:role_session_name] = c.role_session_name
|
94
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
95
|
+
credentials_options[:policy] = c.policy if c.policy
|
96
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
97
|
+
if @region
|
98
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
99
|
+
end
|
100
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
71
101
|
else
|
72
102
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
73
103
|
end
|
@@ -99,20 +129,28 @@ module Fluent::Plugin
|
|
99
129
|
end
|
100
130
|
end
|
101
131
|
|
102
|
-
def
|
103
|
-
|
104
|
-
|
132
|
+
def state_key_for(log_stream_name)
|
133
|
+
if log_stream_name
|
134
|
+
"#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
|
135
|
+
else
|
136
|
+
@state_file
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def migrate_state_file_to_storage(log_stream_name)
|
141
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
|
142
|
+
File.delete(state_key_for(log_stream_name))
|
105
143
|
end
|
106
144
|
|
107
145
|
def next_token(log_stream_name)
|
108
|
-
|
109
|
-
|
146
|
+
if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
|
147
|
+
migrate_state_file_to_storage(log_stream_name)
|
148
|
+
end
|
149
|
+
@next_token_storage.get(:"#{state_key_for(log_stream_name)}")
|
110
150
|
end
|
111
151
|
|
112
152
|
def store_next_token(token, log_stream_name = nil)
|
113
|
-
|
114
|
-
f.write token
|
115
|
-
end
|
153
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
|
116
154
|
end
|
117
155
|
|
118
156
|
def run
|
@@ -130,8 +168,16 @@ module Fluent::Plugin
|
|
130
168
|
log_streams.each do |log_stream|
|
131
169
|
log_stream_name = log_stream.log_stream_name
|
132
170
|
events = get_events(log_stream_name)
|
171
|
+
metadata = if @include_metadata
|
172
|
+
{
|
173
|
+
"log_stream_name" => log_stream_name,
|
174
|
+
"log_group_name" => @log_group_name
|
175
|
+
}
|
176
|
+
else
|
177
|
+
{}
|
178
|
+
end
|
133
179
|
events.each do |event|
|
134
|
-
emit(log_stream_name, event)
|
180
|
+
emit(log_stream_name, event, metadata)
|
135
181
|
end
|
136
182
|
end
|
137
183
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
|
@@ -140,8 +186,16 @@ module Fluent::Plugin
|
|
140
186
|
end
|
141
187
|
else
|
142
188
|
events = get_events(@log_stream_name)
|
189
|
+
metadata = if @include_metadata
|
190
|
+
{
|
191
|
+
"log_stream_name" => @log_stream_name,
|
192
|
+
"log_group_name" => @log_group_name
|
193
|
+
}
|
194
|
+
else
|
195
|
+
{}
|
196
|
+
end
|
143
197
|
events.each do |event|
|
144
|
-
emit(log_stream_name, event)
|
198
|
+
emit(log_stream_name, event, metadata)
|
145
199
|
end
|
146
200
|
end
|
147
201
|
end
|
@@ -149,18 +203,24 @@ module Fluent::Plugin
|
|
149
203
|
end
|
150
204
|
end
|
151
205
|
|
152
|
-
def emit(stream, event)
|
206
|
+
def emit(stream, event, metadata)
|
153
207
|
if @parser
|
154
208
|
@parser.parse(event.message) {|time,record|
|
155
209
|
if @use_aws_timestamp
|
156
210
|
time = (event.timestamp / 1000).floor
|
157
211
|
end
|
212
|
+
unless metadata.empty?
|
213
|
+
record.merge!("metadata" => metadata)
|
214
|
+
end
|
158
215
|
router.emit(@tag, time, record)
|
159
216
|
}
|
160
217
|
else
|
161
218
|
time = (event.timestamp / 1000).floor
|
162
219
|
begin
|
163
220
|
record = @json_handler.load(event.message)
|
221
|
+
unless metadata.empty?
|
222
|
+
record.merge!("metadata" => metadata)
|
223
|
+
end
|
164
224
|
router.emit(@tag, time, record)
|
165
225
|
rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
|
166
226
|
log.error "Invalid JSON encountered while parsing event.message"
|
@@ -170,49 +230,57 @@ module Fluent::Plugin
|
|
170
230
|
end
|
171
231
|
|
172
232
|
def get_events(log_stream_name)
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
233
|
+
throttling_handler('get_log_events') do
|
234
|
+
request = {
|
235
|
+
log_group_name: @log_group_name,
|
236
|
+
log_stream_name: log_stream_name
|
237
|
+
}
|
238
|
+
request.merge!(start_time: @start_time) if @start_time
|
239
|
+
request.merge!(end_time: @end_time) if @end_time
|
240
|
+
log_next_token = next_token(log_stream_name)
|
241
|
+
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
242
|
+
response = @logs.get_log_events(request)
|
243
|
+
if valid_next_token(log_next_token, response.next_forward_token)
|
244
|
+
store_next_token(response.next_forward_token, log_stream_name)
|
245
|
+
end
|
246
|
+
|
247
|
+
response.events
|
184
248
|
end
|
249
|
+
end
|
185
250
|
|
186
|
-
|
251
|
+
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
|
252
|
+
throttling_handler('describe_log_streams') do
|
253
|
+
request = {
|
254
|
+
log_group_name: @log_group_name
|
255
|
+
}
|
256
|
+
request[:next_token] = next_token if next_token
|
257
|
+
request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
|
258
|
+
response = @logs.describe_log_streams(request)
|
259
|
+
if log_streams
|
260
|
+
log_streams.concat(response.log_streams)
|
261
|
+
else
|
262
|
+
log_streams = response.log_streams
|
263
|
+
end
|
264
|
+
if response.next_token
|
265
|
+
log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
|
266
|
+
end
|
267
|
+
log_streams
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
def throttling_handler(method_name)
|
272
|
+
yield
|
187
273
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
|
188
274
|
if throttling_retry_seconds
|
189
|
-
log.warn "ThrottlingException
|
275
|
+
log.warn "ThrottlingException #{method_name}. Waiting #{throttling_retry_seconds} seconds to retry."
|
190
276
|
sleep throttling_retry_seconds
|
191
277
|
|
192
|
-
|
278
|
+
throttling_handler(method_name) { yield }
|
193
279
|
else
|
194
280
|
raise err
|
195
281
|
end
|
196
282
|
end
|
197
283
|
|
198
|
-
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
|
199
|
-
request = {
|
200
|
-
log_group_name: @log_group_name
|
201
|
-
}
|
202
|
-
request[:next_token] = next_token if next_token
|
203
|
-
request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
|
204
|
-
response = @logs.describe_log_streams(request)
|
205
|
-
if log_streams
|
206
|
-
log_streams.concat(response.log_streams)
|
207
|
-
else
|
208
|
-
log_streams = response.log_streams
|
209
|
-
end
|
210
|
-
if response.next_token
|
211
|
-
log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
|
212
|
-
end
|
213
|
-
log_streams
|
214
|
-
end
|
215
|
-
|
216
284
|
def valid_next_token(prev_token, next_token)
|
217
285
|
next_token && prev_token != next_token.chomp
|
218
286
|
end
|
@@ -7,6 +7,8 @@ module Fluent::Plugin
|
|
7
7
|
class CloudwatchLogsOutput < Output
|
8
8
|
Fluent::Plugin.register_output('cloudwatch_logs', self)
|
9
9
|
|
10
|
+
class TooLargeEventError < Fluent::UnrecoverableError; end
|
11
|
+
|
10
12
|
helpers :compat_parameters, :inject
|
11
13
|
|
12
14
|
DEFAULT_BUFFER_TYPE = "memory"
|
@@ -44,6 +46,13 @@ module Fluent::Plugin
|
|
44
46
|
config_param :remove_retention_in_days_key, :bool, default: false
|
45
47
|
config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
|
46
48
|
config_param :log_rejected_request, :bool, :default => false
|
49
|
+
config_section :web_identity_credentials, multi: false do
|
50
|
+
config_param :role_arn, :string
|
51
|
+
config_param :role_session_name, :string
|
52
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
53
|
+
config_param :policy, :string, default: nil
|
54
|
+
config_param :duration_seconds, :time, default: nil
|
55
|
+
end
|
47
56
|
|
48
57
|
config_section :buffer do
|
49
58
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -96,6 +105,18 @@ module Fluent::Plugin
|
|
96
105
|
role_arn: @aws_sts_role_arn,
|
97
106
|
role_session_name: @aws_sts_session_name
|
98
107
|
)
|
108
|
+
elsif @web_identity_credentials
|
109
|
+
c = @web_identity_credentials
|
110
|
+
credentials_options = {}
|
111
|
+
credentials_options[:role_arn] = c.role_arn
|
112
|
+
credentials_options[:role_session_name] = c.role_session_name
|
113
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
114
|
+
credentials_options[:policy] = c.policy if c.policy
|
115
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
116
|
+
if @region
|
117
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
118
|
+
end
|
119
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
99
120
|
else
|
100
121
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
101
122
|
end
|
@@ -130,6 +151,9 @@ module Fluent::Plugin
|
|
130
151
|
def write(chunk)
|
131
152
|
log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
|
132
153
|
log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
|
154
|
+
aws_tags = @log_group_aws_tags.each {|k, v|
|
155
|
+
@log_group_aws_tags[extract_placeholders(k, chunk)] = extract_placeholders(v, chunk)
|
156
|
+
} if @log_group_aws_tags
|
133
157
|
|
134
158
|
queue = Thread::Queue.new
|
135
159
|
|
@@ -182,7 +206,7 @@ module Fluent::Plugin
|
|
182
206
|
#as we create log group only once, values from first record will persist
|
183
207
|
record = rs[0][2]
|
184
208
|
|
185
|
-
awstags =
|
209
|
+
awstags = aws_tags
|
186
210
|
unless @log_group_aws_tags_key.nil?
|
187
211
|
if @remove_log_group_aws_tags_key
|
188
212
|
awstags = record.delete(@log_group_aws_tags_key)
|
@@ -319,8 +343,7 @@ module Fluent::Plugin
|
|
319
343
|
while event = events.shift
|
320
344
|
event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
|
321
345
|
if MAX_EVENT_SIZE < event_bytesize
|
322
|
-
|
323
|
-
break
|
346
|
+
raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
|
324
347
|
end
|
325
348
|
|
326
349
|
new_chunk = chunk + [event]
|
@@ -377,8 +400,7 @@ module Fluent::Plugin
|
|
377
400
|
end
|
378
401
|
rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
|
379
402
|
sleep 1 # to avoid too many API calls
|
380
|
-
|
381
|
-
store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
|
403
|
+
store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
|
382
404
|
log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
|
383
405
|
"error" => err,
|
384
406
|
"log_group" => group_name,
|
@@ -400,7 +422,13 @@ module Fluent::Plugin
|
|
400
422
|
raise err
|
401
423
|
end
|
402
424
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
|
403
|
-
if
|
425
|
+
if @put_log_events_retry_limit < 1
|
426
|
+
log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
|
427
|
+
"error_class" => err.class.to_s,
|
428
|
+
"error" => err.message,
|
429
|
+
}
|
430
|
+
return
|
431
|
+
elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
|
404
432
|
log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
|
405
433
|
"error_class" => err.class.to_s,
|
406
434
|
"error" => err.message,
|
@@ -99,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
99
99
|
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
100
100
|
end
|
101
101
|
|
102
|
+
def test_emit_with_metadata
|
103
|
+
create_log_stream
|
104
|
+
|
105
|
+
time_ms = (Time.now.to_f * 1000).floor
|
106
|
+
put_log_events([
|
107
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
|
108
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
|
109
|
+
])
|
110
|
+
|
111
|
+
sleep 5
|
112
|
+
|
113
|
+
d = create_driver(default_config + %[include_metadata true])
|
114
|
+
d.run(expect_emits: 2, timeout: 5)
|
115
|
+
|
116
|
+
emits = d.events
|
117
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
118
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
119
|
+
emits[0][2].delete_if {|k, v|
|
120
|
+
k == "metadata"
|
121
|
+
}
|
122
|
+
emits[1][2].delete_if {|k, v|
|
123
|
+
k == "metadata"
|
124
|
+
}
|
125
|
+
assert_equal(2, emits.size)
|
126
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
|
127
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
128
|
+
end
|
129
|
+
|
102
130
|
def test_emit_with_aws_timestamp
|
103
131
|
create_log_stream
|
104
132
|
|
@@ -173,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
173
201
|
'@type' => 'cloudwatch_logs',
|
174
202
|
'log_group_name' => "#{log_group_name}",
|
175
203
|
'log_stream_name' => "#{log_stream_name}",
|
176
|
-
'state_file' => '/tmp/state',
|
177
204
|
}
|
178
205
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
179
206
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -183,7 +210,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
183
210
|
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
184
211
|
config_element('parse', '', {'@type' => 'csv',
|
185
212
|
'keys' => 'time,message',
|
186
|
-
'time_key' => 'time'})
|
213
|
+
'time_key' => 'time'}),
|
214
|
+
config_element('storage', '', {'@type' => 'local',
|
215
|
+
'path' => '/tmp/state'})
|
187
216
|
])
|
188
217
|
create_log_stream
|
189
218
|
|
@@ -205,6 +234,54 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
205
234
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
206
235
|
end
|
207
236
|
|
237
|
+
test "emit with <parse> csv with metadata" do
|
238
|
+
cloudwatch_config = {'tag' => "test",
|
239
|
+
'@type' => 'cloudwatch_logs',
|
240
|
+
'log_group_name' => "#{log_group_name}",
|
241
|
+
'log_stream_name' => "#{log_stream_name}",
|
242
|
+
'include_metadata' => true,
|
243
|
+
}
|
244
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
245
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
246
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
|
247
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
|
248
|
+
|
249
|
+
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
250
|
+
config_element('parse', '', {'@type' => 'csv',
|
251
|
+
'keys' => 'time,message',
|
252
|
+
'time_key' => 'time'}),
|
253
|
+
config_element('storage', '', {'@type' => 'local',
|
254
|
+
'path' => '/tmp/state'})
|
255
|
+
|
256
|
+
])
|
257
|
+
create_log_stream
|
258
|
+
|
259
|
+
time_ms = (Time.now.to_f * 1000).floor
|
260
|
+
log_time_ms = time_ms - 10000
|
261
|
+
put_log_events([
|
262
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
|
263
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
|
264
|
+
])
|
265
|
+
|
266
|
+
sleep 5
|
267
|
+
|
268
|
+
d = create_driver(csv_format_config)
|
269
|
+
d.run(expect_emits: 2, timeout: 5)
|
270
|
+
|
271
|
+
emits = d.events
|
272
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
273
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
274
|
+
emits[0][2].delete_if {|k, v|
|
275
|
+
k == "metadata"
|
276
|
+
}
|
277
|
+
emits[1][2].delete_if {|k, v|
|
278
|
+
k == "metadata"
|
279
|
+
}
|
280
|
+
assert_equal(2, emits.size)
|
281
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
282
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
283
|
+
end
|
284
|
+
|
208
285
|
def test_emit_width_format
|
209
286
|
create_log_stream
|
210
287
|
|
@@ -246,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
246
323
|
'@type' => 'cloudwatch_logs',
|
247
324
|
'log_group_name' => "#{log_group_name}",
|
248
325
|
'log_stream_name' => "#{log_stream_name}",
|
249
|
-
'state_file' => '/tmp/state',
|
250
326
|
}
|
251
327
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
252
328
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -256,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
256
332
|
regex_format_config = config_element('ROOT', '', cloudwatch_config, [
|
257
333
|
config_element('parse', '', {'@type' => 'regexp',
|
258
334
|
'expression' => "/^(?<cloudwatch>[^ ]*)?/",
|
259
|
-
})
|
335
|
+
}),
|
336
|
+
config_element('storage', '', {'@type' => 'local',
|
337
|
+
'path' => '/tmp/state'})
|
260
338
|
])
|
261
339
|
create_log_stream
|
262
340
|
|
@@ -552,6 +630,8 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
552
630
|
end
|
553
631
|
|
554
632
|
test "emit with today's log stream" do
|
633
|
+
omit "This testcase is unstable in CI." if ENV["CI"] == "true"
|
634
|
+
|
555
635
|
config = <<-CONFIG
|
556
636
|
tag test
|
557
637
|
@type cloudwatch_logs
|
@@ -611,7 +691,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
611
691
|
assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
|
612
692
|
end
|
613
693
|
|
614
|
-
test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException" do
|
694
|
+
test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in get_log_events" do
|
615
695
|
config = <<-CONFIG
|
616
696
|
tag test
|
617
697
|
@type cloudwatch_logs
|
@@ -634,11 +714,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
634
714
|
# so, it is expected to valid_next_token once
|
635
715
|
mock(d.instance).valid_next_token(nil, nil).once
|
636
716
|
|
717
|
+
d.run
|
718
|
+
assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException get_log_events. Waiting 0.2 seconds to retry/ }.size)
|
719
|
+
end
|
720
|
+
|
721
|
+
test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in describe_log_streams" do
|
722
|
+
config = <<-CONFIG
|
723
|
+
tag test
|
724
|
+
@type cloudwatch_logs
|
725
|
+
log_group_name #{log_group_name}
|
726
|
+
use_log_stream_name_prefix true
|
727
|
+
state_file /tmp/state
|
728
|
+
fetch_interval 0.1
|
729
|
+
throttling_retry_seconds 0.2
|
730
|
+
CONFIG
|
731
|
+
|
732
|
+
# it will raises the error 2 times
|
637
733
|
log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
|
638
|
-
|
734
|
+
counter = 0
|
735
|
+
times = 2
|
736
|
+
stub(@client).describe_log_streams(anything) {
|
737
|
+
counter += 1
|
738
|
+
counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(log_streams: [log_stream], next_token: nil)
|
739
|
+
}
|
740
|
+
|
741
|
+
d = create_driver(config)
|
639
742
|
|
640
743
|
d.run
|
641
|
-
assert_equal(2, d.logs.select {|l| l =~ /Waiting 0.2 seconds to retry/ }.size)
|
744
|
+
assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException describe_log_streams. Waiting 0.2 seconds to retry/ }.size)
|
642
745
|
end
|
643
746
|
end
|
644
747
|
|
@@ -492,6 +492,47 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
492
492
|
assert_equal("value2", awstags.fetch("tag2"))
|
493
493
|
end
|
494
494
|
|
495
|
+
def test_log_group_aws_tags_with_placeholders
|
496
|
+
clear_log_group
|
497
|
+
|
498
|
+
config = {
|
499
|
+
"@type" => "cloudwatch_logs",
|
500
|
+
"auto_create_stream" => true,
|
501
|
+
"use_tag_as_stream" => true,
|
502
|
+
"log_group_name_key" => "group_name_key",
|
503
|
+
"log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
|
504
|
+
}
|
505
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
506
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
507
|
+
config.merge!(config_elementify(region)) if region
|
508
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
509
|
+
|
510
|
+
d = create_driver(
|
511
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
512
|
+
Fluent::Config::Element.new('buffer', 'tag, namespace_name', {
|
513
|
+
'@type' => 'memory',
|
514
|
+
}, [])
|
515
|
+
])
|
516
|
+
)
|
517
|
+
|
518
|
+
records = [
|
519
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
520
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
521
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
522
|
+
]
|
523
|
+
|
524
|
+
time = Time.now
|
525
|
+
d.run(default_tag: fluentd_tag) do
|
526
|
+
records.each_with_index do |record, i|
|
527
|
+
d.feed(time.to_i + i, record)
|
528
|
+
end
|
529
|
+
end
|
530
|
+
|
531
|
+
awstags = get_log_group_tags
|
532
|
+
assert_equal(fluentd_tag, awstags.fetch("tag1"))
|
533
|
+
assert_equal("fluentd", awstags.fetch("tag2"))
|
534
|
+
end
|
535
|
+
|
495
536
|
def test_retention_in_days
|
496
537
|
clear_log_group
|
497
538
|
|
@@ -713,6 +754,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
713
754
|
assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
|
714
755
|
end
|
715
756
|
|
757
|
+
def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
|
758
|
+
client = Aws::CloudWatchLogs::Client.new
|
759
|
+
@called = false
|
760
|
+
stub(client).put_log_events(anything) {
|
761
|
+
raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
|
762
|
+
}.once.ordered
|
763
|
+
|
764
|
+
d = create_driver(<<-EOC)
|
765
|
+
#{default_config}
|
766
|
+
log_group_name #{log_group_name}
|
767
|
+
log_stream_name #{log_stream_name}
|
768
|
+
@log_level debug
|
769
|
+
put_log_events_retry_limit 0
|
770
|
+
EOC
|
771
|
+
time = event_time
|
772
|
+
d.instance.instance_variable_set(:@logs, client)
|
773
|
+
d.run(default_tag: fluentd_tag) do
|
774
|
+
d.feed(time, {'message' => 'message1'})
|
775
|
+
end
|
776
|
+
|
777
|
+
logs = d.logs
|
778
|
+
assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
779
|
+
assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
|
780
|
+
assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
|
781
|
+
end
|
782
|
+
|
716
783
|
def test_retrying_on_throttling_exception
|
717
784
|
resp = Object.new
|
718
785
|
mock(resp).rejected_log_events_info {}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,13 +108,14 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
description:
|
111
|
+
description:
|
112
112
|
email:
|
113
113
|
- ryota.arai@gmail.com
|
114
114
|
executables: []
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
+
- ".github/workflows/issue-auto-closer.yml"
|
118
119
|
- ".gitignore"
|
119
120
|
- ".travis.yml"
|
120
121
|
- Gemfile
|
@@ -135,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
|
|
135
136
|
licenses:
|
136
137
|
- MIT
|
137
138
|
metadata: {}
|
138
|
-
post_install_message:
|
139
|
+
post_install_message:
|
139
140
|
rdoc_options: []
|
140
141
|
require_paths:
|
141
142
|
- lib
|
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
150
151
|
- !ruby/object:Gem::Version
|
151
152
|
version: '0'
|
152
153
|
requirements: []
|
153
|
-
rubygems_version: 3.
|
154
|
-
signing_key:
|
154
|
+
rubygems_version: 3.1.2
|
155
|
+
signing_key:
|
155
156
|
specification_version: 4
|
156
157
|
summary: CloudWatch Logs Plugin for Fluentd
|
157
158
|
test_files:
|