fluent-plugin-cloudwatch-logs 0.9.5 → 0.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/README.md +72 -1
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +83 -15
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +44 -8
- data/test/plugin/test_in_cloudwatch_logs.rb +84 -4
- data/test/plugin/test_out_cloudwatch_logs.rb +67 -0
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7d78abf2df32b76a85618e07a9164b6b4b59085bb56054b9cff15a947674eb26
|
4
|
+
data.tar.gz: 935d10a414f4ac4e83d619978695aa98bc2ec784c02ef5e60ec082ac7b39f667
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f5113e44e0fb327000423a12292f1d6c25be4e5e491773e3e074b06a9c4206797ac0a3873c5b252da9dbbfa705d7989bb6b84f71d39e4080c5eb882555c914e
|
7
|
+
data.tar.gz: ee663a6f50e7788703d3bea274c194ea83d4b330d25b284112682247984cf9de344d00a4a7aba8d7f6e7589c396510e0a0970cb48bea73d73841eb1c22e3cd77
|
@@ -0,0 +1,12 @@
|
|
1
|
+
name: Autocloser
|
2
|
+
on: [issues]
|
3
|
+
jobs:
|
4
|
+
autoclose:
|
5
|
+
runs-on: ubuntu-latest
|
6
|
+
steps:
|
7
|
+
- name: Autoclose issues that did not follow issue template
|
8
|
+
uses: roots/issue-closer-action@v1.1
|
9
|
+
with:
|
10
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
11
|
+
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
12
|
+
issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"
|
data/README.md
CHANGED
@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
|
|
43
43
|
}
|
44
44
|
```
|
45
45
|
|
46
|
+
More restricted IAM policy for `out_cloudwatch_logs` is:
|
47
|
+
|
48
|
+
```json
|
49
|
+
{
|
50
|
+
"Version": "2012-10-17",
|
51
|
+
"Statement": [
|
52
|
+
{
|
53
|
+
"Action": [
|
54
|
+
"logs:PutLogEvents",
|
55
|
+
"logs:CreateLogGroup",
|
56
|
+
"logs:PutRetentionPolicy",
|
57
|
+
"logs:CreateLogStream",
|
58
|
+
"logs:DescribeLogGroups",
|
59
|
+
"logs:DescribeLogStreams"
|
60
|
+
],
|
61
|
+
"Effect": "Allow",
|
62
|
+
"Resource": "*"
|
63
|
+
}
|
64
|
+
]
|
65
|
+
}
|
66
|
+
```
|
67
|
+
|
68
|
+
Also, more restricted IAM policy for `in_cloudwatch_logs` is:
|
69
|
+
|
70
|
+
```json
|
71
|
+
{
|
72
|
+
"Version": "2012-10-17",
|
73
|
+
"Statement": [
|
74
|
+
{
|
75
|
+
"Action": [
|
76
|
+
"logs:GetLogEvents",
|
77
|
+
"logs:DescribeLogStreams"
|
78
|
+
],
|
79
|
+
"Effect": "Allow",
|
80
|
+
"Resource": "*"
|
81
|
+
}
|
82
|
+
]
|
83
|
+
}
|
84
|
+
```
|
85
|
+
|
46
86
|
## Authentication
|
47
87
|
|
48
88
|
There are several methods to provide authentication credentials. Be aware that there are various tradeoffs for these methods,
|
@@ -120,6 +160,11 @@ Fetch sample log from CloudWatch Logs:
|
|
120
160
|
#endpoint http://localhost:5000/
|
121
161
|
#json_handler json
|
122
162
|
#log_rejected_request true
|
163
|
+
#<web_identity_credentials>
|
164
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
165
|
+
# role_session_name ROLE_SESSION_NAME
|
166
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
167
|
+
#</web_identity_credentials>
|
123
168
|
</match>
|
124
169
|
```
|
125
170
|
|
@@ -154,6 +199,14 @@ Fetch sample log from CloudWatch Logs:
|
|
154
199
|
* `retention_in_days_key`: use specified field of records as retention period
|
155
200
|
* `use_tag_as_group`: to use tag as a group name
|
156
201
|
* `use_tag_as_stream`: to use tag as a stream name
|
202
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
203
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
204
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
205
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
206
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
207
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
208
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
209
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
157
210
|
|
158
211
|
**NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
|
159
212
|
Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
|
@@ -178,6 +231,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
178
231
|
#<parse>
|
179
232
|
# @type none # or csv, tsv, regexp etc.
|
180
233
|
#</parse>
|
234
|
+
#<storage>
|
235
|
+
# @type local # or redis, memcached, etc.
|
236
|
+
#</storage>
|
237
|
+
#<web_identity_credentials>
|
238
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
239
|
+
# role_session_name ROLE_SESSION_NAME
|
240
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
241
|
+
#</web_identity_credentials>
|
181
242
|
</source>
|
182
243
|
```
|
183
244
|
|
@@ -194,7 +255,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
194
255
|
* `log_stream_name`: name of log stream to fetch logs
|
195
256
|
* `region`: AWS Region. See [Authentication](#authentication) for more information.
|
196
257
|
* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
|
197
|
-
* `
|
258
|
+
* `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
|
259
|
+
* `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
|
198
260
|
* `tag`: fluentd tag
|
199
261
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
200
262
|
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
@@ -204,6 +266,15 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
204
266
|
* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
|
205
267
|
* `format`: specify CloudWatchLogs' log format. (default `nil`)
|
206
268
|
* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
|
269
|
+
* `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
|
270
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
271
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
272
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
273
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
274
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
275
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
276
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
277
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
207
278
|
|
208
279
|
## Test
|
209
280
|
|
@@ -8,20 +8,24 @@ module Fluent::Plugin
|
|
8
8
|
class CloudwatchLogsInput < Input
|
9
9
|
Fluent::Plugin.register_input('cloudwatch_logs', self)
|
10
10
|
|
11
|
-
helpers :parser, :thread, :compat_parameters
|
11
|
+
helpers :parser, :thread, :compat_parameters, :storage
|
12
|
+
|
13
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
12
14
|
|
13
15
|
config_param :aws_key_id, :string, default: nil, secret: true
|
14
16
|
config_param :aws_sec_key, :string, default: nil, secret: true
|
15
17
|
config_param :aws_use_sts, :bool, default: false
|
16
18
|
config_param :aws_sts_role_arn, :string, default: nil
|
17
19
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
20
|
+
config_param :aws_sts_endpoint_url, :string, default: nil
|
18
21
|
config_param :region, :string, default: nil
|
19
22
|
config_param :endpoint, :string, default: nil
|
20
23
|
config_param :tag, :string
|
21
24
|
config_param :log_group_name, :string
|
22
25
|
config_param :log_stream_name, :string, default: nil
|
23
26
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
24
|
-
config_param :state_file, :string
|
27
|
+
config_param :state_file, :string, default: nil,
|
28
|
+
deprecated: "Use <stroage> instead."
|
25
29
|
config_param :fetch_interval, :time, default: 60
|
26
30
|
config_param :http_proxy, :string, default: nil
|
27
31
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
@@ -31,11 +35,25 @@ module Fluent::Plugin
|
|
31
35
|
config_param :end_time, :string, default: nil
|
32
36
|
config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
|
33
37
|
config_param :throttling_retry_seconds, :time, default: nil
|
38
|
+
config_param :include_metadata, :bool, default: false
|
39
|
+
config_section :web_identity_credentials, multi: false do
|
40
|
+
config_param :role_arn, :string
|
41
|
+
config_param :role_session_name, :string
|
42
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
43
|
+
config_param :policy, :string, default: nil
|
44
|
+
config_param :duration_seconds, :time, default: nil
|
45
|
+
end
|
34
46
|
|
35
47
|
config_section :parse do
|
36
48
|
config_set_default :@type, 'none'
|
37
49
|
end
|
38
50
|
|
51
|
+
config_section :storage do
|
52
|
+
config_set_default :usage, 'store_next_tokens'
|
53
|
+
config_set_default :@type, DEFAULT_STORAGE_TYPE
|
54
|
+
config_set_default :persistent, false
|
55
|
+
end
|
56
|
+
|
39
57
|
def initialize
|
40
58
|
super
|
41
59
|
|
@@ -53,6 +71,7 @@ module Fluent::Plugin
|
|
53
71
|
if @start_time && @end_time && (@end_time < @start_time)
|
54
72
|
raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
|
55
73
|
end
|
74
|
+
@next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
56
75
|
end
|
57
76
|
|
58
77
|
def start
|
@@ -64,10 +83,29 @@ module Fluent::Plugin
|
|
64
83
|
|
65
84
|
if @aws_use_sts
|
66
85
|
Aws.config[:region] = options[:region]
|
67
|
-
|
86
|
+
credentials_options = {
|
68
87
|
role_arn: @aws_sts_role_arn,
|
69
88
|
role_session_name: @aws_sts_session_name
|
70
|
-
|
89
|
+
}
|
90
|
+
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
91
|
+
if @region and @aws_sts_endpoint_url
|
92
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
|
93
|
+
elsif @region
|
94
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region)
|
95
|
+
end
|
96
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
97
|
+
elsif @web_identity_credentials
|
98
|
+
c = @web_identity_credentials
|
99
|
+
credentials_options = {}
|
100
|
+
credentials_options[:role_arn] = c.role_arn
|
101
|
+
credentials_options[:role_session_name] = c.role_session_name
|
102
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
103
|
+
credentials_options[:policy] = c.policy if c.policy
|
104
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
105
|
+
if @region
|
106
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
107
|
+
end
|
108
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
71
109
|
else
|
72
110
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
73
111
|
end
|
@@ -99,20 +137,28 @@ module Fluent::Plugin
|
|
99
137
|
end
|
100
138
|
end
|
101
139
|
|
102
|
-
def
|
103
|
-
|
104
|
-
|
140
|
+
def state_key_for(log_stream_name)
|
141
|
+
if log_stream_name
|
142
|
+
"#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
|
143
|
+
else
|
144
|
+
@state_file
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def migrate_state_file_to_storage(log_stream_name)
|
149
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
|
150
|
+
File.delete(state_key_for(log_stream_name))
|
105
151
|
end
|
106
152
|
|
107
153
|
def next_token(log_stream_name)
|
108
|
-
|
109
|
-
|
154
|
+
if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
|
155
|
+
migrate_state_file_to_storage(log_stream_name)
|
156
|
+
end
|
157
|
+
@next_token_storage.get(:"#{state_key_for(log_stream_name)}")
|
110
158
|
end
|
111
159
|
|
112
160
|
def store_next_token(token, log_stream_name = nil)
|
113
|
-
|
114
|
-
f.write token
|
115
|
-
end
|
161
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
|
116
162
|
end
|
117
163
|
|
118
164
|
def run
|
@@ -130,8 +176,16 @@ module Fluent::Plugin
|
|
130
176
|
log_streams.each do |log_stream|
|
131
177
|
log_stream_name = log_stream.log_stream_name
|
132
178
|
events = get_events(log_stream_name)
|
179
|
+
metadata = if @include_metadata
|
180
|
+
{
|
181
|
+
"log_stream_name" => log_stream_name,
|
182
|
+
"log_group_name" => @log_group_name
|
183
|
+
}
|
184
|
+
else
|
185
|
+
{}
|
186
|
+
end
|
133
187
|
events.each do |event|
|
134
|
-
emit(log_stream_name, event)
|
188
|
+
emit(log_stream_name, event, metadata)
|
135
189
|
end
|
136
190
|
end
|
137
191
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
|
@@ -140,8 +194,16 @@ module Fluent::Plugin
|
|
140
194
|
end
|
141
195
|
else
|
142
196
|
events = get_events(@log_stream_name)
|
197
|
+
metadata = if @include_metadata
|
198
|
+
{
|
199
|
+
"log_stream_name" => @log_stream_name,
|
200
|
+
"log_group_name" => @log_group_name
|
201
|
+
}
|
202
|
+
else
|
203
|
+
{}
|
204
|
+
end
|
143
205
|
events.each do |event|
|
144
|
-
emit(log_stream_name, event)
|
206
|
+
emit(log_stream_name, event, metadata)
|
145
207
|
end
|
146
208
|
end
|
147
209
|
end
|
@@ -149,18 +211,24 @@ module Fluent::Plugin
|
|
149
211
|
end
|
150
212
|
end
|
151
213
|
|
152
|
-
def emit(stream, event)
|
214
|
+
def emit(stream, event, metadata)
|
153
215
|
if @parser
|
154
216
|
@parser.parse(event.message) {|time,record|
|
155
217
|
if @use_aws_timestamp
|
156
218
|
time = (event.timestamp / 1000).floor
|
157
219
|
end
|
220
|
+
unless metadata.empty?
|
221
|
+
record.merge!("metadata" => metadata)
|
222
|
+
end
|
158
223
|
router.emit(@tag, time, record)
|
159
224
|
}
|
160
225
|
else
|
161
226
|
time = (event.timestamp / 1000).floor
|
162
227
|
begin
|
163
228
|
record = @json_handler.load(event.message)
|
229
|
+
unless metadata.empty?
|
230
|
+
record.merge!("metadata" => metadata)
|
231
|
+
end
|
164
232
|
router.emit(@tag, time, record)
|
165
233
|
rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
|
166
234
|
log.error "Invalid JSON encountered while parsing event.message"
|
@@ -7,6 +7,8 @@ module Fluent::Plugin
|
|
7
7
|
class CloudwatchLogsOutput < Output
|
8
8
|
Fluent::Plugin.register_output('cloudwatch_logs', self)
|
9
9
|
|
10
|
+
class TooLargeEventError < Fluent::UnrecoverableError; end
|
11
|
+
|
10
12
|
helpers :compat_parameters, :inject
|
11
13
|
|
12
14
|
DEFAULT_BUFFER_TYPE = "memory"
|
@@ -17,6 +19,7 @@ module Fluent::Plugin
|
|
17
19
|
config_param :aws_use_sts, :bool, default: false
|
18
20
|
config_param :aws_sts_role_arn, :string, default: nil
|
19
21
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
22
|
+
config_param :aws_sts_endpoint_url, :string, default: nil
|
20
23
|
config_param :region, :string, :default => nil
|
21
24
|
config_param :endpoint, :string, :default => nil
|
22
25
|
config_param :log_group_name, :string, :default => nil
|
@@ -44,6 +47,13 @@ module Fluent::Plugin
|
|
44
47
|
config_param :remove_retention_in_days_key, :bool, default: false
|
45
48
|
config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
|
46
49
|
config_param :log_rejected_request, :bool, :default => false
|
50
|
+
config_section :web_identity_credentials, multi: false do
|
51
|
+
config_param :role_arn, :string
|
52
|
+
config_param :role_session_name, :string
|
53
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
54
|
+
config_param :policy, :string, default: nil
|
55
|
+
config_param :duration_seconds, :time, default: nil
|
56
|
+
end
|
47
57
|
|
48
58
|
config_section :buffer do
|
49
59
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -92,10 +102,29 @@ module Fluent::Plugin
|
|
92
102
|
|
93
103
|
if @aws_use_sts
|
94
104
|
Aws.config[:region] = options[:region]
|
95
|
-
|
105
|
+
credentials_options = {
|
96
106
|
role_arn: @aws_sts_role_arn,
|
97
107
|
role_session_name: @aws_sts_session_name
|
98
|
-
|
108
|
+
}
|
109
|
+
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
110
|
+
if @region and @aws_sts_endpoint_url
|
111
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
|
112
|
+
elsif @region
|
113
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region)
|
114
|
+
end
|
115
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
116
|
+
elsif @web_identity_credentials
|
117
|
+
c = @web_identity_credentials
|
118
|
+
credentials_options = {}
|
119
|
+
credentials_options[:role_arn] = c.role_arn
|
120
|
+
credentials_options[:role_session_name] = c.role_session_name
|
121
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
122
|
+
credentials_options[:policy] = c.policy if c.policy
|
123
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
124
|
+
if @region
|
125
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
126
|
+
end
|
127
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
99
128
|
else
|
100
129
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
101
130
|
end
|
@@ -130,6 +159,9 @@ module Fluent::Plugin
|
|
130
159
|
def write(chunk)
|
131
160
|
log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
|
132
161
|
log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
|
162
|
+
aws_tags = @log_group_aws_tags.each {|k, v|
|
163
|
+
@log_group_aws_tags[extract_placeholders(k, chunk)] = extract_placeholders(v, chunk)
|
164
|
+
} if @log_group_aws_tags
|
133
165
|
|
134
166
|
queue = Thread::Queue.new
|
135
167
|
|
@@ -182,7 +214,7 @@ module Fluent::Plugin
|
|
182
214
|
#as we create log group only once, values from first record will persist
|
183
215
|
record = rs[0][2]
|
184
216
|
|
185
|
-
awstags =
|
217
|
+
awstags = aws_tags
|
186
218
|
unless @log_group_aws_tags_key.nil?
|
187
219
|
if @remove_log_group_aws_tags_key
|
188
220
|
awstags = record.delete(@log_group_aws_tags_key)
|
@@ -319,8 +351,7 @@ module Fluent::Plugin
|
|
319
351
|
while event = events.shift
|
320
352
|
event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
|
321
353
|
if MAX_EVENT_SIZE < event_bytesize
|
322
|
-
|
323
|
-
break
|
354
|
+
raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
|
324
355
|
end
|
325
356
|
|
326
357
|
new_chunk = chunk + [event]
|
@@ -377,8 +408,7 @@ module Fluent::Plugin
|
|
377
408
|
end
|
378
409
|
rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
|
379
410
|
sleep 1 # to avoid too many API calls
|
380
|
-
|
381
|
-
store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
|
411
|
+
store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
|
382
412
|
log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
|
383
413
|
"error" => err,
|
384
414
|
"log_group" => group_name,
|
@@ -400,7 +430,13 @@ module Fluent::Plugin
|
|
400
430
|
raise err
|
401
431
|
end
|
402
432
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
|
403
|
-
if
|
433
|
+
if @put_log_events_retry_limit < 1
|
434
|
+
log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
|
435
|
+
"error_class" => err.class.to_s,
|
436
|
+
"error" => err.message,
|
437
|
+
}
|
438
|
+
return
|
439
|
+
elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
|
404
440
|
log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
|
405
441
|
"error_class" => err.class.to_s,
|
406
442
|
"error" => err.message,
|
@@ -99,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
99
99
|
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
100
100
|
end
|
101
101
|
|
102
|
+
def test_emit_with_metadata
|
103
|
+
create_log_stream
|
104
|
+
|
105
|
+
time_ms = (Time.now.to_f * 1000).floor
|
106
|
+
put_log_events([
|
107
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
|
108
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
|
109
|
+
])
|
110
|
+
|
111
|
+
sleep 5
|
112
|
+
|
113
|
+
d = create_driver(default_config + %[include_metadata true])
|
114
|
+
d.run(expect_emits: 2, timeout: 5)
|
115
|
+
|
116
|
+
emits = d.events
|
117
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
118
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
119
|
+
emits[0][2].delete_if {|k, v|
|
120
|
+
k == "metadata"
|
121
|
+
}
|
122
|
+
emits[1][2].delete_if {|k, v|
|
123
|
+
k == "metadata"
|
124
|
+
}
|
125
|
+
assert_equal(2, emits.size)
|
126
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
|
127
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
128
|
+
end
|
129
|
+
|
102
130
|
def test_emit_with_aws_timestamp
|
103
131
|
create_log_stream
|
104
132
|
|
@@ -173,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
173
201
|
'@type' => 'cloudwatch_logs',
|
174
202
|
'log_group_name' => "#{log_group_name}",
|
175
203
|
'log_stream_name' => "#{log_stream_name}",
|
176
|
-
'state_file' => '/tmp/state',
|
177
204
|
}
|
178
205
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
179
206
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -183,7 +210,49 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
183
210
|
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
184
211
|
config_element('parse', '', {'@type' => 'csv',
|
185
212
|
'keys' => 'time,message',
|
186
|
-
'time_key' => 'time'})
|
213
|
+
'time_key' => 'time'}),
|
214
|
+
config_element('storage', '', {'@type' => 'local',
|
215
|
+
'path' => '/tmp/state'})
|
216
|
+
])
|
217
|
+
create_log_stream
|
218
|
+
|
219
|
+
time_ms = (Time.now.to_f * 1000).floor
|
220
|
+
log_time_ms = time_ms - 10000
|
221
|
+
put_log_events([
|
222
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
|
223
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
|
224
|
+
])
|
225
|
+
|
226
|
+
sleep 5
|
227
|
+
|
228
|
+
d = create_driver(csv_format_config)
|
229
|
+
d.run(expect_emits: 2, timeout: 5)
|
230
|
+
|
231
|
+
emits = d.events
|
232
|
+
assert_equal(2, emits.size)
|
233
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
234
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
235
|
+
end
|
236
|
+
|
237
|
+
test "emit with <parse> csv with metadata" do
|
238
|
+
cloudwatch_config = {'tag' => "test",
|
239
|
+
'@type' => 'cloudwatch_logs',
|
240
|
+
'log_group_name' => "#{log_group_name}",
|
241
|
+
'log_stream_name' => "#{log_stream_name}",
|
242
|
+
'include_metadata' => true,
|
243
|
+
}
|
244
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
245
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
246
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
|
247
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
|
248
|
+
|
249
|
+
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
250
|
+
config_element('parse', '', {'@type' => 'csv',
|
251
|
+
'keys' => 'time,message',
|
252
|
+
'time_key' => 'time'}),
|
253
|
+
config_element('storage', '', {'@type' => 'local',
|
254
|
+
'path' => '/tmp/state'})
|
255
|
+
|
187
256
|
])
|
188
257
|
create_log_stream
|
189
258
|
|
@@ -200,6 +269,14 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
200
269
|
d.run(expect_emits: 2, timeout: 5)
|
201
270
|
|
202
271
|
emits = d.events
|
272
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
273
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
274
|
+
emits[0][2].delete_if {|k, v|
|
275
|
+
k == "metadata"
|
276
|
+
}
|
277
|
+
emits[1][2].delete_if {|k, v|
|
278
|
+
k == "metadata"
|
279
|
+
}
|
203
280
|
assert_equal(2, emits.size)
|
204
281
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
205
282
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
@@ -246,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
246
323
|
'@type' => 'cloudwatch_logs',
|
247
324
|
'log_group_name' => "#{log_group_name}",
|
248
325
|
'log_stream_name' => "#{log_stream_name}",
|
249
|
-
'state_file' => '/tmp/state',
|
250
326
|
}
|
251
327
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
252
328
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -256,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
256
332
|
regex_format_config = config_element('ROOT', '', cloudwatch_config, [
|
257
333
|
config_element('parse', '', {'@type' => 'regexp',
|
258
334
|
'expression' => "/^(?<cloudwatch>[^ ]*)?/",
|
259
|
-
})
|
335
|
+
}),
|
336
|
+
config_element('storage', '', {'@type' => 'local',
|
337
|
+
'path' => '/tmp/state'})
|
260
338
|
])
|
261
339
|
create_log_stream
|
262
340
|
|
@@ -552,6 +630,8 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
552
630
|
end
|
553
631
|
|
554
632
|
test "emit with today's log stream" do
|
633
|
+
omit "This testcase is unstable in CI." if ENV["CI"] == "true"
|
634
|
+
|
555
635
|
config = <<-CONFIG
|
556
636
|
tag test
|
557
637
|
@type cloudwatch_logs
|
@@ -492,6 +492,47 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
492
492
|
assert_equal("value2", awstags.fetch("tag2"))
|
493
493
|
end
|
494
494
|
|
495
|
+
def test_log_group_aws_tags_with_placeholders
|
496
|
+
clear_log_group
|
497
|
+
|
498
|
+
config = {
|
499
|
+
"@type" => "cloudwatch_logs",
|
500
|
+
"auto_create_stream" => true,
|
501
|
+
"use_tag_as_stream" => true,
|
502
|
+
"log_group_name_key" => "group_name_key",
|
503
|
+
"log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
|
504
|
+
}
|
505
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
506
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
507
|
+
config.merge!(config_elementify(region)) if region
|
508
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
509
|
+
|
510
|
+
d = create_driver(
|
511
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
512
|
+
Fluent::Config::Element.new('buffer', 'tag, namespace_name', {
|
513
|
+
'@type' => 'memory',
|
514
|
+
}, [])
|
515
|
+
])
|
516
|
+
)
|
517
|
+
|
518
|
+
records = [
|
519
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
520
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
521
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
522
|
+
]
|
523
|
+
|
524
|
+
time = Time.now
|
525
|
+
d.run(default_tag: fluentd_tag) do
|
526
|
+
records.each_with_index do |record, i|
|
527
|
+
d.feed(time.to_i + i, record)
|
528
|
+
end
|
529
|
+
end
|
530
|
+
|
531
|
+
awstags = get_log_group_tags
|
532
|
+
assert_equal(fluentd_tag, awstags.fetch("tag1"))
|
533
|
+
assert_equal("fluentd", awstags.fetch("tag2"))
|
534
|
+
end
|
535
|
+
|
495
536
|
def test_retention_in_days
|
496
537
|
clear_log_group
|
497
538
|
|
@@ -713,6 +754,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
713
754
|
assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
|
714
755
|
end
|
715
756
|
|
757
|
+
def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
|
758
|
+
client = Aws::CloudWatchLogs::Client.new
|
759
|
+
@called = false
|
760
|
+
stub(client).put_log_events(anything) {
|
761
|
+
raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
|
762
|
+
}.once.ordered
|
763
|
+
|
764
|
+
d = create_driver(<<-EOC)
|
765
|
+
#{default_config}
|
766
|
+
log_group_name #{log_group_name}
|
767
|
+
log_stream_name #{log_stream_name}
|
768
|
+
@log_level debug
|
769
|
+
put_log_events_retry_limit 0
|
770
|
+
EOC
|
771
|
+
time = event_time
|
772
|
+
d.instance.instance_variable_set(:@logs, client)
|
773
|
+
d.run(default_tag: fluentd_tag) do
|
774
|
+
d.feed(time, {'message' => 'message1'})
|
775
|
+
end
|
776
|
+
|
777
|
+
logs = d.logs
|
778
|
+
assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
779
|
+
assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
|
780
|
+
assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
|
781
|
+
end
|
782
|
+
|
716
783
|
def test_retrying_on_throttling_exception
|
717
784
|
resp = Object.new
|
718
785
|
mock(resp).rejected_log_events_info {}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,13 +108,14 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
description:
|
111
|
+
description:
|
112
112
|
email:
|
113
113
|
- ryota.arai@gmail.com
|
114
114
|
executables: []
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
+
- ".github/workflows/issue-auto-closer.yml"
|
118
119
|
- ".gitignore"
|
119
120
|
- ".travis.yml"
|
120
121
|
- Gemfile
|
@@ -135,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
|
|
135
136
|
licenses:
|
136
137
|
- MIT
|
137
138
|
metadata: {}
|
138
|
-
post_install_message:
|
139
|
+
post_install_message:
|
139
140
|
rdoc_options: []
|
140
141
|
require_paths:
|
141
142
|
- lib
|
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
150
151
|
- !ruby/object:Gem::Version
|
151
152
|
version: '0'
|
152
153
|
requirements: []
|
153
|
-
rubygems_version: 3.
|
154
|
-
signing_key:
|
154
|
+
rubygems_version: 3.1.2
|
155
|
+
signing_key:
|
155
156
|
specification_version: 4
|
156
157
|
summary: CloudWatch Logs Plugin for Fluentd
|
157
158
|
test_files:
|