fluent-plugin-cloudwatch-logs 0.9.5 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/README.md +72 -1
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +83 -15
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +44 -8
- data/test/plugin/test_in_cloudwatch_logs.rb +84 -4
- data/test/plugin/test_out_cloudwatch_logs.rb +67 -0
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7d78abf2df32b76a85618e07a9164b6b4b59085bb56054b9cff15a947674eb26
|
4
|
+
data.tar.gz: 935d10a414f4ac4e83d619978695aa98bc2ec784c02ef5e60ec082ac7b39f667
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f5113e44e0fb327000423a12292f1d6c25be4e5e491773e3e074b06a9c4206797ac0a3873c5b252da9dbbfa705d7989bb6b84f71d39e4080c5eb882555c914e
|
7
|
+
data.tar.gz: ee663a6f50e7788703d3bea274c194ea83d4b330d25b284112682247984cf9de344d00a4a7aba8d7f6e7589c396510e0a0970cb48bea73d73841eb1c22e3cd77
|
@@ -0,0 +1,12 @@
|
|
1
|
+
name: Autocloser
|
2
|
+
on: [issues]
|
3
|
+
jobs:
|
4
|
+
autoclose:
|
5
|
+
runs-on: ubuntu-latest
|
6
|
+
steps:
|
7
|
+
- name: Autoclose issues that did not follow issue template
|
8
|
+
uses: roots/issue-closer-action@v1.1
|
9
|
+
with:
|
10
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
11
|
+
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
12
|
+
issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"
|
data/README.md
CHANGED
@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
|
|
43
43
|
}
|
44
44
|
```
|
45
45
|
|
46
|
+
More restricted IAM policy for `out_cloudwatch_logs` is:
|
47
|
+
|
48
|
+
```json
|
49
|
+
{
|
50
|
+
"Version": "2012-10-17",
|
51
|
+
"Statement": [
|
52
|
+
{
|
53
|
+
"Action": [
|
54
|
+
"logs:PutLogEvents",
|
55
|
+
"logs:CreateLogGroup",
|
56
|
+
"logs:PutRetentionPolicy",
|
57
|
+
"logs:CreateLogStream",
|
58
|
+
"logs:DescribeLogGroups",
|
59
|
+
"logs:DescribeLogStreams"
|
60
|
+
],
|
61
|
+
"Effect": "Allow",
|
62
|
+
"Resource": "*"
|
63
|
+
}
|
64
|
+
]
|
65
|
+
}
|
66
|
+
```
|
67
|
+
|
68
|
+
Also, more restricted IAM policy for `in_cloudwatch_logs` is:
|
69
|
+
|
70
|
+
```json
|
71
|
+
{
|
72
|
+
"Version": "2012-10-17",
|
73
|
+
"Statement": [
|
74
|
+
{
|
75
|
+
"Action": [
|
76
|
+
"logs:GetLogEvents",
|
77
|
+
"logs:DescribeLogStreams"
|
78
|
+
],
|
79
|
+
"Effect": "Allow",
|
80
|
+
"Resource": "*"
|
81
|
+
}
|
82
|
+
]
|
83
|
+
}
|
84
|
+
```
|
85
|
+
|
46
86
|
## Authentication
|
47
87
|
|
48
88
|
There are several methods to provide authentication credentials. Be aware that there are various tradeoffs for these methods,
|
@@ -120,6 +160,11 @@ Fetch sample log from CloudWatch Logs:
|
|
120
160
|
#endpoint http://localhost:5000/
|
121
161
|
#json_handler json
|
122
162
|
#log_rejected_request true
|
163
|
+
#<web_identity_credentials>
|
164
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
165
|
+
# role_session_name ROLE_SESSION_NAME
|
166
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
167
|
+
#</web_identity_credentials>
|
123
168
|
</match>
|
124
169
|
```
|
125
170
|
|
@@ -154,6 +199,14 @@ Fetch sample log from CloudWatch Logs:
|
|
154
199
|
* `retention_in_days_key`: use specified field of records as retention period
|
155
200
|
* `use_tag_as_group`: to use tag as a group name
|
156
201
|
* `use_tag_as_stream`: to use tag as a stream name
|
202
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
203
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
204
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
205
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
206
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
207
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
208
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
209
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
157
210
|
|
158
211
|
**NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
|
159
212
|
Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
|
@@ -178,6 +231,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
178
231
|
#<parse>
|
179
232
|
# @type none # or csv, tsv, regexp etc.
|
180
233
|
#</parse>
|
234
|
+
#<storage>
|
235
|
+
# @type local # or redis, memcached, etc.
|
236
|
+
#</storage>
|
237
|
+
#<web_identity_credentials>
|
238
|
+
# role_arn "#{ENV['AWS_ROLE_ARN']}"
|
239
|
+
# role_session_name ROLE_SESSION_NAME
|
240
|
+
# web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
241
|
+
#</web_identity_credentials>
|
181
242
|
</source>
|
182
243
|
```
|
183
244
|
|
@@ -194,7 +255,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
194
255
|
* `log_stream_name`: name of log stream to fetch logs
|
195
256
|
* `region`: AWS Region. See [Authentication](#authentication) for more information.
|
196
257
|
* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
|
197
|
-
* `
|
258
|
+
* `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
|
259
|
+
* `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
|
198
260
|
* `tag`: fluentd tag
|
199
261
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
200
262
|
* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
|
@@ -204,6 +266,15 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
204
266
|
* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
|
205
267
|
* `format`: specify CloudWatchLogs' log format. (default `nil`)
|
206
268
|
* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
|
269
|
+
* `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
|
270
|
+
* `<web_identity_credentials>`: For EKS authentication.
|
271
|
+
* `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
|
272
|
+
* `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
|
273
|
+
* `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
|
274
|
+
* `policy`: An IAM policy in JSON format. (default `nil`)
|
275
|
+
* `duration_seconds`: The duration, in seconds, of the role session. The value can range from
|
276
|
+
900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
|
277
|
+
is set to 3600 seconds (1 hour). (default `nil`)
|
207
278
|
|
208
279
|
## Test
|
209
280
|
|
@@ -8,20 +8,24 @@ module Fluent::Plugin
|
|
8
8
|
class CloudwatchLogsInput < Input
|
9
9
|
Fluent::Plugin.register_input('cloudwatch_logs', self)
|
10
10
|
|
11
|
-
helpers :parser, :thread, :compat_parameters
|
11
|
+
helpers :parser, :thread, :compat_parameters, :storage
|
12
|
+
|
13
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
12
14
|
|
13
15
|
config_param :aws_key_id, :string, default: nil, secret: true
|
14
16
|
config_param :aws_sec_key, :string, default: nil, secret: true
|
15
17
|
config_param :aws_use_sts, :bool, default: false
|
16
18
|
config_param :aws_sts_role_arn, :string, default: nil
|
17
19
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
20
|
+
config_param :aws_sts_endpoint_url, :string, default: nil
|
18
21
|
config_param :region, :string, default: nil
|
19
22
|
config_param :endpoint, :string, default: nil
|
20
23
|
config_param :tag, :string
|
21
24
|
config_param :log_group_name, :string
|
22
25
|
config_param :log_stream_name, :string, default: nil
|
23
26
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
24
|
-
config_param :state_file, :string
|
27
|
+
config_param :state_file, :string, default: nil,
|
28
|
+
deprecated: "Use <stroage> instead."
|
25
29
|
config_param :fetch_interval, :time, default: 60
|
26
30
|
config_param :http_proxy, :string, default: nil
|
27
31
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
@@ -31,11 +35,25 @@ module Fluent::Plugin
|
|
31
35
|
config_param :end_time, :string, default: nil
|
32
36
|
config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
|
33
37
|
config_param :throttling_retry_seconds, :time, default: nil
|
38
|
+
config_param :include_metadata, :bool, default: false
|
39
|
+
config_section :web_identity_credentials, multi: false do
|
40
|
+
config_param :role_arn, :string
|
41
|
+
config_param :role_session_name, :string
|
42
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
43
|
+
config_param :policy, :string, default: nil
|
44
|
+
config_param :duration_seconds, :time, default: nil
|
45
|
+
end
|
34
46
|
|
35
47
|
config_section :parse do
|
36
48
|
config_set_default :@type, 'none'
|
37
49
|
end
|
38
50
|
|
51
|
+
config_section :storage do
|
52
|
+
config_set_default :usage, 'store_next_tokens'
|
53
|
+
config_set_default :@type, DEFAULT_STORAGE_TYPE
|
54
|
+
config_set_default :persistent, false
|
55
|
+
end
|
56
|
+
|
39
57
|
def initialize
|
40
58
|
super
|
41
59
|
|
@@ -53,6 +71,7 @@ module Fluent::Plugin
|
|
53
71
|
if @start_time && @end_time && (@end_time < @start_time)
|
54
72
|
raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
|
55
73
|
end
|
74
|
+
@next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
56
75
|
end
|
57
76
|
|
58
77
|
def start
|
@@ -64,10 +83,29 @@ module Fluent::Plugin
|
|
64
83
|
|
65
84
|
if @aws_use_sts
|
66
85
|
Aws.config[:region] = options[:region]
|
67
|
-
|
86
|
+
credentials_options = {
|
68
87
|
role_arn: @aws_sts_role_arn,
|
69
88
|
role_session_name: @aws_sts_session_name
|
70
|
-
|
89
|
+
}
|
90
|
+
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
91
|
+
if @region and @aws_sts_endpoint_url
|
92
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
|
93
|
+
elsif @region
|
94
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region)
|
95
|
+
end
|
96
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
97
|
+
elsif @web_identity_credentials
|
98
|
+
c = @web_identity_credentials
|
99
|
+
credentials_options = {}
|
100
|
+
credentials_options[:role_arn] = c.role_arn
|
101
|
+
credentials_options[:role_session_name] = c.role_session_name
|
102
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
103
|
+
credentials_options[:policy] = c.policy if c.policy
|
104
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
105
|
+
if @region
|
106
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
107
|
+
end
|
108
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
71
109
|
else
|
72
110
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
73
111
|
end
|
@@ -99,20 +137,28 @@ module Fluent::Plugin
|
|
99
137
|
end
|
100
138
|
end
|
101
139
|
|
102
|
-
def
|
103
|
-
|
104
|
-
|
140
|
+
def state_key_for(log_stream_name)
|
141
|
+
if log_stream_name
|
142
|
+
"#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
|
143
|
+
else
|
144
|
+
@state_file
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def migrate_state_file_to_storage(log_stream_name)
|
149
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
|
150
|
+
File.delete(state_key_for(log_stream_name))
|
105
151
|
end
|
106
152
|
|
107
153
|
def next_token(log_stream_name)
|
108
|
-
|
109
|
-
|
154
|
+
if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
|
155
|
+
migrate_state_file_to_storage(log_stream_name)
|
156
|
+
end
|
157
|
+
@next_token_storage.get(:"#{state_key_for(log_stream_name)}")
|
110
158
|
end
|
111
159
|
|
112
160
|
def store_next_token(token, log_stream_name = nil)
|
113
|
-
|
114
|
-
f.write token
|
115
|
-
end
|
161
|
+
@next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
|
116
162
|
end
|
117
163
|
|
118
164
|
def run
|
@@ -130,8 +176,16 @@ module Fluent::Plugin
|
|
130
176
|
log_streams.each do |log_stream|
|
131
177
|
log_stream_name = log_stream.log_stream_name
|
132
178
|
events = get_events(log_stream_name)
|
179
|
+
metadata = if @include_metadata
|
180
|
+
{
|
181
|
+
"log_stream_name" => log_stream_name,
|
182
|
+
"log_group_name" => @log_group_name
|
183
|
+
}
|
184
|
+
else
|
185
|
+
{}
|
186
|
+
end
|
133
187
|
events.each do |event|
|
134
|
-
emit(log_stream_name, event)
|
188
|
+
emit(log_stream_name, event, metadata)
|
135
189
|
end
|
136
190
|
end
|
137
191
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
|
@@ -140,8 +194,16 @@ module Fluent::Plugin
|
|
140
194
|
end
|
141
195
|
else
|
142
196
|
events = get_events(@log_stream_name)
|
197
|
+
metadata = if @include_metadata
|
198
|
+
{
|
199
|
+
"log_stream_name" => @log_stream_name,
|
200
|
+
"log_group_name" => @log_group_name
|
201
|
+
}
|
202
|
+
else
|
203
|
+
{}
|
204
|
+
end
|
143
205
|
events.each do |event|
|
144
|
-
emit(log_stream_name, event)
|
206
|
+
emit(log_stream_name, event, metadata)
|
145
207
|
end
|
146
208
|
end
|
147
209
|
end
|
@@ -149,18 +211,24 @@ module Fluent::Plugin
|
|
149
211
|
end
|
150
212
|
end
|
151
213
|
|
152
|
-
def emit(stream, event)
|
214
|
+
def emit(stream, event, metadata)
|
153
215
|
if @parser
|
154
216
|
@parser.parse(event.message) {|time,record|
|
155
217
|
if @use_aws_timestamp
|
156
218
|
time = (event.timestamp / 1000).floor
|
157
219
|
end
|
220
|
+
unless metadata.empty?
|
221
|
+
record.merge!("metadata" => metadata)
|
222
|
+
end
|
158
223
|
router.emit(@tag, time, record)
|
159
224
|
}
|
160
225
|
else
|
161
226
|
time = (event.timestamp / 1000).floor
|
162
227
|
begin
|
163
228
|
record = @json_handler.load(event.message)
|
229
|
+
unless metadata.empty?
|
230
|
+
record.merge!("metadata" => metadata)
|
231
|
+
end
|
164
232
|
router.emit(@tag, time, record)
|
165
233
|
rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
|
166
234
|
log.error "Invalid JSON encountered while parsing event.message"
|
@@ -7,6 +7,8 @@ module Fluent::Plugin
|
|
7
7
|
class CloudwatchLogsOutput < Output
|
8
8
|
Fluent::Plugin.register_output('cloudwatch_logs', self)
|
9
9
|
|
10
|
+
class TooLargeEventError < Fluent::UnrecoverableError; end
|
11
|
+
|
10
12
|
helpers :compat_parameters, :inject
|
11
13
|
|
12
14
|
DEFAULT_BUFFER_TYPE = "memory"
|
@@ -17,6 +19,7 @@ module Fluent::Plugin
|
|
17
19
|
config_param :aws_use_sts, :bool, default: false
|
18
20
|
config_param :aws_sts_role_arn, :string, default: nil
|
19
21
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
22
|
+
config_param :aws_sts_endpoint_url, :string, default: nil
|
20
23
|
config_param :region, :string, :default => nil
|
21
24
|
config_param :endpoint, :string, :default => nil
|
22
25
|
config_param :log_group_name, :string, :default => nil
|
@@ -44,6 +47,13 @@ module Fluent::Plugin
|
|
44
47
|
config_param :remove_retention_in_days_key, :bool, default: false
|
45
48
|
config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
|
46
49
|
config_param :log_rejected_request, :bool, :default => false
|
50
|
+
config_section :web_identity_credentials, multi: false do
|
51
|
+
config_param :role_arn, :string
|
52
|
+
config_param :role_session_name, :string
|
53
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
54
|
+
config_param :policy, :string, default: nil
|
55
|
+
config_param :duration_seconds, :time, default: nil
|
56
|
+
end
|
47
57
|
|
48
58
|
config_section :buffer do
|
49
59
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -92,10 +102,29 @@ module Fluent::Plugin
|
|
92
102
|
|
93
103
|
if @aws_use_sts
|
94
104
|
Aws.config[:region] = options[:region]
|
95
|
-
|
105
|
+
credentials_options = {
|
96
106
|
role_arn: @aws_sts_role_arn,
|
97
107
|
role_session_name: @aws_sts_session_name
|
98
|
-
|
108
|
+
}
|
109
|
+
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
110
|
+
if @region and @aws_sts_endpoint_url
|
111
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
|
112
|
+
elsif @region
|
113
|
+
credentails_options[:client] = Aws::STS::Client.new(:region => @region)
|
114
|
+
end
|
115
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
116
|
+
elsif @web_identity_credentials
|
117
|
+
c = @web_identity_credentials
|
118
|
+
credentials_options = {}
|
119
|
+
credentials_options[:role_arn] = c.role_arn
|
120
|
+
credentials_options[:role_session_name] = c.role_session_name
|
121
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
122
|
+
credentials_options[:policy] = c.policy if c.policy
|
123
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
124
|
+
if @region
|
125
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
126
|
+
end
|
127
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
99
128
|
else
|
100
129
|
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
101
130
|
end
|
@@ -130,6 +159,9 @@ module Fluent::Plugin
|
|
130
159
|
def write(chunk)
|
131
160
|
log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
|
132
161
|
log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
|
162
|
+
aws_tags = @log_group_aws_tags.each {|k, v|
|
163
|
+
@log_group_aws_tags[extract_placeholders(k, chunk)] = extract_placeholders(v, chunk)
|
164
|
+
} if @log_group_aws_tags
|
133
165
|
|
134
166
|
queue = Thread::Queue.new
|
135
167
|
|
@@ -182,7 +214,7 @@ module Fluent::Plugin
|
|
182
214
|
#as we create log group only once, values from first record will persist
|
183
215
|
record = rs[0][2]
|
184
216
|
|
185
|
-
awstags =
|
217
|
+
awstags = aws_tags
|
186
218
|
unless @log_group_aws_tags_key.nil?
|
187
219
|
if @remove_log_group_aws_tags_key
|
188
220
|
awstags = record.delete(@log_group_aws_tags_key)
|
@@ -319,8 +351,7 @@ module Fluent::Plugin
|
|
319
351
|
while event = events.shift
|
320
352
|
event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
|
321
353
|
if MAX_EVENT_SIZE < event_bytesize
|
322
|
-
|
323
|
-
break
|
354
|
+
raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
|
324
355
|
end
|
325
356
|
|
326
357
|
new_chunk = chunk + [event]
|
@@ -377,8 +408,7 @@ module Fluent::Plugin
|
|
377
408
|
end
|
378
409
|
rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
|
379
410
|
sleep 1 # to avoid too many API calls
|
380
|
-
|
381
|
-
store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
|
411
|
+
store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
|
382
412
|
log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
|
383
413
|
"error" => err,
|
384
414
|
"log_group" => group_name,
|
@@ -400,7 +430,13 @@ module Fluent::Plugin
|
|
400
430
|
raise err
|
401
431
|
end
|
402
432
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
|
403
|
-
if
|
433
|
+
if @put_log_events_retry_limit < 1
|
434
|
+
log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
|
435
|
+
"error_class" => err.class.to_s,
|
436
|
+
"error" => err.message,
|
437
|
+
}
|
438
|
+
return
|
439
|
+
elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
|
404
440
|
log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
|
405
441
|
"error_class" => err.class.to_s,
|
406
442
|
"error" => err.message,
|
@@ -99,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
99
99
|
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
100
100
|
end
|
101
101
|
|
102
|
+
def test_emit_with_metadata
|
103
|
+
create_log_stream
|
104
|
+
|
105
|
+
time_ms = (Time.now.to_f * 1000).floor
|
106
|
+
put_log_events([
|
107
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
|
108
|
+
{timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
|
109
|
+
])
|
110
|
+
|
111
|
+
sleep 5
|
112
|
+
|
113
|
+
d = create_driver(default_config + %[include_metadata true])
|
114
|
+
d.run(expect_emits: 2, timeout: 5)
|
115
|
+
|
116
|
+
emits = d.events
|
117
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
118
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
119
|
+
emits[0][2].delete_if {|k, v|
|
120
|
+
k == "metadata"
|
121
|
+
}
|
122
|
+
emits[1][2].delete_if {|k, v|
|
123
|
+
k == "metadata"
|
124
|
+
}
|
125
|
+
assert_equal(2, emits.size)
|
126
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
|
127
|
+
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
128
|
+
end
|
129
|
+
|
102
130
|
def test_emit_with_aws_timestamp
|
103
131
|
create_log_stream
|
104
132
|
|
@@ -173,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
173
201
|
'@type' => 'cloudwatch_logs',
|
174
202
|
'log_group_name' => "#{log_group_name}",
|
175
203
|
'log_stream_name' => "#{log_stream_name}",
|
176
|
-
'state_file' => '/tmp/state',
|
177
204
|
}
|
178
205
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
179
206
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -183,7 +210,49 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
183
210
|
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
184
211
|
config_element('parse', '', {'@type' => 'csv',
|
185
212
|
'keys' => 'time,message',
|
186
|
-
'time_key' => 'time'})
|
213
|
+
'time_key' => 'time'}),
|
214
|
+
config_element('storage', '', {'@type' => 'local',
|
215
|
+
'path' => '/tmp/state'})
|
216
|
+
])
|
217
|
+
create_log_stream
|
218
|
+
|
219
|
+
time_ms = (Time.now.to_f * 1000).floor
|
220
|
+
log_time_ms = time_ms - 10000
|
221
|
+
put_log_events([
|
222
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
|
223
|
+
{timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
|
224
|
+
])
|
225
|
+
|
226
|
+
sleep 5
|
227
|
+
|
228
|
+
d = create_driver(csv_format_config)
|
229
|
+
d.run(expect_emits: 2, timeout: 5)
|
230
|
+
|
231
|
+
emits = d.events
|
232
|
+
assert_equal(2, emits.size)
|
233
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
234
|
+
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
235
|
+
end
|
236
|
+
|
237
|
+
test "emit with <parse> csv with metadata" do
|
238
|
+
cloudwatch_config = {'tag' => "test",
|
239
|
+
'@type' => 'cloudwatch_logs',
|
240
|
+
'log_group_name' => "#{log_group_name}",
|
241
|
+
'log_stream_name' => "#{log_stream_name}",
|
242
|
+
'include_metadata' => true,
|
243
|
+
}
|
244
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
245
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
246
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
|
247
|
+
cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
|
248
|
+
|
249
|
+
csv_format_config = config_element('ROOT', '', cloudwatch_config, [
|
250
|
+
config_element('parse', '', {'@type' => 'csv',
|
251
|
+
'keys' => 'time,message',
|
252
|
+
'time_key' => 'time'}),
|
253
|
+
config_element('storage', '', {'@type' => 'local',
|
254
|
+
'path' => '/tmp/state'})
|
255
|
+
|
187
256
|
])
|
188
257
|
create_log_stream
|
189
258
|
|
@@ -200,6 +269,14 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
200
269
|
d.run(expect_emits: 2, timeout: 5)
|
201
270
|
|
202
271
|
emits = d.events
|
272
|
+
assert_true(emits[0][2].has_key?("metadata"))
|
273
|
+
assert_true(emits[1][2].has_key?("metadata"))
|
274
|
+
emits[0][2].delete_if {|k, v|
|
275
|
+
k == "metadata"
|
276
|
+
}
|
277
|
+
emits[1][2].delete_if {|k, v|
|
278
|
+
k == "metadata"
|
279
|
+
}
|
203
280
|
assert_equal(2, emits.size)
|
204
281
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
|
205
282
|
assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
|
@@ -246,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
246
323
|
'@type' => 'cloudwatch_logs',
|
247
324
|
'log_group_name' => "#{log_group_name}",
|
248
325
|
'log_stream_name' => "#{log_stream_name}",
|
249
|
-
'state_file' => '/tmp/state',
|
250
326
|
}
|
251
327
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
|
252
328
|
cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
|
@@ -256,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
256
332
|
regex_format_config = config_element('ROOT', '', cloudwatch_config, [
|
257
333
|
config_element('parse', '', {'@type' => 'regexp',
|
258
334
|
'expression' => "/^(?<cloudwatch>[^ ]*)?/",
|
259
|
-
})
|
335
|
+
}),
|
336
|
+
config_element('storage', '', {'@type' => 'local',
|
337
|
+
'path' => '/tmp/state'})
|
260
338
|
])
|
261
339
|
create_log_stream
|
262
340
|
|
@@ -552,6 +630,8 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
552
630
|
end
|
553
631
|
|
554
632
|
test "emit with today's log stream" do
|
633
|
+
omit "This testcase is unstable in CI." if ENV["CI"] == "true"
|
634
|
+
|
555
635
|
config = <<-CONFIG
|
556
636
|
tag test
|
557
637
|
@type cloudwatch_logs
|
@@ -492,6 +492,47 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
492
492
|
assert_equal("value2", awstags.fetch("tag2"))
|
493
493
|
end
|
494
494
|
|
495
|
+
def test_log_group_aws_tags_with_placeholders
|
496
|
+
clear_log_group
|
497
|
+
|
498
|
+
config = {
|
499
|
+
"@type" => "cloudwatch_logs",
|
500
|
+
"auto_create_stream" => true,
|
501
|
+
"use_tag_as_stream" => true,
|
502
|
+
"log_group_name_key" => "group_name_key",
|
503
|
+
"log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
|
504
|
+
}
|
505
|
+
config.merge!(config_elementify(aws_key_id)) if aws_key_id
|
506
|
+
config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
|
507
|
+
config.merge!(config_elementify(region)) if region
|
508
|
+
config.merge!(config_elementify(endpoint)) if endpoint
|
509
|
+
|
510
|
+
d = create_driver(
|
511
|
+
Fluent::Config::Element.new('ROOT', '', config, [
|
512
|
+
Fluent::Config::Element.new('buffer', 'tag, namespace_name', {
|
513
|
+
'@type' => 'memory',
|
514
|
+
}, [])
|
515
|
+
])
|
516
|
+
)
|
517
|
+
|
518
|
+
records = [
|
519
|
+
{'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
520
|
+
{'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
521
|
+
{'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
|
522
|
+
]
|
523
|
+
|
524
|
+
time = Time.now
|
525
|
+
d.run(default_tag: fluentd_tag) do
|
526
|
+
records.each_with_index do |record, i|
|
527
|
+
d.feed(time.to_i + i, record)
|
528
|
+
end
|
529
|
+
end
|
530
|
+
|
531
|
+
awstags = get_log_group_tags
|
532
|
+
assert_equal(fluentd_tag, awstags.fetch("tag1"))
|
533
|
+
assert_equal("fluentd", awstags.fetch("tag2"))
|
534
|
+
end
|
535
|
+
|
495
536
|
def test_retention_in_days
|
496
537
|
clear_log_group
|
497
538
|
|
@@ -713,6 +754,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
713
754
|
assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
|
714
755
|
end
|
715
756
|
|
757
|
+
def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
|
758
|
+
client = Aws::CloudWatchLogs::Client.new
|
759
|
+
@called = false
|
760
|
+
stub(client).put_log_events(anything) {
|
761
|
+
raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
|
762
|
+
}.once.ordered
|
763
|
+
|
764
|
+
d = create_driver(<<-EOC)
|
765
|
+
#{default_config}
|
766
|
+
log_group_name #{log_group_name}
|
767
|
+
log_stream_name #{log_stream_name}
|
768
|
+
@log_level debug
|
769
|
+
put_log_events_retry_limit 0
|
770
|
+
EOC
|
771
|
+
time = event_time
|
772
|
+
d.instance.instance_variable_set(:@logs, client)
|
773
|
+
d.run(default_tag: fluentd_tag) do
|
774
|
+
d.feed(time, {'message' => 'message1'})
|
775
|
+
end
|
776
|
+
|
777
|
+
logs = d.logs
|
778
|
+
assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
|
779
|
+
assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
|
780
|
+
assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
|
781
|
+
end
|
782
|
+
|
716
783
|
def test_retrying_on_throttling_exception
|
717
784
|
resp = Object.new
|
718
785
|
mock(resp).rejected_log_events_info {}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,13 +108,14 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
description:
|
111
|
+
description:
|
112
112
|
email:
|
113
113
|
- ryota.arai@gmail.com
|
114
114
|
executables: []
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
+
- ".github/workflows/issue-auto-closer.yml"
|
118
119
|
- ".gitignore"
|
119
120
|
- ".travis.yml"
|
120
121
|
- Gemfile
|
@@ -135,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
|
|
135
136
|
licenses:
|
136
137
|
- MIT
|
137
138
|
metadata: {}
|
138
|
-
post_install_message:
|
139
|
+
post_install_message:
|
139
140
|
rdoc_options: []
|
140
141
|
require_paths:
|
141
142
|
- lib
|
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
150
151
|
- !ruby/object:Gem::Version
|
151
152
|
version: '0'
|
152
153
|
requirements: []
|
153
|
-
rubygems_version: 3.
|
154
|
-
signing_key:
|
154
|
+
rubygems_version: 3.1.2
|
155
|
+
signing_key:
|
155
156
|
specification_version: 4
|
156
157
|
summary: CloudWatch Logs Plugin for Fluentd
|
157
158
|
test_files:
|