fluent-plugin-cloudwatch-logs 0.13.2 → 0.14.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/README.md +11 -3
- data/fluent-plugin-cloudwatch-logs.gemspec +1 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +21 -4
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +9 -1
- metadata +21 -8
- data/.travis.yml +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e5c502cb8ad443406678f2c8ce7f6c8ea031c75f18f502d2b85ee1401e42faf
|
4
|
+
data.tar.gz: 3a06cf5ea24ef24bab2078ed216930508a2bd51940d659ff104c2de1515072db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e5e8033091ea7f2f461095271b64ede8b5d807c4a7215ba403b25c7f0028bdc0d68ad4a2ed3171f9ea50b9997902e52140bdd6587c8d67af1a925f7dc5be726
|
7
|
+
data.tar.gz: 21d9c62b75f66b6fb8e5b65915e91dd26696612e8439ea3ffd65c2b2fff7fd169556e47b8d451874d0134c97d5523c5d8afe5f1f052bc016b08c6ad147cecd8f
|
data/.github/workflows/linux.yml
CHANGED
data/README.md
CHANGED
@@ -16,10 +16,18 @@
|
|
16
16
|
|
17
17
|
## Installation
|
18
18
|
|
19
|
+
### For Fluentd
|
20
|
+
|
19
21
|
```sh
|
20
22
|
gem install fluent-plugin-cloudwatch-logs
|
21
23
|
```
|
22
24
|
|
25
|
+
### For td-agent
|
26
|
+
|
27
|
+
```sh
|
28
|
+
td-agent-gem install fluent-plugin-cloudwatch-logs
|
29
|
+
```
|
30
|
+
|
23
31
|
## Preparation
|
24
32
|
|
25
33
|
Create IAM user with a policy like the following:
|
@@ -98,7 +106,7 @@ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
|
|
98
106
|
export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
|
99
107
|
```
|
100
108
|
|
101
|
-
Note: For this to work persistently the
|
109
|
+
Note: For this to work persistently the environment will need to be set in the startup scripts or docker variables.
|
102
110
|
|
103
111
|
### AWS Configuration
|
104
112
|
|
@@ -176,6 +184,7 @@ Fetch sample log from CloudWatch Logs:
|
|
176
184
|
* `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information.
|
177
185
|
* `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
|
178
186
|
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
187
|
+
* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
|
179
188
|
* `http_proxy`: use to set an optional HTTP proxy
|
180
189
|
* `include_time_key`: include time key as part of the log entry (defaults to UTC)
|
181
190
|
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
@@ -255,6 +264,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
255
264
|
* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
|
256
265
|
* `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
|
257
266
|
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
267
|
+
* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
|
258
268
|
* `fetch_interval`: time period in seconds between checking CloudWatch for new logs. (default: 60)
|
259
269
|
* `http_proxy`: use to set an optional HTTP proxy
|
260
270
|
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
@@ -444,10 +454,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
|
|
444
454
|
|
445
455
|
* out_cloudwatch_logs
|
446
456
|
* if the data is too big for API, split into multiple requests
|
447
|
-
* format
|
448
457
|
* check data size
|
449
458
|
* in_cloudwatch_logs
|
450
|
-
* format
|
451
459
|
* fallback to start_time because next_token expires after 24 hours
|
452
460
|
|
453
461
|
## Contributing
|
@@ -17,9 +17,13 @@ module Fluent::Plugin
|
|
17
17
|
config_param :aws_use_sts, :bool, default: false
|
18
18
|
config_param :aws_sts_role_arn, :string, default: nil
|
19
19
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
20
|
+
config_param :aws_sts_external_id, :string, default: nil
|
21
|
+
config_param :aws_sts_policy, :string, default: nil
|
22
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
20
23
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
21
24
|
config_param :region, :string, default: nil
|
22
25
|
config_param :endpoint, :string, default: nil
|
26
|
+
config_param :ssl_verify_peer, :bool, :default => true
|
23
27
|
config_param :tag, :string
|
24
28
|
config_param :log_group_name, :string
|
25
29
|
config_param :add_log_group_name, :bool, default: false
|
@@ -28,7 +32,7 @@ module Fluent::Plugin
|
|
28
32
|
config_param :log_stream_name, :string, default: nil
|
29
33
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
30
34
|
config_param :state_file, :string, default: nil,
|
31
|
-
deprecated: "Use <
|
35
|
+
deprecated: "Use <storage> instead."
|
32
36
|
config_param :fetch_interval, :time, default: 60
|
33
37
|
config_param :http_proxy, :string, default: nil
|
34
38
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
@@ -82,13 +86,17 @@ module Fluent::Plugin
|
|
82
86
|
options = {}
|
83
87
|
options[:region] = @region if @region
|
84
88
|
options[:endpoint] = @endpoint if @endpoint
|
89
|
+
options[:ssl_verify_peer] = @ssl_verify_peer
|
85
90
|
options[:http_proxy] = @http_proxy if @http_proxy
|
86
91
|
|
87
92
|
if @aws_use_sts
|
88
93
|
Aws.config[:region] = options[:region]
|
89
94
|
credentials_options = {
|
90
95
|
role_arn: @aws_sts_role_arn,
|
91
|
-
role_session_name: @aws_sts_session_name
|
96
|
+
role_session_name: @aws_sts_session_name,
|
97
|
+
external_id: @aws_sts_external_id,
|
98
|
+
policy: @aws_sts_policy,
|
99
|
+
duration_seconds: @aws_sts_duration_seconds
|
92
100
|
}
|
93
101
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
94
102
|
if @region and @aws_sts_endpoint_url
|
@@ -221,8 +229,8 @@ module Fluent::Plugin
|
|
221
229
|
end
|
222
230
|
end
|
223
231
|
end
|
224
|
-
sleep 1
|
225
232
|
end
|
233
|
+
sleep 1
|
226
234
|
end
|
227
235
|
end
|
228
236
|
|
@@ -266,8 +274,13 @@ module Fluent::Plugin
|
|
266
274
|
}
|
267
275
|
request.merge!(start_time: @start_time) if @start_time
|
268
276
|
request.merge!(end_time: @end_time) if @end_time
|
269
|
-
|
277
|
+
if @use_log_group_name_prefix
|
278
|
+
log_next_token = next_token(log_stream_name, log_group_name)
|
279
|
+
else
|
280
|
+
log_next_token = next_token(log_stream_name)
|
281
|
+
end
|
270
282
|
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
283
|
+
request[:start_from_head] = true if read_from_head?(log_next_token)
|
271
284
|
response = @logs.get_log_events(request)
|
272
285
|
if valid_next_token(log_next_token, response.next_forward_token)
|
273
286
|
if @use_log_group_name_prefix
|
@@ -281,6 +294,10 @@ module Fluent::Plugin
|
|
281
294
|
end
|
282
295
|
end
|
283
296
|
|
297
|
+
def read_from_head?(next_token)
|
298
|
+
(!next_token.nil? && !next_token.empty?) || @start_time || @end_time
|
299
|
+
end
|
300
|
+
|
284
301
|
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
|
285
302
|
throttling_handler('describe_log_streams') do
|
286
303
|
request = {
|
@@ -19,9 +19,13 @@ module Fluent::Plugin
|
|
19
19
|
config_param :aws_use_sts, :bool, default: false
|
20
20
|
config_param :aws_sts_role_arn, :string, default: nil
|
21
21
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
22
|
+
config_param :aws_sts_external_id, :string, default: nil
|
23
|
+
config_param :aws_sts_policy, :string, default: nil
|
24
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
22
25
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
23
26
|
config_param :region, :string, :default => nil
|
24
27
|
config_param :endpoint, :string, :default => nil
|
28
|
+
config_param :ssl_verify_peer, :bool, :default => true
|
25
29
|
config_param :log_group_name, :string, :default => nil
|
26
30
|
config_param :log_stream_name, :string, :default => nil
|
27
31
|
config_param :auto_create_stream, :bool, default: false
|
@@ -117,13 +121,17 @@ module Fluent::Plugin
|
|
117
121
|
options[:log_level] = :debug if log
|
118
122
|
options[:region] = @region if @region
|
119
123
|
options[:endpoint] = @endpoint if @endpoint
|
124
|
+
options[:ssl_verify_peer] = @ssl_verify_peer
|
120
125
|
options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
|
121
126
|
|
122
127
|
if @aws_use_sts
|
123
128
|
Aws.config[:region] = options[:region]
|
124
129
|
credentials_options = {
|
125
130
|
role_arn: @aws_sts_role_arn,
|
126
|
-
role_session_name: @aws_sts_session_name
|
131
|
+
role_session_name: @aws_sts_session_name,
|
132
|
+
external_id: @aws_sts_external_id,
|
133
|
+
policy: @aws_sts_policy,
|
134
|
+
duration_seconds: @aws_sts_duration_seconds
|
127
135
|
}
|
128
136
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
129
137
|
if @region and @aws_sts_endpoint_url
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-09-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,7 +108,21 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: nokogiri
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description:
|
112
126
|
email:
|
113
127
|
- ryota.arai@gmail.com
|
114
128
|
executables: []
|
@@ -119,7 +133,6 @@ files:
|
|
119
133
|
- ".github/workflows/linux.yml"
|
120
134
|
- ".github/workflows/windows.yml"
|
121
135
|
- ".gitignore"
|
122
|
-
- ".travis.yml"
|
123
136
|
- Gemfile
|
124
137
|
- ISSUE_TEMPLATE.md
|
125
138
|
- LICENSE.txt
|
@@ -138,7 +151,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
|
|
138
151
|
licenses:
|
139
152
|
- MIT
|
140
153
|
metadata: {}
|
141
|
-
post_install_message:
|
154
|
+
post_install_message:
|
142
155
|
rdoc_options: []
|
143
156
|
require_paths:
|
144
157
|
- lib
|
@@ -153,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
166
|
- !ruby/object:Gem::Version
|
154
167
|
version: '0'
|
155
168
|
requirements: []
|
156
|
-
rubygems_version: 3.
|
157
|
-
signing_key:
|
169
|
+
rubygems_version: 3.2.5
|
170
|
+
signing_key:
|
158
171
|
specification_version: 4
|
159
172
|
summary: CloudWatch Logs Plugin for Fluentd
|
160
173
|
test_files:
|