fluent-plugin-cloudwatch-logs 0.13.2 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/README.md +11 -3
- data/fluent-plugin-cloudwatch-logs.gemspec +1 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +21 -4
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +9 -1
- metadata +21 -8
- data/.travis.yml +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e5c502cb8ad443406678f2c8ce7f6c8ea031c75f18f502d2b85ee1401e42faf
|
4
|
+
data.tar.gz: 3a06cf5ea24ef24bab2078ed216930508a2bd51940d659ff104c2de1515072db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e5e8033091ea7f2f461095271b64ede8b5d807c4a7215ba403b25c7f0028bdc0d68ad4a2ed3171f9ea50b9997902e52140bdd6587c8d67af1a925f7dc5be726
|
7
|
+
data.tar.gz: 21d9c62b75f66b6fb8e5b65915e91dd26696612e8439ea3ffd65c2b2fff7fd169556e47b8d451874d0134c97d5523c5d8afe5f1f052bc016b08c6ad147cecd8f
|
data/.github/workflows/linux.yml
CHANGED
data/README.md
CHANGED
@@ -16,10 +16,18 @@
|
|
16
16
|
|
17
17
|
## Installation
|
18
18
|
|
19
|
+
### For Fluentd
|
20
|
+
|
19
21
|
```sh
|
20
22
|
gem install fluent-plugin-cloudwatch-logs
|
21
23
|
```
|
22
24
|
|
25
|
+
### For td-agent
|
26
|
+
|
27
|
+
```sh
|
28
|
+
td-agent-gem install fluent-plugin-cloudwatch-logs
|
29
|
+
```
|
30
|
+
|
23
31
|
## Preparation
|
24
32
|
|
25
33
|
Create IAM user with a policy like the following:
|
@@ -98,7 +106,7 @@ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
|
|
98
106
|
export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
|
99
107
|
```
|
100
108
|
|
101
|
-
Note: For this to work persistently the
|
109
|
+
Note: For this to work persistently the environment will need to be set in the startup scripts or docker variables.
|
102
110
|
|
103
111
|
### AWS Configuration
|
104
112
|
|
@@ -176,6 +184,7 @@ Fetch sample log from CloudWatch Logs:
|
|
176
184
|
* `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information.
|
177
185
|
* `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
|
178
186
|
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
187
|
+
* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
|
179
188
|
* `http_proxy`: use to set an optional HTTP proxy
|
180
189
|
* `include_time_key`: include time key as part of the log entry (defaults to UTC)
|
181
190
|
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
@@ -255,6 +264,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
255
264
|
* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
|
256
265
|
* `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
|
257
266
|
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
267
|
+
* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
|
258
268
|
* `fetch_interval`: time period in seconds between checking CloudWatch for new logs. (default: 60)
|
259
269
|
* `http_proxy`: use to set an optional HTTP proxy
|
260
270
|
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
@@ -444,10 +454,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
|
|
444
454
|
|
445
455
|
* out_cloudwatch_logs
|
446
456
|
* if the data is too big for API, split into multiple requests
|
447
|
-
* format
|
448
457
|
* check data size
|
449
458
|
* in_cloudwatch_logs
|
450
|
-
* format
|
451
459
|
* fallback to start_time because next_token expires after 24 hours
|
452
460
|
|
453
461
|
## Contributing
|
@@ -17,9 +17,13 @@ module Fluent::Plugin
|
|
17
17
|
config_param :aws_use_sts, :bool, default: false
|
18
18
|
config_param :aws_sts_role_arn, :string, default: nil
|
19
19
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
20
|
+
config_param :aws_sts_external_id, :string, default: nil
|
21
|
+
config_param :aws_sts_policy, :string, default: nil
|
22
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
20
23
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
21
24
|
config_param :region, :string, default: nil
|
22
25
|
config_param :endpoint, :string, default: nil
|
26
|
+
config_param :ssl_verify_peer, :bool, :default => true
|
23
27
|
config_param :tag, :string
|
24
28
|
config_param :log_group_name, :string
|
25
29
|
config_param :add_log_group_name, :bool, default: false
|
@@ -28,7 +32,7 @@ module Fluent::Plugin
|
|
28
32
|
config_param :log_stream_name, :string, default: nil
|
29
33
|
config_param :use_log_stream_name_prefix, :bool, default: false
|
30
34
|
config_param :state_file, :string, default: nil,
|
31
|
-
deprecated: "Use <
|
35
|
+
deprecated: "Use <storage> instead."
|
32
36
|
config_param :fetch_interval, :time, default: 60
|
33
37
|
config_param :http_proxy, :string, default: nil
|
34
38
|
config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
|
@@ -82,13 +86,17 @@ module Fluent::Plugin
|
|
82
86
|
options = {}
|
83
87
|
options[:region] = @region if @region
|
84
88
|
options[:endpoint] = @endpoint if @endpoint
|
89
|
+
options[:ssl_verify_peer] = @ssl_verify_peer
|
85
90
|
options[:http_proxy] = @http_proxy if @http_proxy
|
86
91
|
|
87
92
|
if @aws_use_sts
|
88
93
|
Aws.config[:region] = options[:region]
|
89
94
|
credentials_options = {
|
90
95
|
role_arn: @aws_sts_role_arn,
|
91
|
-
role_session_name: @aws_sts_session_name
|
96
|
+
role_session_name: @aws_sts_session_name,
|
97
|
+
external_id: @aws_sts_external_id,
|
98
|
+
policy: @aws_sts_policy,
|
99
|
+
duration_seconds: @aws_sts_duration_seconds
|
92
100
|
}
|
93
101
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
94
102
|
if @region and @aws_sts_endpoint_url
|
@@ -221,8 +229,8 @@ module Fluent::Plugin
|
|
221
229
|
end
|
222
230
|
end
|
223
231
|
end
|
224
|
-
sleep 1
|
225
232
|
end
|
233
|
+
sleep 1
|
226
234
|
end
|
227
235
|
end
|
228
236
|
|
@@ -266,8 +274,13 @@ module Fluent::Plugin
|
|
266
274
|
}
|
267
275
|
request.merge!(start_time: @start_time) if @start_time
|
268
276
|
request.merge!(end_time: @end_time) if @end_time
|
269
|
-
|
277
|
+
if @use_log_group_name_prefix
|
278
|
+
log_next_token = next_token(log_stream_name, log_group_name)
|
279
|
+
else
|
280
|
+
log_next_token = next_token(log_stream_name)
|
281
|
+
end
|
270
282
|
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
283
|
+
request[:start_from_head] = true if read_from_head?(log_next_token)
|
271
284
|
response = @logs.get_log_events(request)
|
272
285
|
if valid_next_token(log_next_token, response.next_forward_token)
|
273
286
|
if @use_log_group_name_prefix
|
@@ -281,6 +294,10 @@ module Fluent::Plugin
|
|
281
294
|
end
|
282
295
|
end
|
283
296
|
|
297
|
+
def read_from_head?(next_token)
|
298
|
+
(!next_token.nil? && !next_token.empty?) || @start_time || @end_time
|
299
|
+
end
|
300
|
+
|
284
301
|
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
|
285
302
|
throttling_handler('describe_log_streams') do
|
286
303
|
request = {
|
@@ -19,9 +19,13 @@ module Fluent::Plugin
|
|
19
19
|
config_param :aws_use_sts, :bool, default: false
|
20
20
|
config_param :aws_sts_role_arn, :string, default: nil
|
21
21
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
22
|
+
config_param :aws_sts_external_id, :string, default: nil
|
23
|
+
config_param :aws_sts_policy, :string, default: nil
|
24
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
22
25
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
23
26
|
config_param :region, :string, :default => nil
|
24
27
|
config_param :endpoint, :string, :default => nil
|
28
|
+
config_param :ssl_verify_peer, :bool, :default => true
|
25
29
|
config_param :log_group_name, :string, :default => nil
|
26
30
|
config_param :log_stream_name, :string, :default => nil
|
27
31
|
config_param :auto_create_stream, :bool, default: false
|
@@ -117,13 +121,17 @@ module Fluent::Plugin
|
|
117
121
|
options[:log_level] = :debug if log
|
118
122
|
options[:region] = @region if @region
|
119
123
|
options[:endpoint] = @endpoint if @endpoint
|
124
|
+
options[:ssl_verify_peer] = @ssl_verify_peer
|
120
125
|
options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
|
121
126
|
|
122
127
|
if @aws_use_sts
|
123
128
|
Aws.config[:region] = options[:region]
|
124
129
|
credentials_options = {
|
125
130
|
role_arn: @aws_sts_role_arn,
|
126
|
-
role_session_name: @aws_sts_session_name
|
131
|
+
role_session_name: @aws_sts_session_name,
|
132
|
+
external_id: @aws_sts_external_id,
|
133
|
+
policy: @aws_sts_policy,
|
134
|
+
duration_seconds: @aws_sts_duration_seconds
|
127
135
|
}
|
128
136
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
129
137
|
if @region and @aws_sts_endpoint_url
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-09-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,7 +108,21 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: nokogiri
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description:
|
112
126
|
email:
|
113
127
|
- ryota.arai@gmail.com
|
114
128
|
executables: []
|
@@ -119,7 +133,6 @@ files:
|
|
119
133
|
- ".github/workflows/linux.yml"
|
120
134
|
- ".github/workflows/windows.yml"
|
121
135
|
- ".gitignore"
|
122
|
-
- ".travis.yml"
|
123
136
|
- Gemfile
|
124
137
|
- ISSUE_TEMPLATE.md
|
125
138
|
- LICENSE.txt
|
@@ -138,7 +151,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
|
|
138
151
|
licenses:
|
139
152
|
- MIT
|
140
153
|
metadata: {}
|
141
|
-
post_install_message:
|
154
|
+
post_install_message:
|
142
155
|
rdoc_options: []
|
143
156
|
require_paths:
|
144
157
|
- lib
|
@@ -153,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
166
|
- !ruby/object:Gem::Version
|
154
167
|
version: '0'
|
155
168
|
requirements: []
|
156
|
-
rubygems_version: 3.
|
157
|
-
signing_key:
|
169
|
+
rubygems_version: 3.2.5
|
170
|
+
signing_key:
|
158
171
|
specification_version: 4
|
159
172
|
summary: CloudWatch Logs Plugin for Fluentd
|
160
173
|
test_files:
|