fluent-plugin-cloudwatch-logs 0.13.0 → 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/README.md +8 -2
- data/fluent-plugin-cloudwatch-logs.gemspec +1 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +23 -8
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +7 -1
- metadata +19 -4
- data/.travis.yml +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ecdab23fc3b9c43ff71a49331694a6f9f88b0a7a27c70259b1a5b4c55894744d
|
4
|
+
data.tar.gz: 4197f274338c7134601dccfbb3e56b8672923b602f59e8fccc2d3ed43c9e4c6b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 228c5d942bba9b410f926136c59a5ad65482580d67076f8703e5dbf7b1cd778beb447b2c175d07c20d20616a2dad8d74607db2665393387edda0e534a7da8362
|
7
|
+
data.tar.gz: 6e3cd9935e8c64b390877240a18175d8b1dd0ea3df34300658ebe66df22d728e0a45beba10ac8cbc63f9159d5a725abedcd4f62cdd7b244bd5c25e17a812153a
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on Ubuntu
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
12
|
+
os:
|
13
|
+
- ubuntu-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on Windows
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
12
|
+
os:
|
13
|
+
- windows-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
data/README.md
CHANGED
@@ -16,10 +16,18 @@
|
|
16
16
|
|
17
17
|
## Installation
|
18
18
|
|
19
|
+
### For Fluentd
|
20
|
+
|
19
21
|
```sh
|
20
22
|
gem install fluent-plugin-cloudwatch-logs
|
21
23
|
```
|
22
24
|
|
25
|
+
### For td-agent
|
26
|
+
|
27
|
+
```sh
|
28
|
+
td-agent-gem install fluent-plugin-cloudwatch-logs
|
29
|
+
```
|
30
|
+
|
23
31
|
## Preparation
|
24
32
|
|
25
33
|
Create IAM user with a policy like the following:
|
@@ -444,10 +452,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
|
|
444
452
|
|
445
453
|
* out_cloudwatch_logs
|
446
454
|
* if the data is too big for API, split into multiple requests
|
447
|
-
* format
|
448
455
|
* check data size
|
449
456
|
* in_cloudwatch_logs
|
450
|
-
* format
|
451
457
|
* fallback to start_time because next_token expires after 24 hours
|
452
458
|
|
453
459
|
## Contributing
|
@@ -17,6 +17,9 @@ module Fluent::Plugin
|
|
17
17
|
config_param :aws_use_sts, :bool, default: false
|
18
18
|
config_param :aws_sts_role_arn, :string, default: nil
|
19
19
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
20
|
+
config_param :aws_sts_external_id, :string, default: nil
|
21
|
+
config_param :aws_sts_policy, :string, default: nil
|
22
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
20
23
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
21
24
|
config_param :region, :string, default: nil
|
22
25
|
config_param :endpoint, :string, default: nil
|
@@ -88,7 +91,10 @@ module Fluent::Plugin
|
|
88
91
|
Aws.config[:region] = options[:region]
|
89
92
|
credentials_options = {
|
90
93
|
role_arn: @aws_sts_role_arn,
|
91
|
-
role_session_name: @aws_sts_session_name
|
94
|
+
role_session_name: @aws_sts_session_name,
|
95
|
+
external_id: @aws_sts_external_id,
|
96
|
+
policy: @aws_sts_policy,
|
97
|
+
duration_seconds: @aws_sts_duration_seconds
|
92
98
|
}
|
93
99
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
94
100
|
if @region and @aws_sts_endpoint_url
|
@@ -185,7 +191,7 @@ module Fluent::Plugin
|
|
185
191
|
if @use_log_stream_name_prefix || @use_todays_log_stream
|
186
192
|
log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
|
187
193
|
begin
|
188
|
-
log_streams = describe_log_streams(log_stream_name_prefix)
|
194
|
+
log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
|
189
195
|
log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
|
190
196
|
log_streams.each do |log_stream|
|
191
197
|
log_stream_name = log_stream.log_stream_name
|
@@ -193,7 +199,7 @@ module Fluent::Plugin
|
|
193
199
|
metadata = if @include_metadata
|
194
200
|
{
|
195
201
|
"log_stream_name" => log_stream_name,
|
196
|
-
"log_group_name" =>
|
202
|
+
"log_group_name" => log_group_name
|
197
203
|
}
|
198
204
|
else
|
199
205
|
{}
|
@@ -221,8 +227,8 @@ module Fluent::Plugin
|
|
221
227
|
end
|
222
228
|
end
|
223
229
|
end
|
224
|
-
sleep 1
|
225
230
|
end
|
231
|
+
sleep 1
|
226
232
|
end
|
227
233
|
end
|
228
234
|
|
@@ -266,8 +272,13 @@ module Fluent::Plugin
|
|
266
272
|
}
|
267
273
|
request.merge!(start_time: @start_time) if @start_time
|
268
274
|
request.merge!(end_time: @end_time) if @end_time
|
269
|
-
|
275
|
+
if @use_log_group_name_prefix
|
276
|
+
log_next_token = next_token(log_stream_name, log_group_name)
|
277
|
+
else
|
278
|
+
log_next_token = next_token(log_stream_name)
|
279
|
+
end
|
270
280
|
request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
|
281
|
+
request[:start_from_head] = true if read_from_head?(log_next_token)
|
271
282
|
response = @logs.get_log_events(request)
|
272
283
|
if valid_next_token(log_next_token, response.next_forward_token)
|
273
284
|
if @use_log_group_name_prefix
|
@@ -281,10 +292,14 @@ module Fluent::Plugin
|
|
281
292
|
end
|
282
293
|
end
|
283
294
|
|
284
|
-
def
|
295
|
+
def read_from_head?(next_token)
|
296
|
+
(!next_token.nil? && !next_token.empty?) || @start_time || @end_time
|
297
|
+
end
|
298
|
+
|
299
|
+
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
|
285
300
|
throttling_handler('describe_log_streams') do
|
286
301
|
request = {
|
287
|
-
log_group_name: @log_group_name
|
302
|
+
log_group_name: log_group_name != nil ? log_group_name : @log_group_name
|
288
303
|
}
|
289
304
|
request[:next_token] = next_token if next_token
|
290
305
|
request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
|
@@ -295,7 +310,7 @@ module Fluent::Plugin
|
|
295
310
|
log_streams = response.log_streams
|
296
311
|
end
|
297
312
|
if response.next_token
|
298
|
-
log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
|
313
|
+
log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token, log_group_name)
|
299
314
|
end
|
300
315
|
log_streams
|
301
316
|
end
|
@@ -19,6 +19,9 @@ module Fluent::Plugin
|
|
19
19
|
config_param :aws_use_sts, :bool, default: false
|
20
20
|
config_param :aws_sts_role_arn, :string, default: nil
|
21
21
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
22
|
+
config_param :aws_sts_external_id, :string, default: nil
|
23
|
+
config_param :aws_sts_policy, :string, default: nil
|
24
|
+
config_param :aws_sts_duration_seconds, :time, default: nil
|
22
25
|
config_param :aws_sts_endpoint_url, :string, default: nil
|
23
26
|
config_param :region, :string, :default => nil
|
24
27
|
config_param :endpoint, :string, :default => nil
|
@@ -123,7 +126,10 @@ module Fluent::Plugin
|
|
123
126
|
Aws.config[:region] = options[:region]
|
124
127
|
credentials_options = {
|
125
128
|
role_arn: @aws_sts_role_arn,
|
126
|
-
role_session_name: @aws_sts_session_name
|
129
|
+
role_session_name: @aws_sts_session_name,
|
130
|
+
external_id: @aws_sts_external_id,
|
131
|
+
policy: @aws_sts_policy,
|
132
|
+
duration_seconds: @aws_sts_duration_seconds
|
127
133
|
}
|
128
134
|
credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
|
129
135
|
if @region and @aws_sts_endpoint_url
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: nokogiri
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
description:
|
112
126
|
email:
|
113
127
|
- ryota.arai@gmail.com
|
@@ -116,8 +130,9 @@ extensions: []
|
|
116
130
|
extra_rdoc_files: []
|
117
131
|
files:
|
118
132
|
- ".github/workflows/issue-auto-closer.yml"
|
133
|
+
- ".github/workflows/linux.yml"
|
134
|
+
- ".github/workflows/windows.yml"
|
119
135
|
- ".gitignore"
|
120
|
-
- ".travis.yml"
|
121
136
|
- Gemfile
|
122
137
|
- ISSUE_TEMPLATE.md
|
123
138
|
- LICENSE.txt
|
@@ -151,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
151
166
|
- !ruby/object:Gem::Version
|
152
167
|
version: '0'
|
153
168
|
requirements: []
|
154
|
-
rubygems_version: 3.
|
169
|
+
rubygems_version: 3.2.15
|
155
170
|
signing_key:
|
156
171
|
specification_version: 4
|
157
172
|
summary: CloudWatch Logs Plugin for Fluentd
|