fluent-plugin-cloudwatch-logs 0.13.0 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 17e367fb1fc1c2b58cd9a22cd9caefaf6396b3542f589b44616f160c40c3219d
4
- data.tar.gz: 37eee2b6a48d17d86451a8eee88435fc1ddd572248b3ed181bf3d833d97a3d96
3
+ metadata.gz: ecdab23fc3b9c43ff71a49331694a6f9f88b0a7a27c70259b1a5b4c55894744d
4
+ data.tar.gz: 4197f274338c7134601dccfbb3e56b8672923b602f59e8fccc2d3ed43c9e4c6b
5
5
  SHA512:
6
- metadata.gz: 30807c60113fe1eed915092e005e83863a4cfc5911a859eb00925299a1a4e45dc0dd581006d0486d3ab176ed2babf7b1fb5e90f0b41e04838c855a578db31faf
7
- data.tar.gz: a44051e6aa3f1010dd5ee17f1046dfccabd59a929ed5b8976eeda135410f681400f1f024c61a84992cc28500285148e598aad7aa2daab10a965aca0204398a28
6
+ metadata.gz: 228c5d942bba9b410f926136c59a5ad65482580d67076f8703e5dbf7b1cd778beb447b2c175d07c20d20616a2dad8d74607db2665393387edda0e534a7da8362
7
+ data.tar.gz: 6e3cd9935e8c64b390877240a18175d8b1dd0ea3df34300658ebe66df22d728e0a45beba10ac8cbc63f9159d5a725abedcd4f62cdd7b244bd5c25e17a812153a
@@ -0,0 +1,26 @@
1
+ name: Testing on Ubuntu
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
+ os:
13
+ - ubuntu-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
@@ -0,0 +1,26 @@
1
+ name: Testing on Windows
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
+ os:
13
+ - windows-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
data/README.md CHANGED
@@ -16,10 +16,18 @@
16
16
 
17
17
  ## Installation
18
18
 
19
+ ### For Fluentd
20
+
19
21
  ```sh
20
22
  gem install fluent-plugin-cloudwatch-logs
21
23
  ```
22
24
 
25
+ ### For td-agent
26
+
27
+ ```sh
28
+ td-agent-gem install fluent-plugin-cloudwatch-logs
29
+ ```
30
+
23
31
  ## Preparation
24
32
 
25
33
  Create IAM user with a policy like the following:
@@ -444,10 +452,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
444
452
 
445
453
  * out_cloudwatch_logs
446
454
  * if the data is too big for API, split into multiple requests
447
- * format
448
455
  * check data size
449
456
  * in_cloudwatch_logs
450
- * format
451
457
  * fallback to start_time because next_token expires after 24 hours
452
458
 
453
459
  ## Contributing
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
25
25
  spec.add_development_dependency "test-unit"
26
26
  spec.add_development_dependency "test-unit-rr"
27
27
  spec.add_development_dependency "mocha"
28
+ spec.add_development_dependency "nokogiri"
28
29
  end
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.13.0"
5
+ VERSION = "0.14.0"
6
6
  end
7
7
  end
8
8
  end
@@ -17,6 +17,9 @@ module Fluent::Plugin
17
17
  config_param :aws_use_sts, :bool, default: false
18
18
  config_param :aws_sts_role_arn, :string, default: nil
19
19
  config_param :aws_sts_session_name, :string, default: 'fluentd'
20
+ config_param :aws_sts_external_id, :string, default: nil
21
+ config_param :aws_sts_policy, :string, default: nil
22
+ config_param :aws_sts_duration_seconds, :time, default: nil
20
23
  config_param :aws_sts_endpoint_url, :string, default: nil
21
24
  config_param :region, :string, default: nil
22
25
  config_param :endpoint, :string, default: nil
@@ -88,7 +91,10 @@ module Fluent::Plugin
88
91
  Aws.config[:region] = options[:region]
89
92
  credentials_options = {
90
93
  role_arn: @aws_sts_role_arn,
91
- role_session_name: @aws_sts_session_name
94
+ role_session_name: @aws_sts_session_name,
95
+ external_id: @aws_sts_external_id,
96
+ policy: @aws_sts_policy,
97
+ duration_seconds: @aws_sts_duration_seconds
92
98
  }
93
99
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
94
100
  if @region and @aws_sts_endpoint_url
@@ -185,7 +191,7 @@ module Fluent::Plugin
185
191
  if @use_log_stream_name_prefix || @use_todays_log_stream
186
192
  log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
187
193
  begin
188
- log_streams = describe_log_streams(log_stream_name_prefix)
194
+ log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
189
195
  log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
190
196
  log_streams.each do |log_stream|
191
197
  log_stream_name = log_stream.log_stream_name
@@ -193,7 +199,7 @@ module Fluent::Plugin
193
199
  metadata = if @include_metadata
194
200
  {
195
201
  "log_stream_name" => log_stream_name,
196
- "log_group_name" => @log_group_name
202
+ "log_group_name" => log_group_name
197
203
  }
198
204
  else
199
205
  {}
@@ -221,8 +227,8 @@ module Fluent::Plugin
221
227
  end
222
228
  end
223
229
  end
224
- sleep 1
225
230
  end
231
+ sleep 1
226
232
  end
227
233
  end
228
234
 
@@ -266,8 +272,13 @@ module Fluent::Plugin
266
272
  }
267
273
  request.merge!(start_time: @start_time) if @start_time
268
274
  request.merge!(end_time: @end_time) if @end_time
269
- log_next_token = next_token(log_group_name, log_stream_name)
275
+ if @use_log_group_name_prefix
276
+ log_next_token = next_token(log_stream_name, log_group_name)
277
+ else
278
+ log_next_token = next_token(log_stream_name)
279
+ end
270
280
  request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
281
+ request[:start_from_head] = true if read_from_head?(log_next_token)
271
282
  response = @logs.get_log_events(request)
272
283
  if valid_next_token(log_next_token, response.next_forward_token)
273
284
  if @use_log_group_name_prefix
@@ -281,10 +292,14 @@ module Fluent::Plugin
281
292
  end
282
293
  end
283
294
 
284
- def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
295
+ def read_from_head?(next_token)
296
+ (!next_token.nil? && !next_token.empty?) || @start_time || @end_time
297
+ end
298
+
299
+ def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
285
300
  throttling_handler('describe_log_streams') do
286
301
  request = {
287
- log_group_name: @log_group_name
302
+ log_group_name: log_group_name != nil ? log_group_name : @log_group_name
288
303
  }
289
304
  request[:next_token] = next_token if next_token
290
305
  request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
@@ -295,7 +310,7 @@ module Fluent::Plugin
295
310
  log_streams = response.log_streams
296
311
  end
297
312
  if response.next_token
298
- log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
313
+ log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token, log_group_name)
299
314
  end
300
315
  log_streams
301
316
  end
@@ -19,6 +19,9 @@ module Fluent::Plugin
19
19
  config_param :aws_use_sts, :bool, default: false
20
20
  config_param :aws_sts_role_arn, :string, default: nil
21
21
  config_param :aws_sts_session_name, :string, default: 'fluentd'
22
+ config_param :aws_sts_external_id, :string, default: nil
23
+ config_param :aws_sts_policy, :string, default: nil
24
+ config_param :aws_sts_duration_seconds, :time, default: nil
22
25
  config_param :aws_sts_endpoint_url, :string, default: nil
23
26
  config_param :region, :string, :default => nil
24
27
  config_param :endpoint, :string, :default => nil
@@ -123,7 +126,10 @@ module Fluent::Plugin
123
126
  Aws.config[:region] = options[:region]
124
127
  credentials_options = {
125
128
  role_arn: @aws_sts_role_arn,
126
- role_session_name: @aws_sts_session_name
129
+ role_session_name: @aws_sts_session_name,
130
+ external_id: @aws_sts_external_id,
131
+ policy: @aws_sts_policy,
132
+ duration_seconds: @aws_sts_duration_seconds
127
133
  }
128
134
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
129
135
  if @region and @aws_sts_endpoint_url
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-12-08 00:00:00.000000000 Z
11
+ date: 2021-05-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: nokogiri
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
111
125
  description:
112
126
  email:
113
127
  - ryota.arai@gmail.com
@@ -116,8 +130,9 @@ extensions: []
116
130
  extra_rdoc_files: []
117
131
  files:
118
132
  - ".github/workflows/issue-auto-closer.yml"
133
+ - ".github/workflows/linux.yml"
134
+ - ".github/workflows/windows.yml"
119
135
  - ".gitignore"
120
- - ".travis.yml"
121
136
  - Gemfile
122
137
  - ISSUE_TEMPLATE.md
123
138
  - LICENSE.txt
@@ -151,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
151
166
  - !ruby/object:Gem::Version
152
167
  version: '0'
153
168
  requirements: []
154
- rubygems_version: 3.1.2
169
+ rubygems_version: 3.2.15
155
170
  signing_key:
156
171
  specification_version: 4
157
172
  summary: CloudWatch Logs Plugin for Fluentd
data/.travis.yml DELETED
@@ -1,8 +0,0 @@
1
- sudo: false
2
- language: ruby
3
-
4
- rvm:
5
- - 2.6
6
- - 2.5
7
- - 2.4
8
- - 2.3