fluent-plugin-cloudwatch-logs 0.13.3 → 0.14.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b5c8b0cb261724720a72342f7c9b5a51dffd93c3deab87b24ca17dec5a342385
4
- data.tar.gz: 85698ef2c6b1d51b507d3d1f27f7990e456b0aa76980c2c9520d4babbbec5fae
3
+ metadata.gz: c43fba0083c45e47060ad0aa61d01acbd7d3b5a114e8e96864075a84c5a18397
4
+ data.tar.gz: 7fb5cce4a8749fd19b53378f8ab0df8b59d233f9491239a2cd98cca704109eff
5
5
  SHA512:
6
- metadata.gz: e8186340af3335ec492110059280fb6bdfdff9c90982b8ec27a7c923cc08fc95d9babbf525ea2f558adfd975f8c5bbff2d53e7e4c00177eaf2d9d8c7711c0aa1
7
- data.tar.gz: ab3172fffdc73a7ccf307a0b0482cd15f017c30fb2a775a69da657bdaa659a5e3ccee5df4bad85666802dce5b8a172539f37edd8cbca149b22073755c2fae050
6
+ metadata.gz: 5854c662f12a1f68068b99f804824547df7c5a6220bb5bf54a29a706422e1bb50625f36fe5f0b51f95f226f887bc28aadfca86d779817b50a13afba1cbf1744c
7
+ data.tar.gz: 83cd966492bbfc1d7ac96d88088628f4b80148bb72dd21a45cca4cf1f2f14cc98ed097de0be07979812220405989c5f07f34d38cb34d490779633bf06807e505
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
12
  os:
13
13
  - ubuntu-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
12
  os:
13
13
  - windows-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
data/README.md CHANGED
@@ -16,10 +16,18 @@
16
16
 
17
17
  ## Installation
18
18
 
19
+ ### For Fluentd
20
+
19
21
  ```sh
20
22
  gem install fluent-plugin-cloudwatch-logs
21
23
  ```
22
24
 
25
+ ### For td-agent
26
+
27
+ ```sh
28
+ td-agent-gem install fluent-plugin-cloudwatch-logs
29
+ ```
30
+
23
31
  ## Preparation
24
32
 
25
33
  Create IAM user with a policy like the following:
@@ -98,7 +106,7 @@ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
98
106
  export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
99
107
  ```
100
108
 
101
- Note: For this to work persistently the enviornment will need to be set in the startup scripts or docker variables.
109
+ Note: For this to work persistently the environment will need to be set in the startup scripts or docker variables.
102
110
 
103
111
  ### AWS Configuration
104
112
 
@@ -176,6 +184,7 @@ Fetch sample log from CloudWatch Logs:
176
184
  * `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information.
177
185
  * `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
178
186
  * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
187
+ * `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
179
188
  * `http_proxy`: use to set an optional HTTP proxy
180
189
  * `include_time_key`: include time key as part of the log entry (defaults to UTC)
181
190
  * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -255,6 +264,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
255
264
  * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
256
265
  * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
257
266
  * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
267
+ * `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
258
268
  * `fetch_interval`: time period in seconds between checking CloudWatch for new logs. (default: 60)
259
269
  * `http_proxy`: use to set an optional HTTP proxy
260
270
  * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -444,10 +454,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
444
454
 
445
455
  * out_cloudwatch_logs
446
456
  * if the data is too big for API, split into multiple requests
447
- * format
448
457
  * check data size
449
458
  * in_cloudwatch_logs
450
- * format
451
459
  * fallback to start_time because next_token expires after 24 hours
452
460
 
453
461
  ## Contributing
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
25
25
  spec.add_development_dependency "test-unit"
26
26
  spec.add_development_dependency "test-unit-rr"
27
27
  spec.add_development_dependency "mocha"
28
+ spec.add_development_dependency "nokogiri"
28
29
  end
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.13.3"
5
+ VERSION = "0.14.2"
6
6
  end
7
7
  end
8
8
  end
@@ -17,9 +17,14 @@ module Fluent::Plugin
17
17
  config_param :aws_use_sts, :bool, default: false
18
18
  config_param :aws_sts_role_arn, :string, default: nil
19
19
  config_param :aws_sts_session_name, :string, default: 'fluentd'
20
+ config_param :aws_sts_external_id, :string, default: nil
21
+ config_param :aws_sts_policy, :string, default: nil
22
+ config_param :aws_sts_duration_seconds, :time, default: nil
20
23
  config_param :aws_sts_endpoint_url, :string, default: nil
24
+ config_param :aws_ecs_authentication, :bool, default: false
21
25
  config_param :region, :string, default: nil
22
26
  config_param :endpoint, :string, default: nil
27
+ config_param :ssl_verify_peer, :bool, :default => true
23
28
  config_param :tag, :string
24
29
  config_param :log_group_name, :string
25
30
  config_param :add_log_group_name, :bool, default: false
@@ -28,7 +33,7 @@ module Fluent::Plugin
28
33
  config_param :log_stream_name, :string, default: nil
29
34
  config_param :use_log_stream_name_prefix, :bool, default: false
30
35
  config_param :state_file, :string, default: nil,
31
- deprecated: "Use <stroage> instead."
36
+ deprecated: "Use <storage> instead."
32
37
  config_param :fetch_interval, :time, default: 60
33
38
  config_param :http_proxy, :string, default: nil
34
39
  config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -82,13 +87,17 @@ module Fluent::Plugin
82
87
  options = {}
83
88
  options[:region] = @region if @region
84
89
  options[:endpoint] = @endpoint if @endpoint
90
+ options[:ssl_verify_peer] = @ssl_verify_peer
85
91
  options[:http_proxy] = @http_proxy if @http_proxy
86
92
 
87
93
  if @aws_use_sts
88
94
  Aws.config[:region] = options[:region]
89
95
  credentials_options = {
90
96
  role_arn: @aws_sts_role_arn,
91
- role_session_name: @aws_sts_session_name
97
+ role_session_name: @aws_sts_session_name,
98
+ external_id: @aws_sts_external_id,
99
+ policy: @aws_sts_policy,
100
+ duration_seconds: @aws_sts_duration_seconds
92
101
  }
93
102
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
94
103
  if @region and @aws_sts_endpoint_url
@@ -109,6 +118,10 @@ module Fluent::Plugin
109
118
  credentials_options[:client] = Aws::STS::Client.new(:region => @region)
110
119
  end
111
120
  options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
121
+ elsif @aws_ecs_authentication
122
+ # collect AWS credential from ECS relative uri ENV variable
123
+ aws_container_credentials_relative_uri = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
124
+ options[:credentials] = Aws::ECSCredentials.new({credential_path: aws_container_credentials_relative_uri}).credentials
112
125
  else
113
126
  options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
114
127
  end
@@ -221,8 +234,8 @@ module Fluent::Plugin
221
234
  end
222
235
  end
223
236
  end
224
- sleep 1
225
237
  end
238
+ sleep 1
226
239
  end
227
240
  end
228
241
 
@@ -272,6 +285,7 @@ module Fluent::Plugin
272
285
  log_next_token = next_token(log_stream_name)
273
286
  end
274
287
  request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
288
+ request[:start_from_head] = true if read_from_head?(log_next_token)
275
289
  response = @logs.get_log_events(request)
276
290
  if valid_next_token(log_next_token, response.next_forward_token)
277
291
  if @use_log_group_name_prefix
@@ -285,6 +299,10 @@ module Fluent::Plugin
285
299
  end
286
300
  end
287
301
 
302
+ def read_from_head?(next_token)
303
+ (!next_token.nil? && !next_token.empty?) || @start_time || @end_time
304
+ end
305
+
288
306
  def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
289
307
  throttling_handler('describe_log_streams') do
290
308
  request = {
@@ -19,9 +19,14 @@ module Fluent::Plugin
19
19
  config_param :aws_use_sts, :bool, default: false
20
20
  config_param :aws_sts_role_arn, :string, default: nil
21
21
  config_param :aws_sts_session_name, :string, default: 'fluentd'
22
+ config_param :aws_sts_external_id, :string, default: nil
23
+ config_param :aws_sts_policy, :string, default: nil
24
+ config_param :aws_sts_duration_seconds, :time, default: nil
22
25
  config_param :aws_sts_endpoint_url, :string, default: nil
26
+ config_param :aws_ecs_authentication, :bool, default: false
23
27
  config_param :region, :string, :default => nil
24
28
  config_param :endpoint, :string, :default => nil
29
+ config_param :ssl_verify_peer, :bool, :default => true
25
30
  config_param :log_group_name, :string, :default => nil
26
31
  config_param :log_stream_name, :string, :default => nil
27
32
  config_param :auto_create_stream, :bool, default: false
@@ -117,13 +122,17 @@ module Fluent::Plugin
117
122
  options[:log_level] = :debug if log
118
123
  options[:region] = @region if @region
119
124
  options[:endpoint] = @endpoint if @endpoint
125
+ options[:ssl_verify_peer] = @ssl_verify_peer
120
126
  options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
121
127
 
122
128
  if @aws_use_sts
123
129
  Aws.config[:region] = options[:region]
124
130
  credentials_options = {
125
131
  role_arn: @aws_sts_role_arn,
126
- role_session_name: @aws_sts_session_name
132
+ role_session_name: @aws_sts_session_name,
133
+ external_id: @aws_sts_external_id,
134
+ policy: @aws_sts_policy,
135
+ duration_seconds: @aws_sts_duration_seconds
127
136
  }
128
137
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
129
138
  if @region and @aws_sts_endpoint_url
@@ -144,6 +153,10 @@ module Fluent::Plugin
144
153
  credentials_options[:client] = Aws::STS::Client.new(:region => @region)
145
154
  end
146
155
  options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
156
+ elsif @aws_ecs_authentication
157
+ # collect AWS credential from ECS relative uri ENV variable
158
+ aws_container_credentials_relative_uri = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
159
+ options[:credentials] = Aws::ECSCredentials.new({credential_path: aws_container_credentials_relative_uri}).credentials
147
160
  else
148
161
  options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
149
162
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.3
4
+ version: 0.14.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-10 00:00:00.000000000 Z
11
+ date: 2021-09-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: nokogiri
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
111
125
  description:
112
126
  email:
113
127
  - ryota.arai@gmail.com
@@ -119,7 +133,6 @@ files:
119
133
  - ".github/workflows/linux.yml"
120
134
  - ".github/workflows/windows.yml"
121
135
  - ".gitignore"
122
- - ".travis.yml"
123
136
  - Gemfile
124
137
  - ISSUE_TEMPLATE.md
125
138
  - LICENSE.txt
@@ -153,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
153
166
  - !ruby/object:Gem::Version
154
167
  version: '0'
155
168
  requirements: []
156
- rubygems_version: 3.1.2
169
+ rubygems_version: 3.2.15
157
170
  signing_key:
158
171
  specification_version: 4
159
172
  summary: CloudWatch Logs Plugin for Fluentd
data/.travis.yml DELETED
@@ -1,8 +0,0 @@
1
- sudo: false
2
- language: ruby
3
-
4
- rvm:
5
- - 2.6
6
- - 2.5
7
- - 2.4
8
- - 2.3