fluent-plugin-cloudwatch-logs 0.13.2 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 51935a06f74f11a53f106e38e4f08132a7342d6509dc38c0d468d54e19e5fa28
4
- data.tar.gz: 738e36cb222adca31a2f79e56ec71cc691f26468eb8025bb9d61773d0dd9d633
3
+ metadata.gz: 9e5c502cb8ad443406678f2c8ce7f6c8ea031c75f18f502d2b85ee1401e42faf
4
+ data.tar.gz: 3a06cf5ea24ef24bab2078ed216930508a2bd51940d659ff104c2de1515072db
5
5
  SHA512:
6
- metadata.gz: 06675b2f4beedcf0d39c79bf93c59586c0d11816b97a22ee3f472fa6fc892e666c96f8600cfc8f98fba3b2b82f40dd452ec9b451caf6170da199fcf71ad42bff
7
- data.tar.gz: 294a5c7fe16c450a58478b982cba8d174d603885234619b5631c1da314b5b7b8a0874424d2b5c997a4790333f99739ecac3cd47a4af97836bf15c9da90ed25af
6
+ metadata.gz: 2e5e8033091ea7f2f461095271b64ede8b5d807c4a7215ba403b25c7f0028bdc0d68ad4a2ed3171f9ea50b9997902e52140bdd6587c8d67af1a925f7dc5be726
7
+ data.tar.gz: 21d9c62b75f66b6fb8e5b65915e91dd26696612e8439ea3ffd65c2b2fff7fd169556e47b8d451874d0134c97d5523c5d8afe5f1f052bc016b08c6ad147cecd8f
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
12
  os:
13
13
  - ubuntu-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6', '2.7' ]
11
+ ruby: [ '2.5', '2.6', '2.7', '3.0' ]
12
12
  os:
13
13
  - windows-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
data/README.md CHANGED
@@ -16,10 +16,18 @@
16
16
 
17
17
  ## Installation
18
18
 
19
+ ### For Fluentd
20
+
19
21
  ```sh
20
22
  gem install fluent-plugin-cloudwatch-logs
21
23
  ```
22
24
 
25
+ ### For td-agent
26
+
27
+ ```sh
28
+ td-agent-gem install fluent-plugin-cloudwatch-logs
29
+ ```
30
+
23
31
  ## Preparation
24
32
 
25
33
  Create IAM user with a policy like the following:
@@ -98,7 +106,7 @@ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
98
106
  export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
99
107
  ```
100
108
 
101
- Note: For this to work persistently the enviornment will need to be set in the startup scripts or docker variables.
109
+ Note: For this to work persistently the environment will need to be set in the startup scripts or docker variables.
102
110
 
103
111
  ### AWS Configuration
104
112
 
@@ -176,6 +184,7 @@ Fetch sample log from CloudWatch Logs:
176
184
  * `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information.
177
185
  * `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
178
186
  * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
187
+ * `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
179
188
  * `http_proxy`: use to set an optional HTTP proxy
180
189
  * `include_time_key`: include time key as part of the log entry (defaults to UTC)
181
190
  * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -255,6 +264,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
255
264
  * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
256
265
  * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
257
266
  * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
267
+ * `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
258
268
  * `fetch_interval`: time period in seconds between checking CloudWatch for new logs. (default: 60)
259
269
  * `http_proxy`: use to set an optional HTTP proxy
260
270
  * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -444,10 +454,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
444
454
 
445
455
  * out_cloudwatch_logs
446
456
  * if the data is too big for API, split into multiple requests
447
- * format
448
457
  * check data size
449
458
  * in_cloudwatch_logs
450
- * format
451
459
  * fallback to start_time because next_token expires after 24 hours
452
460
 
453
461
  ## Contributing
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
25
25
  spec.add_development_dependency "test-unit"
26
26
  spec.add_development_dependency "test-unit-rr"
27
27
  spec.add_development_dependency "mocha"
28
+ spec.add_development_dependency "nokogiri"
28
29
  end
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.13.2"
5
+ VERSION = "0.14.1"
6
6
  end
7
7
  end
8
8
  end
@@ -17,9 +17,13 @@ module Fluent::Plugin
17
17
  config_param :aws_use_sts, :bool, default: false
18
18
  config_param :aws_sts_role_arn, :string, default: nil
19
19
  config_param :aws_sts_session_name, :string, default: 'fluentd'
20
+ config_param :aws_sts_external_id, :string, default: nil
21
+ config_param :aws_sts_policy, :string, default: nil
22
+ config_param :aws_sts_duration_seconds, :time, default: nil
20
23
  config_param :aws_sts_endpoint_url, :string, default: nil
21
24
  config_param :region, :string, default: nil
22
25
  config_param :endpoint, :string, default: nil
26
+ config_param :ssl_verify_peer, :bool, :default => true
23
27
  config_param :tag, :string
24
28
  config_param :log_group_name, :string
25
29
  config_param :add_log_group_name, :bool, default: false
@@ -28,7 +32,7 @@ module Fluent::Plugin
28
32
  config_param :log_stream_name, :string, default: nil
29
33
  config_param :use_log_stream_name_prefix, :bool, default: false
30
34
  config_param :state_file, :string, default: nil,
31
- deprecated: "Use <stroage> instead."
35
+ deprecated: "Use <storage> instead."
32
36
  config_param :fetch_interval, :time, default: 60
33
37
  config_param :http_proxy, :string, default: nil
34
38
  config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -82,13 +86,17 @@ module Fluent::Plugin
82
86
  options = {}
83
87
  options[:region] = @region if @region
84
88
  options[:endpoint] = @endpoint if @endpoint
89
+ options[:ssl_verify_peer] = @ssl_verify_peer
85
90
  options[:http_proxy] = @http_proxy if @http_proxy
86
91
 
87
92
  if @aws_use_sts
88
93
  Aws.config[:region] = options[:region]
89
94
  credentials_options = {
90
95
  role_arn: @aws_sts_role_arn,
91
- role_session_name: @aws_sts_session_name
96
+ role_session_name: @aws_sts_session_name,
97
+ external_id: @aws_sts_external_id,
98
+ policy: @aws_sts_policy,
99
+ duration_seconds: @aws_sts_duration_seconds
92
100
  }
93
101
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
94
102
  if @region and @aws_sts_endpoint_url
@@ -221,8 +229,8 @@ module Fluent::Plugin
221
229
  end
222
230
  end
223
231
  end
224
- sleep 1
225
232
  end
233
+ sleep 1
226
234
  end
227
235
  end
228
236
 
@@ -266,8 +274,13 @@ module Fluent::Plugin
266
274
  }
267
275
  request.merge!(start_time: @start_time) if @start_time
268
276
  request.merge!(end_time: @end_time) if @end_time
269
- log_next_token = next_token(log_group_name, log_stream_name)
277
+ if @use_log_group_name_prefix
278
+ log_next_token = next_token(log_stream_name, log_group_name)
279
+ else
280
+ log_next_token = next_token(log_stream_name)
281
+ end
270
282
  request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
283
+ request[:start_from_head] = true if read_from_head?(log_next_token)
271
284
  response = @logs.get_log_events(request)
272
285
  if valid_next_token(log_next_token, response.next_forward_token)
273
286
  if @use_log_group_name_prefix
@@ -281,6 +294,10 @@ module Fluent::Plugin
281
294
  end
282
295
  end
283
296
 
297
+ def read_from_head?(next_token)
298
+ (!next_token.nil? && !next_token.empty?) || @start_time || @end_time
299
+ end
300
+
284
301
  def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
285
302
  throttling_handler('describe_log_streams') do
286
303
  request = {
@@ -19,9 +19,13 @@ module Fluent::Plugin
19
19
  config_param :aws_use_sts, :bool, default: false
20
20
  config_param :aws_sts_role_arn, :string, default: nil
21
21
  config_param :aws_sts_session_name, :string, default: 'fluentd'
22
+ config_param :aws_sts_external_id, :string, default: nil
23
+ config_param :aws_sts_policy, :string, default: nil
24
+ config_param :aws_sts_duration_seconds, :time, default: nil
22
25
  config_param :aws_sts_endpoint_url, :string, default: nil
23
26
  config_param :region, :string, :default => nil
24
27
  config_param :endpoint, :string, :default => nil
28
+ config_param :ssl_verify_peer, :bool, :default => true
25
29
  config_param :log_group_name, :string, :default => nil
26
30
  config_param :log_stream_name, :string, :default => nil
27
31
  config_param :auto_create_stream, :bool, default: false
@@ -117,13 +121,17 @@ module Fluent::Plugin
117
121
  options[:log_level] = :debug if log
118
122
  options[:region] = @region if @region
119
123
  options[:endpoint] = @endpoint if @endpoint
124
+ options[:ssl_verify_peer] = @ssl_verify_peer
120
125
  options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
121
126
 
122
127
  if @aws_use_sts
123
128
  Aws.config[:region] = options[:region]
124
129
  credentials_options = {
125
130
  role_arn: @aws_sts_role_arn,
126
- role_session_name: @aws_sts_session_name
131
+ role_session_name: @aws_sts_session_name,
132
+ external_id: @aws_sts_external_id,
133
+ policy: @aws_sts_policy,
134
+ duration_seconds: @aws_sts_duration_seconds
127
135
  }
128
136
  credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
129
137
  if @region and @aws_sts_endpoint_url
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.2
4
+ version: 0.14.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-01-18 00:00:00.000000000 Z
11
+ date: 2021-09-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -108,7 +108,21 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
- description:
111
+ - !ruby/object:Gem::Dependency
112
+ name: nokogiri
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ description:
112
126
  email:
113
127
  - ryota.arai@gmail.com
114
128
  executables: []
@@ -119,7 +133,6 @@ files:
119
133
  - ".github/workflows/linux.yml"
120
134
  - ".github/workflows/windows.yml"
121
135
  - ".gitignore"
122
- - ".travis.yml"
123
136
  - Gemfile
124
137
  - ISSUE_TEMPLATE.md
125
138
  - LICENSE.txt
@@ -138,7 +151,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
138
151
  licenses:
139
152
  - MIT
140
153
  metadata: {}
141
- post_install_message:
154
+ post_install_message:
142
155
  rdoc_options: []
143
156
  require_paths:
144
157
  - lib
@@ -153,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
153
166
  - !ruby/object:Gem::Version
154
167
  version: '0'
155
168
  requirements: []
156
- rubygems_version: 3.1.2
157
- signing_key:
169
+ rubygems_version: 3.2.5
170
+ signing_key:
158
171
  specification_version: 4
159
172
  summary: CloudWatch Logs Plugin for Fluentd
160
173
  test_files:
data/.travis.yml DELETED
@@ -1,8 +0,0 @@
1
- sudo: false
2
- language: ruby
3
-
4
- rvm:
5
- - 2.6
6
- - 2.5
7
- - 2.4
8
- - 2.3