fluent-plugin-aws-elasticsearch-service 2.0.1 → 2.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/README.md +15 -0
- data/fluent-plugin-aws-elasticsearch-service.gemspec +5 -5
- data/lib/fluent/plugin/out_aws-elasticsearch-service.rb +34 -9
- metadata +21 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cebbe8fcb3c9376c92e9095b2544753b3807c9d0b1277fff316e7a4381d4ad58
|
4
|
+
data.tar.gz: f3c64aff68c8c4b12d16d2efd9795d17969dcdffd92998d549eae67ac6873098
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b404ec67d64d23a0e1116dca4d954111e33a360c31eda7dd9541939d0404e7ba8f8c466898a73fdec17023726e3d0055bced10dcafb918fcb88e3943cf313db6
|
7
|
+
data.tar.gz: 9307b273506a941a003abc26d02abf2be7ca422a36152541d0aa86fff81a571f12d164de4fb762ca6bc5f9735742ccca8b786527afe7cdd4f4ffb669868984fe
|
data/Gemfile
CHANGED
@@ -3,6 +3,6 @@ source 'https://rubygems.org'
|
|
3
3
|
# Specify your gem's dependencies in fluent-plugin-aws-elasticsearch-service.gemspec
|
4
4
|
gemspec
|
5
5
|
|
6
|
-
gem 'fluent-plugin-elasticsearch', [">= 2.4.0", "<
|
6
|
+
gem 'fluent-plugin-elasticsearch', [">= 2.4.0", "< 5"], require: false
|
7
7
|
gem 'aws-sdk-core', '~> 3', require: false
|
8
|
-
gem 'faraday_middleware-aws-sigv4', '
|
8
|
+
gem 'faraday_middleware-aws-sigv4', '~> 0.3.0', require: false
|
data/README.md
CHANGED
@@ -92,6 +92,7 @@ Additionally, you can use an STS assumed role as the authenticating factor and i
|
|
92
92
|
region eu-west-1
|
93
93
|
assume_role_arn arn:aws:sts::ACCOUNT:role/ROLE
|
94
94
|
assume_role_session_name SESSION_ID # Defaults to fluentd if omitted
|
95
|
+
sts_credentials_region us-west-2 # Defaults to region if omitted
|
95
96
|
</endpoint>
|
96
97
|
```
|
97
98
|
|
@@ -126,6 +127,20 @@ You'll need to ensure that the environment in which the fluentd plugin runs has
|
|
126
127
|
}
|
127
128
|
```
|
128
129
|
|
130
|
+
### EKS
|
131
|
+
If you want to use IAM roles for service accounts on Amazon EKS clusters, please refer to the official documentation and specify a Service Account for your fluentd Pod.
|
132
|
+
|
133
|
+
Then, the endpoint configuration looks like:
|
134
|
+
|
135
|
+
```ruby
|
136
|
+
<endpoint>
|
137
|
+
url https://CLUSTER_ENDPOINT_URL
|
138
|
+
region eu-west-1
|
139
|
+
assume_role_arn "#{ENV['AWS_ROLE_ARN']}"
|
140
|
+
assume_role_web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
|
141
|
+
</endpoint>
|
142
|
+
```
|
143
|
+
|
129
144
|
## Troubleshooting
|
130
145
|
|
131
146
|
* "Elasticsearch::Transport::Transport::Errors::Forbidden" error="[403]" even after verifying the access keys/roles/policies.
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "fluent-plugin-aws-elasticsearch-service"
|
8
|
-
spec.version = "2.
|
8
|
+
spec.version = "2.4.1"
|
9
9
|
spec.authors = ["atomita"]
|
10
10
|
spec.email = ["sleeping.cait.sith+gh@gmail.com"]
|
11
11
|
|
@@ -20,12 +20,12 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "
|
24
|
-
spec.add_development_dependency "rake", "~>
|
23
|
+
spec.add_development_dependency "bundler", [">= 1.10", "< 3"]
|
24
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
25
25
|
spec.add_development_dependency "rspec", "~> 3.0"
|
26
26
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
27
27
|
spec.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
|
28
|
-
spec.add_runtime_dependency "fluent-plugin-elasticsearch", [">=
|
28
|
+
spec.add_runtime_dependency "fluent-plugin-elasticsearch", [">= 3.3.0", "< 6"]
|
29
29
|
spec.add_runtime_dependency "aws-sdk-core", "~> 3"
|
30
|
-
spec.add_runtime_dependency "faraday_middleware-aws-sigv4", "
|
30
|
+
spec.add_runtime_dependency "faraday_middleware-aws-sigv4", "~> 0.3.0"
|
31
31
|
end
|
@@ -15,10 +15,12 @@ module Fluent::Plugin
|
|
15
15
|
config_param :region, :string
|
16
16
|
config_param :url, :string
|
17
17
|
config_param :access_key_id, :string, :default => ""
|
18
|
-
config_param :secret_access_key, :string, :default => ""
|
18
|
+
config_param :secret_access_key, :string, :default => "", secret: true
|
19
19
|
config_param :assume_role_arn, :string, :default => nil
|
20
20
|
config_param :ecs_container_credentials_relative_uri, :string, :default => nil #Set with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable value
|
21
21
|
config_param :assume_role_session_name, :string, :default => "fluentd"
|
22
|
+
config_param :assume_role_web_identity_token_file, :string, :default => nil
|
23
|
+
config_param :sts_credentials_region, :string, :default => nil
|
22
24
|
end
|
23
25
|
|
24
26
|
# here overrides default value of reload_connections to false because
|
@@ -31,9 +33,13 @@ module Fluent::Plugin
|
|
31
33
|
#
|
32
34
|
# @override
|
33
35
|
#
|
34
|
-
def get_connection_options
|
36
|
+
def get_connection_options(con_host=nil)
|
35
37
|
raise "`endpoint` require." if @endpoint.empty?
|
36
38
|
|
39
|
+
@endpoint.map do |ep|
|
40
|
+
raise Fluent::ConfigError, "Ensure you don't have a trailing slash on the endpoint URL in your fluentd configuration." if ep[:url].end_with?("/")
|
41
|
+
end
|
42
|
+
|
37
43
|
hosts =
|
38
44
|
begin
|
39
45
|
@endpoint.map do |ep|
|
@@ -74,21 +80,31 @@ module Fluent::Plugin
|
|
74
80
|
credentials = Aws::Credentials.new opts[:access_key_id], opts[:secret_access_key]
|
75
81
|
else
|
76
82
|
if opts[:assume_role_arn].nil?
|
77
|
-
|
83
|
+
aws_container_credentials_relative_uri = opts[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
|
84
|
+
if aws_container_credentials_relative_uri.nil?
|
78
85
|
credentials = Aws::SharedCredentials.new({retries: 2}).credentials
|
79
86
|
credentials ||= Aws::InstanceProfileCredentials.new.credentials
|
80
87
|
credentials ||= Aws::ECSCredentials.new.credentials
|
81
88
|
else
|
82
89
|
credentials = Aws::ECSCredentials.new({
|
83
|
-
credential_path:
|
90
|
+
credential_path: aws_container_credentials_relative_uri
|
84
91
|
}).credentials
|
85
92
|
end
|
86
93
|
else
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
94
|
+
|
95
|
+
if opts[:assume_role_web_identity_token_file].nil?
|
96
|
+
credentials = sts_credential_provider({
|
97
|
+
role_arn: opts[:assume_role_arn],
|
98
|
+
role_session_name: opts[:assume_role_session_name],
|
99
|
+
region: sts_credentials_region(opts)
|
100
|
+
}).credentials
|
101
|
+
else
|
102
|
+
credentials = sts_web_identity_credential_provider({
|
103
|
+
role_arn: opts[:assume_role_arn],
|
104
|
+
web_identity_token_file: opts[:assume_role_web_identity_token_file],
|
105
|
+
region: sts_credentials_region(opts)
|
106
|
+
}).credentials
|
107
|
+
end
|
92
108
|
end
|
93
109
|
end
|
94
110
|
raise "No valid AWS credentials found." unless credentials.set?
|
@@ -101,11 +117,20 @@ module Fluent::Plugin
|
|
101
117
|
calback
|
102
118
|
end
|
103
119
|
|
120
|
+
def sts_credentials_region(opts)
|
121
|
+
opts[:sts_credentials_region] || opts[:region]
|
122
|
+
end
|
123
|
+
|
104
124
|
def sts_credential_provider(opts)
|
105
125
|
# AssumeRoleCredentials is an auto-refreshing credential provider
|
106
126
|
@sts ||= Aws::AssumeRoleCredentials.new(opts)
|
107
127
|
end
|
108
128
|
|
129
|
+
def sts_web_identity_credential_provider(opts)
|
130
|
+
# AssumeRoleWebIdentityCredentials is an auto-refreshing credential provider
|
131
|
+
@sts ||= Aws::AssumeRoleWebIdentityCredentials.new(opts)
|
132
|
+
end
|
133
|
+
|
109
134
|
end
|
110
135
|
|
111
136
|
|
metadata
CHANGED
@@ -1,43 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-aws-elasticsearch-service
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- atomita
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.10'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3'
|
20
23
|
type: :development
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '1.10'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: rake
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
39
|
+
version: '13.0'
|
34
40
|
type: :development
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
46
|
+
version: '13.0'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: rspec
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,20 +98,20 @@ dependencies:
|
|
92
98
|
requirements:
|
93
99
|
- - ">="
|
94
100
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
101
|
+
version: 3.3.0
|
96
102
|
- - "<"
|
97
103
|
- !ruby/object:Gem::Version
|
98
|
-
version:
|
104
|
+
version: '6'
|
99
105
|
type: :runtime
|
100
106
|
prerelease: false
|
101
107
|
version_requirements: !ruby/object:Gem::Requirement
|
102
108
|
requirements:
|
103
109
|
- - ">="
|
104
110
|
- !ruby/object:Gem::Version
|
105
|
-
version:
|
111
|
+
version: 3.3.0
|
106
112
|
- - "<"
|
107
113
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
114
|
+
version: '6'
|
109
115
|
- !ruby/object:Gem::Dependency
|
110
116
|
name: aws-sdk-core
|
111
117
|
requirement: !ruby/object:Gem::Requirement
|
@@ -124,20 +130,14 @@ dependencies:
|
|
124
130
|
name: faraday_middleware-aws-sigv4
|
125
131
|
requirement: !ruby/object:Gem::Requirement
|
126
132
|
requirements:
|
127
|
-
- - "
|
128
|
-
- !ruby/object:Gem::Version
|
129
|
-
version: 0.2.4
|
130
|
-
- - "<"
|
133
|
+
- - "~>"
|
131
134
|
- !ruby/object:Gem::Version
|
132
135
|
version: 0.3.0
|
133
136
|
type: :runtime
|
134
137
|
prerelease: false
|
135
138
|
version_requirements: !ruby/object:Gem::Requirement
|
136
139
|
requirements:
|
137
|
-
- - "
|
138
|
-
- !ruby/object:Gem::Version
|
139
|
-
version: 0.2.4
|
140
|
-
- - "<"
|
140
|
+
- - "~>"
|
141
141
|
- !ruby/object:Gem::Version
|
142
142
|
version: 0.3.0
|
143
143
|
description: this is a Output plugin. Post to "Amazon Elasticsearch Service".
|
@@ -159,7 +159,7 @@ homepage: https://github.com/atomita/fluent-plugin-aws-elasticsearch-service
|
|
159
159
|
licenses:
|
160
160
|
- MIT
|
161
161
|
metadata: {}
|
162
|
-
post_install_message:
|
162
|
+
post_install_message:
|
163
163
|
rdoc_options: []
|
164
164
|
require_paths:
|
165
165
|
- lib
|
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
175
175
|
version: '0'
|
176
176
|
requirements: []
|
177
177
|
rubygems_version: 3.0.1
|
178
|
-
signing_key:
|
178
|
+
signing_key:
|
179
179
|
specification_version: 4
|
180
180
|
summary: Output plugin to post to "Amazon Elasticsearch Service".
|
181
181
|
test_files: []
|