fluent-plugin-cloudwatch-logs 0.3.7.pre → 0.3.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +97 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +14 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ecc88d81a0eac8a7998557fdd380081c9f39d9ad
|
4
|
+
data.tar.gz: 315a53cdfdd9adadbb7a79a35d1c3ac73d424815
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c8b60e1d0b52b200d018dc0b0b4fcb3aa180cdf70b3b852317e0163e70e1c6da7bf4b03d572d8acc4130029b2ab17af526de0e934e19ff7c79f081d750d39e9
|
7
|
+
data.tar.gz: d3465bb052e2d526770cc2b7245f13ce51f0fe04141462b4767002c065f7adf17a14485d5c02b60b21f761fdba499e4264fc05bde20dafc28bcf994d8ddcee3e
|
data/README.md
CHANGED
@@ -121,6 +121,9 @@ Fetch sample log from CloudWatch Logs:
|
|
121
121
|
* `log_stream_name`: name of log stream to fetch logs
|
122
122
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
123
123
|
* `state_file`: file to store current state (e.g. next\_forward\_token)
|
124
|
+
* `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
|
125
|
+
* `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
|
126
|
+
* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
|
124
127
|
|
125
128
|
This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
|
126
129
|
|
@@ -150,6 +153,100 @@ $ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 t
|
|
150
153
|
|
151
154
|
- If an event message exceeds API limit (256KB), the event will be discarded.
|
152
155
|
|
156
|
+
## Cross-Account Operation
|
157
|
+
In order to have an instance of this plugin running in one AWS account to fetch logs from another account cross-account IAM authentication is required. Whilst this can be accomplished by configuring specific instances of the plugin manually with credentials for the source account in question this is not desirable for a number of reasons.
|
158
|
+
|
159
|
+
In this case IAM can be used to allow the fluentd instance in one account ("A") to ingest Cloudwatch logs from another ("B") via the following mechanic:
|
160
|
+
|
161
|
+
* plugin instance running in account "A" has an IAM instance role assigned to the underlying EC2 instance
|
162
|
+
* The IAM instance role and associated policies permit the EC2 instance to assume a role in another account
|
163
|
+
* An IAM role in account "B" and associated policies allow read access to the Cloudwatch Logs service, as appropriate.
|
164
|
+
|
165
|
+
### IAM Detail: Consuming Account "A"
|
166
|
+
|
167
|
+
* Create an IAM role `cloudwatch`
|
168
|
+
* Attach a policy to allow the role holder to assume another role (where `ACCOUNT-B` is substituted for the appropriate account number):
|
169
|
+
|
170
|
+
```
|
171
|
+
{
|
172
|
+
"Version": "2012-10-17",
|
173
|
+
"Statement": [
|
174
|
+
{
|
175
|
+
"Effect": "Allow",
|
176
|
+
"Action": [
|
177
|
+
"sts:*"
|
178
|
+
],
|
179
|
+
"Resource": [
|
180
|
+
"arn:aws:iam::ACCOUNT-B:role/fluentd"
|
181
|
+
]
|
182
|
+
}
|
183
|
+
]
|
184
|
+
}
|
185
|
+
```
|
186
|
+
|
187
|
+
* Ensure the EC2 instance on which this plugin is executing as role `cloudwatch` as its assigned IAM instance role.
|
188
|
+
|
189
|
+
### IAM Detail: Log Source Account "B"
|
190
|
+
|
191
|
+
* Create an IAM role `fluentd`
|
192
|
+
* Ensure the `fluentd` role as account "A" as a trusted entity:
|
193
|
+
|
194
|
+
```
|
195
|
+
{
|
196
|
+
"Version": "2012-10-17",
|
197
|
+
"Statement": [
|
198
|
+
{
|
199
|
+
"Effect": "Allow",
|
200
|
+
"Principal": {
|
201
|
+
"AWS": "arn:aws:iam::ACCOUNT-A:root"
|
202
|
+
},
|
203
|
+
"Action": "sts:AssumeRole"
|
204
|
+
}
|
205
|
+
]
|
206
|
+
}
|
207
|
+
```
|
208
|
+
|
209
|
+
* Attach a policy:
|
210
|
+
|
211
|
+
```
|
212
|
+
{
|
213
|
+
"Version": "2012-10-17",
|
214
|
+
"Statement": [
|
215
|
+
{
|
216
|
+
"Effect": "Allow",
|
217
|
+
"Action": [
|
218
|
+
"logs:DescribeDestinations",
|
219
|
+
"logs:DescribeExportTasks",
|
220
|
+
"logs:DescribeLogGroups",
|
221
|
+
"logs:DescribeLogStreams",
|
222
|
+
"logs:DescribeMetricFilters",
|
223
|
+
"logs:DescribeSubscriptionFilters",
|
224
|
+
"logs:FilterLogEvents",
|
225
|
+
"logs:GetLogEvents"
|
226
|
+
],
|
227
|
+
"Resource": [
|
228
|
+
"arn:aws:logs:eu-west-1:ACCOUNT-B:log-group:LOG_GROUP_NAME_FOR_CONSUMPTION:*"
|
229
|
+
]
|
230
|
+
}
|
231
|
+
]
|
232
|
+
}
|
233
|
+
```
|
234
|
+
|
235
|
+
### Configuring the plugin for STS authentication
|
236
|
+
```
|
237
|
+
<source>
|
238
|
+
type cloudwatch_logs
|
239
|
+
region us-east-1 # You must supply a region
|
240
|
+
aws_use_sts true
|
241
|
+
aws_sts_role_arn arn:aws:iam::ACCOUNT-B:role/fluentd
|
242
|
+
log_group_name LOG_GROUP_NAME_FOR_CONSUMPTION
|
243
|
+
log_stream_name SOME_PREFIX
|
244
|
+
use_log_stream_name_prefix true
|
245
|
+
state_file /path/to/state_file
|
246
|
+
format /(?<message>.+)/
|
247
|
+
</source>
|
248
|
+
```
|
249
|
+
|
153
250
|
## TODO
|
154
251
|
|
155
252
|
* out_cloudwatch_logs
|
@@ -13,6 +13,9 @@ module Fluent
|
|
13
13
|
|
14
14
|
config_param :aws_key_id, :string, :default => nil, :secret => true
|
15
15
|
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
16
|
+
config_param :aws_use_sts, :bool, default: false
|
17
|
+
config_param :aws_sts_role_arn, :string, default: nil
|
18
|
+
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
16
19
|
config_param :region, :string, :default => nil
|
17
20
|
config_param :tag, :string
|
18
21
|
config_param :log_group_name, :string
|
@@ -39,9 +42,19 @@ module Fluent
|
|
39
42
|
|
40
43
|
def start
|
41
44
|
options = {}
|
42
|
-
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
43
45
|
options[:region] = @region if @region
|
44
46
|
options[:http_proxy] = @http_proxy if @http_proxy
|
47
|
+
|
48
|
+
if @aws_use_sts
|
49
|
+
Aws.config[:region] = options[:region]
|
50
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(
|
51
|
+
role_arn: @aws_sts_role_arn,
|
52
|
+
role_session_name: @aws_sts_session_name
|
53
|
+
)
|
54
|
+
else
|
55
|
+
options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
|
56
|
+
end
|
57
|
+
|
45
58
|
@logs = Aws::CloudWatchLogs::Client.new(options)
|
46
59
|
|
47
60
|
@finished = false
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-02-
|
11
|
+
date: 2017-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -144,12 +144,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
144
144
|
version: '0'
|
145
145
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
146
146
|
requirements:
|
147
|
-
- - "
|
147
|
+
- - ">="
|
148
148
|
- !ruby/object:Gem::Version
|
149
|
-
version:
|
149
|
+
version: '0'
|
150
150
|
requirements: []
|
151
151
|
rubyforge_project:
|
152
|
-
rubygems_version: 2.
|
152
|
+
rubygems_version: 2.5.1
|
153
153
|
signing_key:
|
154
154
|
specification_version: 4
|
155
155
|
summary: CloudWatch Logs Plugin for Fluentd
|