sf-hiera-aws 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +94 -22
- data/lib/hiera/backend/sf_hiera_aws_backend.rb +87 -0
- data/sf-hiera-aws.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b0f6bfe5a16cc429253eb75464f79de6cbbf1d3
|
4
|
+
data.tar.gz: 13f8fe61331c0a2b7399d6eb8fde29a70e07617e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7db30954f794f1a0af1ac9484eb415f7813a902689465032200a65fcb745512935e35d779675547f27f4bed146fa0fd6f2a64565db8afeaea90545bf4e1f1741
|
7
|
+
data.tar.gz: 9fe5adbb420f3e17995943c71f5bd32497f00e33932775d70f69c098f98f1a346b7b9a3fc9d90b3f0b864b1ac11fec3ba8a8d44a8168accecec2f4beee26184a
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -2,7 +2,10 @@
|
|
2
2
|
|
3
3
|
## About
|
4
4
|
|
5
|
-
This is a Hiera backend to provide access to the EC2 API for a small number of
|
5
|
+
This is a Hiera backend to provide access to the EC2 API for a small number of
|
6
|
+
resource types. Its purpose is to prevent it from ever being necessary to copy
|
7
|
+
and paste EC2, RDS, AutoScaling Instance members, and ElastiCache addresses from
|
8
|
+
the AWS console into Puppet configs anywhere.
|
6
9
|
|
7
10
|
## Usage and Setup
|
8
11
|
|
@@ -14,9 +17,12 @@ To add this backend to hiera, edit `/etc/puppet/hiera.yaml`:
|
|
14
17
|
- sf_hiera_aws
|
15
18
|
```
|
16
19
|
|
17
|
-
This plugin will attempt to use a machine's IAM role to perform AWS lookups -
|
20
|
+
This plugin will attempt to use a machine's IAM role to perform AWS lookups -
|
21
|
+
this is the recommended method of operation.
|
18
22
|
|
19
|
-
Absent an IAM role, the plugin will fall back to looking up credentials in the
|
23
|
+
Absent an IAM role, the plugin will fall back to looking up credentials in the
|
24
|
+
environment. Use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`
|
25
|
+
variables.
|
20
26
|
|
21
27
|
The IAM role will need the following permissions:
|
22
28
|
|
@@ -28,7 +34,8 @@ The IAM role will need the following permissions:
|
|
28
34
|
"Action": [
|
29
35
|
"ec2:DescribeInstances",
|
30
36
|
"rds:DescribeDBInstances",
|
31
|
-
"elasticache:DescribeCacheClusters"
|
37
|
+
"elasticache:DescribeCacheClusters",
|
38
|
+
"autoscaling:DescribeAutoScalingGroups"
|
32
39
|
],
|
33
40
|
"Effect": "Allow",
|
34
41
|
"Resource": [
|
@@ -41,8 +48,15 @@ The IAM role will need the following permissions:
|
|
41
48
|
|
42
49
|
## Configuration
|
43
50
|
|
44
|
-
The plugin expects to find a configuration file under
|
45
|
-
|
51
|
+
The plugin expects to find a configuration file under
|
52
|
+
`/etc/puppet/sf_hiera_aws.yaml`, defining how we look up named keys. The keys
|
53
|
+
at the top level of this file determine the names of the hiera keys the plugin
|
54
|
+
will provide; the configuration determines how these are looked up.
|
55
|
+
|
56
|
+
Additional configuration can be given in files under
|
57
|
+
`/etc/puppet/sf_hiera_aws.d`, which are evaluated in alphanumerical order. If a
|
58
|
+
duplicate key is encountered in files evaluated later, this will override the
|
59
|
+
earlier config.
|
46
60
|
|
47
61
|
### Example - EC2 nodes by tag
|
48
62
|
|
@@ -60,9 +74,16 @@ aws_am_search_nodes:
|
|
60
74
|
- :private_dns_name
|
61
75
|
```
|
62
76
|
|
63
|
-
The value of `return` here is also the default, and so can be omitted. You can
|
77
|
+
The value of `return` here is also the default, and so can be omitted. You can
|
78
|
+
use any of the methods listed at
|
79
|
+
http://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Instance.html to obtain other
|
80
|
+
details from the Instance object. Calls to this key will return a list of
|
81
|
+
hashes, each containing `instace_id`, `private_ip_address` and
|
82
|
+
`private_dns_name` keys.
|
64
83
|
|
65
|
-
Note that by default all EC2 instances will be returned, including stopped
|
84
|
+
Note that by default all EC2 instances will be returned, including stopped
|
85
|
+
instances. To return only the running instances, add a filter as shown in this
|
86
|
+
example.
|
66
87
|
|
67
88
|
### Example - EC2 nodes by tag, single item list
|
68
89
|
|
@@ -75,7 +96,9 @@ aws_am_search_nodes:
|
|
75
96
|
return: :private_ip_address
|
76
97
|
```
|
77
98
|
|
78
|
-
Here, we pass a single symbol to the `return` argument. In this case, we'll get
|
99
|
+
Here, we pass a single symbol to the `return` argument. In this case, we'll get
|
100
|
+
back a list of strings containing private ip addresses (rather than a list of
|
101
|
+
hashes).
|
79
102
|
|
80
103
|
|
81
104
|
|
@@ -87,9 +110,14 @@ aws_am_bullseye_rds:
|
|
87
110
|
db_instance_identifier: "%{::sf_location}-%{::sf_environment}-db"
|
88
111
|
```
|
89
112
|
|
90
|
-
Calls to `:rds_db_instance` type keys return the instance identifier, endpoint
|
91
|
-
|
92
|
-
|
113
|
+
Calls to `:rds_db_instance` type keys return the instance identifier, endpoint
|
114
|
+
address and endpoint port in a hash.
|
115
|
+
|
116
|
+
Pass a `return` key with value `:hostname` to have the hostname of the first
|
117
|
+
matching instance returned.
|
118
|
+
|
119
|
+
Pass a `return` key with value `:hostname_and_port` to have a
|
120
|
+
`"<hostname>:<port>"` string of the first matching instance returned.
|
93
121
|
|
94
122
|
### Example - ElastiCache cluster by name
|
95
123
|
|
@@ -99,9 +127,14 @@ aws_am_bullseye_redis:
|
|
99
127
|
cache_cluster_id: "%{::sf_location}-%{::sf_environment}-redis"
|
100
128
|
```
|
101
129
|
|
102
|
-
Calls to `:elasticache_cache_cluster` type keys return a list of cache nodes,
|
103
|
-
|
104
|
-
|
130
|
+
Calls to `:elasticache_cache_cluster` type keys return a list of cache nodes,
|
131
|
+
their IDs and endpoint address/ports.
|
132
|
+
|
133
|
+
Pass a `return` key with value `:hostname` to have a list of hostnames of keys
|
134
|
+
of all cache nodes matching the cache_cluster_id returned.
|
135
|
+
|
136
|
+
Pass a `return` key with value `:hostname_and_port` to have a list of
|
137
|
+
`"<hostname>:<port>"` strings returned.
|
105
138
|
|
106
139
|
### Example - ElastiCache replication group by name
|
107
140
|
|
@@ -111,13 +144,52 @@ aws_app_redis:
|
|
111
144
|
replication_group_id: "%{::sf_location}-%{::sf_environment}-redis"
|
112
145
|
```
|
113
146
|
|
114
|
-
Calls to `:elasticache_replication_group` return a list of replication groups,
|
115
|
-
|
116
|
-
|
117
|
-
Pass a `return` key with value `:
|
118
|
-
|
147
|
+
Calls to `:elasticache_replication_group` return a list of replication groups,
|
148
|
+
their primary endpoints and node group members.
|
149
|
+
|
150
|
+
Pass a `return` key with value `:primary_endpoint` to have the hostname for the
|
151
|
+
primary end point of the node group returned.
|
152
|
+
|
153
|
+
Pass a `return` key with value `:primary_endpoint_and_port` to have the hostname
|
154
|
+
and port returned as a colon-separated string.
|
155
|
+
|
156
|
+
Pass a `return` key with value `:read_endpoints` to return an array of read
|
157
|
+
endpoint hostnames, if a `replication_group_id` is specified. Returns `nil` if
|
158
|
+
`replication_group_id` is unspecified.
|
159
|
+
|
160
|
+
Pass a `return` key with value `:read_endpoints_with_ports` to return an array
|
161
|
+
of read endpoint hostnames and ports as colon delimted strongs. Returns `nil` if
|
162
|
+
`replication_group_id` is unspecified.
|
163
|
+
|
164
|
+
### Example - AutoScaling Instance members
|
165
|
+
|
166
|
+
```
|
167
|
+
---
|
168
|
+
aws_asg_group:
|
169
|
+
type: :autoscaling_group
|
170
|
+
auto_scaling_group_names: ["euwest1-test-api"]
|
171
|
+
return: :instance_details_inservice_ip
|
172
|
+
```
|
173
|
+
|
174
|
+
Calls to `:autoscaling_group` return a list of autoscaling groups and
|
175
|
+
instance-id.
|
176
|
+
|
177
|
+
Pass a `return` key with value `:instance_details_inservice_ip` to have the
|
178
|
+
instance IP's returned for any matching instances in those autoscaling groups
|
179
|
+
that are in the 'InService' state. This prevents nodes which are coming online,
|
180
|
+
or have been marked for termination as appearing in this list.
|
181
|
+
|
182
|
+
You will need to setup an ASG Lifecycle hook to put the machine into a Waiting
|
183
|
+
state for slightly more that your puppet run, e.g. 20 minutes.
|
119
184
|
|
120
185
|
## Notes
|
121
186
|
|
122
|
-
* The order in which items are returned, for example EC2 nodes matching a tag,
|
123
|
-
|
187
|
+
* The order in which items are returned, for example EC2 nodes matching a tag,
|
188
|
+
is undefined. If you are using an array of items in a configuration file
|
189
|
+
template, for example, you are advised to sort the array in the template. This
|
190
|
+
eliminates the likelihood of unnecessary configuration file changes, and the
|
191
|
+
consequential unnecessary restart of dependent services.
|
192
|
+
* By default, all EC2 instances are returned, including those in a non-running
|
193
|
+
state. To return only running instances, add a filter on
|
194
|
+
`name: instance-state-name` and `values: ['running']` as per the example
|
195
|
+
above.
|
@@ -37,6 +37,10 @@ class Hiera
|
|
37
37
|
Aws::ElastiCache::Client.new
|
38
38
|
end
|
39
39
|
|
40
|
+
def get_autoscaling_client
|
41
|
+
Aws::AutoScaling::Client.new
|
42
|
+
end
|
43
|
+
|
40
44
|
public
|
41
45
|
|
42
46
|
def initialize
|
@@ -386,6 +390,89 @@ class Hiera
|
|
386
390
|
end
|
387
391
|
|
388
392
|
end
|
393
|
+
|
394
|
+
def type_autoscaling_group(options)
|
395
|
+
autoscaling = get_autoscaling_client
|
396
|
+
|
397
|
+
if options.key? 'auto_scaling_group_names'
|
398
|
+
asgs = autoscaling.describe_auto_scaling_groups(
|
399
|
+
auto_scaling_group_names: options['auto_scaling_group_names'],
|
400
|
+
).auto_scaling_groups
|
401
|
+
else
|
402
|
+
asgs = autoscaling.describe_auto_scaling_groups.auto_scaling_groups
|
403
|
+
end
|
404
|
+
|
405
|
+
if !options.key? 'return'
|
406
|
+
return asgs.collect do |asg|
|
407
|
+
{
|
408
|
+
'auto_scaling_group_name' => asg.auto_scaling_group_name,
|
409
|
+
'launch_configuration_name' => asg.launch_configuration_name,
|
410
|
+
'load_balancer_names' => asg.load_balancer_names,
|
411
|
+
'instances' => asg.instances.collect do | instance |
|
412
|
+
{
|
413
|
+
'instance_id' => instance.instance_id,
|
414
|
+
'availability_zone' => instance.availability_zone,
|
415
|
+
'lifecycle_state' => instance.lifecycle_state,
|
416
|
+
'health_status' => instance.health_status,
|
417
|
+
'launch_configuration_name' => instance.launch_configuration_name,
|
418
|
+
'protected_from_scale_in' => instance.protected_from_scale_in
|
419
|
+
}
|
420
|
+
end
|
421
|
+
}
|
422
|
+
end
|
423
|
+
else
|
424
|
+
unless options.key? 'auto_scaling_group_names'
|
425
|
+
Hiera.warn('Requested to return hash of instances for auto scaling group, but no auto_scaling_group_names specified')
|
426
|
+
return nil
|
427
|
+
end
|
428
|
+
end
|
429
|
+
|
430
|
+
if options['return'] == :instance_details_inservice_ip
|
431
|
+
instances = []
|
432
|
+
asgs = asgs.collect do |asg|
|
433
|
+
{
|
434
|
+
'instances' => asg.instances.select{|i| i.lifecycle_state == 'InService'}.map { |instance|
|
435
|
+
{ 'instance_id' => instance.instance_id }
|
436
|
+
}
|
437
|
+
}
|
438
|
+
end
|
439
|
+
|
440
|
+
asgs.each do |asg|
|
441
|
+
instances += asg['instances']
|
442
|
+
end
|
443
|
+
|
444
|
+
ec2_options = {
|
445
|
+
'filters' => [{
|
446
|
+
'name' => 'instance-id',
|
447
|
+
'values' => instances.map { |i| i['instance_id'] }
|
448
|
+
}],
|
449
|
+
'return' => [ 'private_ip_address']
|
450
|
+
}
|
451
|
+
return type_ec2_instance(ec2_options)
|
452
|
+
else
|
453
|
+
asgs.collect do |a|
|
454
|
+
if options['return'].is_a?(Array)
|
455
|
+
|
456
|
+
# If the 'return' option is a list, we treat these
|
457
|
+
# as a list of desired hash keys, and return a hash
|
458
|
+
# containing only those keys from the API call
|
459
|
+
|
460
|
+
Hash[options['return'].map do |f|
|
461
|
+
[f.to_s, a.key?(f) ? a[f] : nil]
|
462
|
+
end]
|
463
|
+
|
464
|
+
elsif options['return'].is_a?(Symbol)
|
465
|
+
|
466
|
+
# If the 'return' option is a symbol, we treat that
|
467
|
+
# as the one hash key we care about, and return a list
|
468
|
+
# of that.
|
469
|
+
|
470
|
+
a.key?(options['return']) ? a[options['return']] : nil
|
471
|
+
|
472
|
+
end
|
473
|
+
end
|
474
|
+
end
|
475
|
+
end
|
389
476
|
end
|
390
477
|
end
|
391
478
|
end
|
data/sf-hiera-aws.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = 'sf-hiera-aws'
|
7
|
-
spec.version = '0.0.
|
7
|
+
spec.version = '0.0.9'
|
8
8
|
spec.authors = ['Jon Topper','Mike Griffiths','Jack Thomas']
|
9
9
|
spec.email = ['jon@scalefactory.com','mike@scalefactory.com','jack@scalefactory.com']
|
10
10
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sf-hiera-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jon Topper
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-
|
13
|
+
date: 2017-02-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|