awsdsl 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,360 @@
1
+ require 'cfndsl'
2
+ require 'netaddr'
3
+ require 'awsdsl/cfn_helpers'
4
+
5
+ module AWSDSL
6
+ class CfnBuilder
7
+ include CfnHelpers
8
+
9
+ def initialize(stack)
10
+ @stack = stack
11
+ end
12
+
13
+ def self.build(stack)
14
+ CfnBuilder.new(stack).build
15
+ end
16
+
17
+ def build
18
+ @t = CfnDsl::CloudFormationTemplate.new
19
+ stack = @stack
20
+ @t.declare do
21
+ Description stack.description
22
+ end
23
+ AWS.memoize do
24
+ build_vpcs
25
+ build_elasticaches
26
+ build_roles
27
+ end
28
+ @t
29
+ end
30
+
31
+ def build_roles
32
+ stack = @stack
33
+ stack.roles.each do |role|
34
+ role_name = role.name.capitalize
35
+ role_vpc = resolve_vpc(role.vpc)
36
+
37
+ # Create ELBs and appropriate security groups etc.
38
+ role.load_balancers.each do |lb|
39
+ listeners = lb.listeners.map { |l| format_listener(l) }
40
+ health_check = health_check_defaults(lb.health_check) if lb.health_check
41
+
42
+ lb_name = "#{role_name}#{lb.name.capitalize}ELB"
43
+ subnets = lb.subnets.empty? ? role.subnets : lb.subnets
44
+ lb_subnets = resolve_subnets(role.vpc, subnets)
45
+
46
+ # ELB
47
+ @t.declare do
48
+ LoadBalancer lb_name do
49
+ Listeners listeners
50
+ ConnectionSettings lb.connection_settings if lb.connection_settings
51
+ HealthCheck health_check if health_check
52
+ CrossZone true
53
+ Subnets lb_subnets
54
+ SecurityGroups [Ref("#{lb_name}SG")]
55
+ end
56
+ end
57
+
58
+ # ELB SG
59
+ @t.declare do
60
+ EC2_SecurityGroup "#{lb_name}SG" do
61
+ GroupDescription "#{lb.name.capitalize} ELB Security Group"
62
+ VpcId role_vpc
63
+ listeners.map { |l| l[:LoadBalancerPort] }.each do |port|
64
+ SecurityGroupIngress IpProtocol: 'tcp',
65
+ FromPort: port,
66
+ ToPort: port,
67
+ CidrIp: '0.0.0.0/0'
68
+ end
69
+ end
70
+ end
71
+
72
+ # ELB DNS records
73
+ lb.dns_records.each do |record|
74
+ zone = record[:zone] || get_zone_for_record(record[:name]).id
75
+ record_name = record[:name].split('.').map(&:capitalize).join
76
+ @t.declare do
77
+ RecordSet record_name do
78
+ HostedZoneId zone
79
+ Name record
80
+ Type 'A'
81
+ AliasTarget HostedZoneId: FnGetAtt(lb_name, 'CanonicalHostedZoneNameID'),
82
+ DNSName: FnGetAtt(lb_name, 'CanonicalHostedZoneName')
83
+ end
84
+ end
85
+ end
86
+ end # end load_balancers
87
+
88
+ # IAM Role
89
+ @t.declare do
90
+ IAM_Role "#{role_name}Role" do
91
+ AssumeRolePolicyDocument Statement: [{
92
+ Effect: 'Allow',
93
+ Principal: {
94
+ Service: ['ec2.amazonaws.com']
95
+ },
96
+ Action: ['sts:AssumeRole']
97
+ }]
98
+ Path '/'
99
+ end
100
+ end
101
+
102
+ # Policy
103
+ statements = role.policy_statements.map { |s| format_policy_statement(s) }
104
+ if statements.count > 1
105
+ policy_name = "#{role_name}Policy"
106
+ @t.declare do
107
+ Policy policy_name do
108
+ PolicyName policy_name
109
+ PolicyDocument Statement: statements
110
+ Roles [Ref("#{role_name}Role")]
111
+ end
112
+ end
113
+ end
114
+
115
+ # Instance Profile
116
+ @t.declare do
117
+ InstanceProfile "#{role_name}InstanceProfile" do
118
+ Path '/'
119
+ Roles [Ref("#{role_name}Role")]
120
+ end
121
+ end
122
+
123
+ # Autoscaling Group
124
+ update_policy = update_policy_defaults(role)
125
+ lb_names = role.load_balancers.map { |lb| "#{role_name}#{lb.name.capitalize}ELB" }
126
+ subnets = resolve_subnets(role.vpc, role.subnets)
127
+ min = role.min_size || 0
128
+ max = role.max_size || 1
129
+ tgt = role.tgt_size || 1
130
+ @t.declare do
131
+ AutoScalingGroup "#{role_name}ASG" do
132
+ LaunchConfigurationName Ref("#{role.name.capitalize}LaunchConfig")
133
+ UpdatePolicy 'AutoScalingRollingUpdate', update_policy if update_policy
134
+ MinSize min
135
+ MaxSize max
136
+ DesiredCapacity tgt
137
+ LoadBalancerNames lb_names.map { |name| Ref(name) }
138
+ VPCZoneIdentifier subnets
139
+ AvailabiltityZones FnGetAZs('')
140
+ end
141
+ end
142
+
143
+ # Launch Configuration
144
+ security_groups = resolve_security_groups(role.vpc, role.security_groups)
145
+ block_devices = format_block_devices(role.block_devices)
146
+ @t.declare do
147
+ LaunchConfiguration "#{role_name}LaunchConfig" do
148
+ ImageId role.ami
149
+ # TODO(jpg): Should support NAT at some stage even though it's nasty on AWS
150
+ AssociatePublicIpAddress true
151
+ InstanceType role.instance_type
152
+ # TODO(jpg): Need to resolve this to IDs or Refs as necessary
153
+ SecurityGroups [Ref("#{role_name}SG")] + security_groups
154
+ IamInstanceProfile Ref("#{role_name}InstanceProfile")
155
+ BlockDeviceMappings block_devices
156
+ KeyName role.key_pair if role.key_pair
157
+ end
158
+ end
159
+
160
+ lb_ingress_rules = role.load_balancers.map do |lb|
161
+ lb.listeners.map do |l|
162
+ h = listener_defaults(l)
163
+ h[:sg] = "#{role_name}#{lb.name.capitalize}ELBSG"
164
+ h
165
+ end
166
+ end.flatten
167
+ # Security Group
168
+ @t.declare do
169
+ EC2_SecurityGroup "#{role_name}SG" do
170
+ GroupDescription "#{role_name} Security Group"
171
+ VpcId role_vpc
172
+ # TODO(jpg): Better way of offering up defaults
173
+ SecurityGroupIngress IpProtocol: 'tcp',
174
+ FromPort: 22,
175
+ ToPort: 22,
176
+ CidrIp: '0.0.0.0/0'
177
+
178
+ # Access from configured load_balancers
179
+ lb_ingress_rules.each do |rule|
180
+ SecurityGroupIngress IpProtocol: 'tcp',
181
+ FromPort: rule[:instance_port],
182
+ ToPort: rule[:instance_port],
183
+ SourceSecurityGroupId: Ref(rule[:sg])
184
+ end
185
+
186
+ # Access from other roles
187
+ # TODO(jpg): catch undefined roles before template generation
188
+ role.allows.select { |r| r[:role] != role.name }.each do |rule|
189
+ ports = rule[:ports].is_a?(Array) ? rule[:ports] : [rule[:ports]]
190
+ ports.each do |port|
191
+ SecurityGroupIngress IpProtocol: rule[:proto] || 'tcp',
192
+ FromPort: port,
193
+ ToPort: port,
194
+ SourceSecurityGroupId: Ref("#{rule[:role].capitalize}SG")
195
+ end
196
+ end
197
+ end
198
+
199
+ # Intracluster communication
200
+ role.allows.select { |r| r[:role] == role.name }.each do |rule|
201
+ ports = rule[:ports].is_a?(Array) ? rule[:ports] : [rule[:ports]]
202
+ proto = rule[:proto] || 'tcp'
203
+ ports.each do |port|
204
+ EC2_SecurityGroupIngress "#{role_name}SG#{proto.upcase}#{port}" do
205
+ GroupId Ref("#{role_name}SG")
206
+ IpProtocol proto
207
+ FromPort port
208
+ ToPort port
209
+ SourceSecurityGroupId Ref("#{role_name}SG")
210
+ end
211
+ end
212
+ end
213
+ end
214
+ end
215
+ end
216
+
217
+ def build_elasticaches
218
+ default_ports = {
219
+ 'redis' => 6379,
220
+ 'memcached' => 11211
221
+ }
222
+ stack = @stack
223
+ stack.elasticaches.each do |cache|
224
+ # Default to Redis, also set default port if unset.
225
+ engine ||= 'redis'
226
+ port ||= default_ports[engine]
227
+ num_nodes ||= 1
228
+ cache_vpc = resolve_vpc(cache.vpc)
229
+ cache_name = "#{cache.name.capitalize}Cache"
230
+
231
+ # SG
232
+ @t.declare do
233
+ EC2_SecurityGroup "#{cache_name}SG" do
234
+ GroupDescription "#{cache.name.capitalize} Cache Security Group"
235
+ VpcId cache_vpc
236
+ cache.allows.each do |rule|
237
+ SecurityGroupIngress IpProtocol: 'tcp',
238
+ FromPort: port,
239
+ ToPort: port,
240
+ SourceSecurityGroupId: Ref("#{rule[:role].capitalize}SG")
241
+ end
242
+ end
243
+ end
244
+
245
+ # ElastiCacheSubnetGroup
246
+ cache_subnets = resolve_subnets(cache.vpc, cache.subnets)
247
+ @t.declare do
248
+ ElastiCache_SubnetGroup "#{cache_name}SubnetGroup" do
249
+ Description "SubnetGroup for #{cache_name}"
250
+ SubnetIds cache_subnets
251
+ end
252
+ end
253
+
254
+ # CacheCluster
255
+ @t.declare do
256
+ CacheCluster cache_name do
257
+ CacheNodeType cache.node_type
258
+ NumCacheNodes num_nodes
259
+ Engine engine
260
+ Port port
261
+ CacheSubnetGroupName Ref("#{cache_name}SubnetGroup")
262
+ VpcSecurityGroupIds [FnGetAtt("#{cache_name}SG", 'GroupId')]
263
+ end
264
+ end
265
+
266
+ # Add additional policy to each Role that can access this Cache
267
+ # This will allow said Role to discover the Cache nodes
268
+ cache.allows.each do |rule|
269
+ role = stack.roles.find { |r| r.name = rule[:role] }
270
+ role.policy_statement effect: 'Allow',
271
+ action: 'elasticache:Describe*',
272
+ resource: '*'
273
+ end
274
+ end
275
+ end
276
+
277
+ def build_vpcs
278
+ stack = @stack
279
+ stack.vpcs.each do |vpc|
280
+ igw = vpc.igw || true
281
+ dns = vpc.dns || true
282
+ cidr = vpc.cidr || '10.0.0.0/16'
283
+ subnet_bits = vpc.subnet_bits || 24
284
+ dns_hostnames = vpc.dns_hostnames || true
285
+
286
+ cidr = NetAddr::CIDR.create(cidr)
287
+ subnets = cidr.subnet(Bits: subnet_bits).to_enum
288
+
289
+ # VPC
290
+ vpc_name = "#{vpc.name.capitalize}VPC"
291
+ @t.declare do
292
+ VPC vpc_name do
293
+ CidrBlock cidr
294
+ EnableDnsSupport dns
295
+ EnableDnsHostnames dns_hostnames
296
+ end
297
+ end
298
+
299
+ if igw # Don't create internet facing stuff if igw is not enabled
300
+ igw_name = "#{vpc.name.capitalize}IGW"
301
+
302
+ # IGW
303
+ @t.declare do
304
+ InternetGateway igw_name
305
+ end
306
+
307
+ # Attach to VPC
308
+ @t.declare do
309
+ VPCGatewayAttachment "#{vpc.name.capitalize}GWAttachment" do
310
+ VpcId Ref(vpc_name)
311
+ InternetGatewayId Ref(igw_name)
312
+ end
313
+ end
314
+
315
+ # RouteTable
316
+ rt_name = "#{vpc.name.capitalize}RouteTable"
317
+ @t.declare do
318
+ RouteTable rt_name do
319
+ VpcId Ref(vpc_name)
320
+ end
321
+ end
322
+
323
+ # Default route for RouteTable
324
+ @t.declare do
325
+ Route "#{vpc.name.capitalize}DefaultRoute" do
326
+ RouteTableId Ref(rt_name)
327
+ DestinationCidrBlock '0.0.0.0/0'
328
+ GatewayId Ref(igw_name)
329
+ # TODO(jpg): DependsOn rt_name
330
+ end
331
+ end
332
+ end
333
+
334
+ vpc.subnets.each do |subnet|
335
+ subnet_igw = subnet.igw || igw
336
+ azs = subnet.azs || fetch_availability_zones(vpc.region)
337
+ subnet_name = "#{vpc.name.capitalize}#{subnet.name.capitalize}Subnet"
338
+ azs.each do |az|
339
+ subnet_name_az = "#{subnet_name}#{az.capitalize}"
340
+ @t.declare do
341
+ Subnet subnet_name_az do
342
+ AvailabilityZone "#{vpc.region}#{az}"
343
+ CidrBlock subnets.next
344
+ VpcId Ref(vpc_name)
345
+ end
346
+
347
+ if subnet_igw
348
+ SubnetRouteTableAssociation "#{subnet_name_az}DefaultRTAssoc" do
349
+ SubnetId Ref(subnet_name_az)
350
+ RouteTableId Ref(rt_name)
351
+ # TODO(jpg): DependsOn rt_name
352
+ end
353
+ end
354
+ end
355
+ end
356
+ end
357
+ end
358
+ end
359
+ end
360
+ end
@@ -0,0 +1,131 @@
1
+ require 'aws'
2
+ require 'cfndsl'
3
+
4
+ module AWSDSL
5
+ module CfnHelpers
6
+ include CfnDsl::Functions
7
+
8
+ def listener_defaults(listener)
9
+ listener[:proto] ||= 'HTTP'
10
+ listener[:instance_port] ||= listener[:port]
11
+ listener[:loadbalancer_port] ||= listener[:port]
12
+ listener
13
+ end
14
+
15
+ def format_listener(listener)
16
+ listener = listener_defaults(listener)
17
+ hash = {
18
+ LoadBalancerPort: listener[:loadbalancer_port],
19
+ InstancePort: listener[:instance_port],
20
+ Protocol: listener[:proto]
21
+ }
22
+ hash[:SSLCertificateId] = listener[:cert] if listener[:cert]
23
+ hash
24
+ end
25
+
26
+ def health_check_defaults(health_check)
27
+ hash = {
28
+ Target: health_check[:target]
29
+ }
30
+ hash[:HealthyThreshold] = health_check[:healthy_threshold] || 3
31
+ hash[:UnhealthyThreshold] = health_check[:unhealthy_threshold] || 5
32
+ hash[:Interval] = health_check[:interval] || 90
33
+ hash[:Timeout] = health_check[:timeout] || 60
34
+ hash
35
+ end
36
+
37
+ def update_policy_defaults(role)
38
+ update_policy = role.update_policy || {}
39
+ return nil if update_policy[:disable] == true
40
+ hash = {}
41
+ hash[:MaxBatchSize] = update_policy[:max_batch] || 1
42
+ hash[:MinInstancesInService] = update_policy[:min_inservice] || role.min_size
43
+ hash[:PauseTime] = update_policy[:pause_time] if update_policy[:pause_time]
44
+ end
45
+
46
+ def format_policy_statement(policy_statement)
47
+ Hash[policy_statement.map { |k, v| [k.to_s.capitalize.to_sym, v] }]
48
+ end
49
+
50
+ def format_block_devices(devices)
51
+ devices.map do |dev|
52
+ h = { DeviceName: dev[:name] }
53
+ if dev[:ephemeral]
54
+ h[:VirtualName] = "ephemeral#{dev[:ephemeral]}"
55
+ else
56
+ h[:Ebs] = {
57
+ VolumeSize: dev[:size],
58
+ VolumeType: dev[:type] || 'gp2'
59
+ }
60
+ end
61
+ h
62
+ end
63
+ end
64
+
65
+ def get_zone_for_record(name)
66
+ r53 = AWS::Route53.new
67
+ zones = r53.hosted_zones.sort_by { |z| z.name.split('.').count }.reverse
68
+ zones.find do |z|
69
+ name.split('.').reverse.take(z.name.split('.').count) == z.name.split('.').reverse
70
+ end
71
+ end
72
+
73
+ def get_vpc_by_name(vpc)
74
+ ec2 = AWS::EC2.new
75
+ ec2.vpcs.with_tag('Name', vpc).first
76
+ end
77
+
78
+ def resolve_vpc(vpc)
79
+ return vpc if vpc.start_with?('vpc-')
80
+ return Ref("#{vpc.capitalize}VPC") if vpc_defined?(vpc)
81
+ get_vpc_by_name(vpc).id
82
+ end
83
+
84
+ def resolve_subnets(vpc, subnets)
85
+ subnets.map do |subnet|
86
+ resolve_subnet(vpc, subnet)
87
+ end.flatten(1)
88
+ end
89
+
90
+ def subnet_refs(vpc, subnet)
91
+ vpc = @stack.vpcs.find {|v| v.name == vpc }
92
+ subnet = vpc.subnets.find {|s| s.name == subnet}
93
+ subnet_name = "#{vpc.name.capitalize}#{subnet.name.capitalize}Subnet"
94
+ azs = subnet.azs || fetch_availability_zones(vpc.region)
95
+ azs.map do |az|
96
+ Ref("#{subnet_name}#{az.capitalize}")
97
+ end
98
+ end
99
+
100
+ def subnet_defined?(vpc, subnet)
101
+ @stack.vpcs.find {|v| v.name == vpc }.subnets.map(&:name).include?(subnet)
102
+ end
103
+
104
+ def vpc_defined?(vpc)
105
+ @stack.vpcs.map(&:name).include?(vpc)
106
+ end
107
+
108
+ def resolve_subnet(vpc, subnet)
109
+ return [subnet] if subnet.start_with?('subnet-')
110
+ return subnet_refs(vpc, subnet) if subnet_defined?(vpc, subnet)
111
+ ec2 = AWS::EC2.new
112
+ v = ec2.vpcs[vpc] if vpc.start_with?('vpc-')
113
+ v ||= get_vpc_by_name(vpc)
114
+ v.subnets.with_tag('Name', subnet).map(&:id)
115
+ end
116
+
117
+ def resolve_security_groups(vpc, security_groups)
118
+ security_groups.map do |sg|
119
+ resolve_security_group(vpc, sg)
120
+ end.flatten
121
+ end
122
+
123
+ def resolve_security_group(vpc, sg)
124
+ return [sg] if sg.start_with?('sg-')
125
+ ec2 = AWS::EC2.new
126
+ v = ec2.vpcs[vpc] if vpc.start_with?('vpc-')
127
+ v ||= get_vpc_by_name(vpc)
128
+ v.security_groups.with_tag('Name', sg).map(&:id)
129
+ end
130
+ end
131
+ end
@@ -0,0 +1,48 @@
1
+ require 'clamp'
2
+ require 'awsdsl'
3
+
4
+ module AWSDSL
5
+ class CommandLine < Clamp::Command
6
+ option '--stackfile',
7
+ 'STACKFILE',
8
+ 'Path to Stackfile',
9
+ default: 'Stackfile'
10
+
11
+ subcommand 'build', 'Build Stack AMIs' do
12
+ def execute
13
+ stack = Loader.load(stackfile)
14
+ AMIBuilder.build(stack)
15
+ end
16
+ end
17
+
18
+ subcommand 'create', 'Create Stack' do
19
+ def execute
20
+ stack = Loader.load(stackfile)
21
+ stack = AMIBuilder.latest_amis(stack)
22
+ template = CfnBuilder.build(stack)
23
+ cfm = AWS::CloudFormation.new
24
+ cfm.stacks.create(stack.name.capitalize, template,
25
+ capabilities: ['CAPABILITY_IAM'])
26
+ end
27
+ end
28
+
29
+ subcommand 'update', 'Update Stack' do
30
+ def execute
31
+ stack = Loader.load(stackfile)
32
+ stack = AMIBuilder.latest_amis(stack)
33
+ template = CfnBuilder.build(stack)
34
+ cfm = AWS::CloudFormation.new
35
+ cfm.stacks[stack.name.capitalize].update(template: template,
36
+ capabilities: ['CAPABILITY_IAM'])
37
+ end
38
+ end
39
+
40
+ subcommand 'delete', 'Delete Stack' do
41
+ def execute
42
+ stack = Loader.load(stackfile)
43
+ cfm = AWS::CloudFormation.new
44
+ cfm.stacks[stack.name.capitalize].delete
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,7 @@
1
+ module AWSDSL
2
+ class Elasticache
3
+ include DSL
4
+ multi_attributes :allow, :subnet
5
+ attributes :engine, :node_type, :port, :num_nodes, :vpc
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ module AWSDSL
2
+ class LoadBalancer
3
+ include DSL
4
+ multi_attributes :listener, :dns_record, :security_group, :subnet
5
+ attributes :health_check, :internal, :connection_settings
6
+ end
7
+ end
@@ -0,0 +1,26 @@
1
+ module AWSDSL
2
+ class Role
3
+ include DSL
4
+ attr_accessor :ami
5
+
6
+ sub_components :load_balancer
7
+ multi_attributes :policy_statement,
8
+ :include_profile,
9
+ :security_group,
10
+ :block_device,
11
+ :file_provisioner,
12
+ :chef_provisioner,
13
+ :ansible_provisioner,
14
+ :subnet,
15
+ :allow
16
+ attributes :min_size,
17
+ :max_size,
18
+ :tgt_size,
19
+ :update_policy,
20
+ :instance_type,
21
+ :vpc,
22
+ :base_ami,
23
+ :vars,
24
+ :key_pair
25
+ end
26
+ end
@@ -0,0 +1,31 @@
1
+ module AWSDSL
2
+ class RoleProfile < Role
3
+ include DSL
4
+ attr_accessor :block
5
+
6
+ sub_components :load_balancer
7
+ multi_attributes :policy_statement,
8
+ :include_profile,
9
+ :security_group,
10
+ :block_device,
11
+ :file_provisioner,
12
+ :chef_provisioner,
13
+ :ansible_provisioner,
14
+ :subnet,
15
+ :allow
16
+ attributes :min_size,
17
+ :max_size,
18
+ :tgt_size,
19
+ :update_policy,
20
+ :instance_type,
21
+ :vpc,
22
+ :base_ami,
23
+ :vars,
24
+ :key_pair
25
+
26
+ def initialize(name, &block)
27
+ @block = block if block_given?
28
+ super
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,16 @@
1
+ module AWSDSL
2
+ class Stack
3
+ include DSL
4
+ attributes :description, :vars
5
+ sub_components :role, :role_profile, :elasticache, :vpc
6
+
7
+ def mixin_profiles
8
+ @roles.each do |role|
9
+ role.include_profiles.each do |profile|
10
+ role_profile = @role_profiles.find { |rp| rp.name == profile }
11
+ role_profile.block.bind(role).call
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,7 @@
1
+ module AWSDSL
2
+ class Subnet
3
+ include DSL
4
+ attributes :igw
5
+ multi_attributes :az
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ module AWSDSL
2
+ class Vpc
3
+ include DSL
4
+ sub_components :subnet
5
+ attributes :cidr, :dns, :dns_hostnames, :igw, :region, :subnet_bits
6
+ end
7
+ end