sumomo 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -2
- data/data/sumomo/custom_resource_utils.js +151 -0
- data/data/sumomo/custom_resources/AMILookup.js +106 -0
- data/data/sumomo/custom_resources/AvailabilityZones.js +89 -0
- data/data/sumomo/custom_resources/CloudflareCNAME.js +231 -0
- data/data/sumomo/custom_resources/DeployTime.js +15 -0
- data/data/sumomo/custom_resources/SelectSpot.js +294 -0
- data/data/sumomo/sources/spot-watcher-poller.sh +22 -0
- data/data/sumomo/sources/spot-watcher.sh +45 -0
- data/lib/sumomo.rb +193 -1
- data/lib/sumomo/ec2.rb +414 -0
- data/lib/sumomo/momo_extensions/resource.rb +10 -0
- data/lib/sumomo/momo_extensions/stack.rb +6 -0
- data/lib/sumomo/network.rb +70 -0
- data/lib/sumomo/stack.rb +152 -0
- data/lib/sumomo/version.rb +1 -1
- data/sumomo.gemspec +2 -0
- metadata +44 -3
data/lib/sumomo/ec2.rb
ADDED
@@ -0,0 +1,414 @@
|
|
1
|
+
|
2
|
+
module Sumomo
|
3
|
+
module Stack
|
4
|
+
|
5
|
+
def get_azs
|
6
|
+
resp = @ec2.describe_availability_zones
|
7
|
+
|
8
|
+
Array(resp.availability_zones.map do |x|
|
9
|
+
x.zone_name
|
10
|
+
end)
|
11
|
+
end
|
12
|
+
|
13
|
+
def allow(thing)
|
14
|
+
if (thing == :all)
|
15
|
+
{
|
16
|
+
"IpProtocol" => "-1",
|
17
|
+
"ToPort" => 65535,
|
18
|
+
"FromPort" => 0,
|
19
|
+
"CidrIp" => "0.0.0.0/0"
|
20
|
+
}
|
21
|
+
elsif thing.is_a? Integer and thing > 0 and thing < 65536
|
22
|
+
# its a port!
|
23
|
+
{
|
24
|
+
"IpProtocol" => "tcp",
|
25
|
+
"ToPort" => thing,
|
26
|
+
"FromPort" => thing,
|
27
|
+
"CidrIp" => "0.0.0.0/0"
|
28
|
+
}
|
29
|
+
elsif thing.is_a? String and /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\/[0-9]+/.match(thing)
|
30
|
+
# its a cidr!
|
31
|
+
{
|
32
|
+
"IpProtocol" => "tcp",
|
33
|
+
"ToPort" => 65535,
|
34
|
+
"FromPort" => 0,
|
35
|
+
"CidrIp" => thing
|
36
|
+
}
|
37
|
+
elsif thing.is_a? Hash
|
38
|
+
# more shit
|
39
|
+
{
|
40
|
+
"IpProtocol" => thing[:protocol] || "tcp",
|
41
|
+
"ToPort" => thing[:port] || thing[:end_port] || 0,
|
42
|
+
"FromPort" => thing[:port] || thing[:start_port] || 65535,
|
43
|
+
"CidrIp" => thing[:cidr] || "0.0.0.0/0"
|
44
|
+
}
|
45
|
+
else
|
46
|
+
raise "utils.rb allow: please allow something"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def http_listener(port: 80, instance_port: port)
|
51
|
+
{
|
52
|
+
"LoadBalancerPort" => port,
|
53
|
+
"InstancePort" => instance_port,
|
54
|
+
"Protocol" => "HTTP"
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
def https_listener(cert_arn:, instance_port: 80, port: 443)
|
59
|
+
res = http_listener(instance_port)
|
60
|
+
res["LoadBalancerPort"] = lb_port
|
61
|
+
res["Protocol"] = "HTTPS"
|
62
|
+
res["SSLCertificateId"] = cert_arn
|
63
|
+
|
64
|
+
return res
|
65
|
+
end
|
66
|
+
|
67
|
+
def elb_tcp_health_check(port: 80, healthy_threshold: 2, interval: 10, timeout: 5, unhealthy_threshold: 10, path: "/")
|
68
|
+
elb_health_check(port: port,
|
69
|
+
healthy_threshold: healthy_threshold,
|
70
|
+
interval: interval,
|
71
|
+
timeout: timeout,
|
72
|
+
unhealthy_threshold: unhealthy_threshold,
|
73
|
+
path: path,
|
74
|
+
check_type: "TCP")
|
75
|
+
end
|
76
|
+
|
77
|
+
def elb_health_check(port: 80,
|
78
|
+
healthy_threshold: 2,
|
79
|
+
interval: 10,
|
80
|
+
timeout: 5,
|
81
|
+
unhealthy_threshold: 10,
|
82
|
+
path: "/",
|
83
|
+
check_type: "HTTP")
|
84
|
+
|
85
|
+
options[:path] = "/#{options[:path]}"
|
86
|
+
options[:path].gsub!(/^[\/]+/, "/")
|
87
|
+
{
|
88
|
+
"HealthyThreshold" => options[:healthy_threshold] || 2,
|
89
|
+
"Interval" => options[:interval] || 10,
|
90
|
+
"Target" => "#{check_type}:#{port}#{options[:path]}",
|
91
|
+
"Timeout" => options[:timeout] || 5,
|
92
|
+
"UnhealthyThreshold" => options[:unhealthy_threshold] || 10
|
93
|
+
}
|
94
|
+
end
|
95
|
+
|
96
|
+
def initscript(wait_handle, asgname, script)
|
97
|
+
|
98
|
+
call("Fn::Base64",
|
99
|
+
call("Fn::Join", "", [
|
100
|
+
|
101
|
+
"#!/bin/bash -v\n",
|
102
|
+
"yum update -y aws-cfn-bootstrap\n",
|
103
|
+
|
104
|
+
"# Helper function\n",
|
105
|
+
"function error_exit\n",
|
106
|
+
"{\n",
|
107
|
+
" /opt/aws/bin/cfn-signal -e 1 -r \"$1\" \"", wait_handle, "\"\n",
|
108
|
+
" exit 1\n",
|
109
|
+
"}\n",
|
110
|
+
|
111
|
+
"# Run init meta\n",
|
112
|
+
"/opt/aws/bin/cfn-init -s ", ref("AWS::StackId"), " -r ", asgname, " ",
|
113
|
+
" --region ", ref("AWS::Region"), " || error_exit 'Failed to run cfn-init'\n",
|
114
|
+
|
115
|
+
"# Run script\n",
|
116
|
+
script,
|
117
|
+
|
118
|
+
"\n",
|
119
|
+
|
120
|
+
"# All is well so signal success\n",
|
121
|
+
"/opt/aws/bin/cfn-signal -e 0 -r \"Setup complete\" \"", wait_handle, "\"\n"
|
122
|
+
]))
|
123
|
+
end
|
124
|
+
|
125
|
+
class EC2Tasks
|
126
|
+
def initialize(bucket_name, &block)
|
127
|
+
@script = ""
|
128
|
+
@bucket_name = bucket_name
|
129
|
+
instance_eval(&block) if block
|
130
|
+
end
|
131
|
+
|
132
|
+
def mkdir(name)
|
133
|
+
@script += <<-SNIPPET
|
134
|
+
mkdir -p #{name}
|
135
|
+
SNIPPET
|
136
|
+
end
|
137
|
+
|
138
|
+
def download_file(name, local_path)
|
139
|
+
@script += <<-SNIPPET
|
140
|
+
aws s3 cp s3://#{@bucket_name}/uploads/#{name} #{local_path}
|
141
|
+
SNIPPET
|
142
|
+
end
|
143
|
+
|
144
|
+
def script
|
145
|
+
@script
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def make_autoscaling_group(
|
150
|
+
network:,
|
151
|
+
layer:,
|
152
|
+
zone:nil,
|
153
|
+
type:"m3.medium",
|
154
|
+
name:nil,
|
155
|
+
elb:nil,
|
156
|
+
min_size:1,
|
157
|
+
max_size:min_size,
|
158
|
+
vol_size:10,
|
159
|
+
vol_type:"gp2",
|
160
|
+
keypair:@master_key_name,
|
161
|
+
has_public_ips:true,
|
162
|
+
ingress:nil,
|
163
|
+
egress:nil,
|
164
|
+
machine_tag:nil,
|
165
|
+
ec2_sns_arn:nil,
|
166
|
+
ami_name:,
|
167
|
+
ebs_root_device:,
|
168
|
+
spot_price:nil,
|
169
|
+
script: nil,
|
170
|
+
ecs_cluster: nil,
|
171
|
+
docker_username:"",
|
172
|
+
docker_email:"",
|
173
|
+
docker_password: "",
|
174
|
+
eip:nil,
|
175
|
+
&block)
|
176
|
+
|
177
|
+
tasks = EC2Tasks.new(@bucket_name, &block).script
|
178
|
+
|
179
|
+
ingress ||= [ allow(:all) ]
|
180
|
+
egress ||= [ allow(:all) ]
|
181
|
+
machine_tag ||= ref("AWS::StackName")
|
182
|
+
name ||= make_default_resource_name("AutoScalingGroup")
|
183
|
+
script ||= ""
|
184
|
+
|
185
|
+
bucket_name = @bucket_name
|
186
|
+
|
187
|
+
script += "\n#{tasks}\n"
|
188
|
+
|
189
|
+
if ecs_cluster
|
190
|
+
script += <<-ECS_START
|
191
|
+
|
192
|
+
yum update
|
193
|
+
yum groupinstall "Development Tools"
|
194
|
+
yum install -y python screen git gcc-c++ ecs-init
|
195
|
+
curl -sSL https://get.docker.com/ | sh
|
196
|
+
|
197
|
+
cp /ecs.config /etc/ecs/ecs.config
|
198
|
+
|
199
|
+
service docker start
|
200
|
+
start ecs
|
201
|
+
|
202
|
+
curl http://localhost:51678/v1/metadata > /home/ec2-user/ecs_info
|
203
|
+
|
204
|
+
ECS_START
|
205
|
+
end
|
206
|
+
|
207
|
+
if eip
|
208
|
+
script += <<-EIP_ALLOCATE
|
209
|
+
aws ec2 associate-address --region `cat /etc/aws_region` --instance-id `curl http://169.254.169.254/latest/meta-data/instance-id` --allocation-id `cat /etc/eip_allocation_id`
|
210
|
+
EIP_ALLOCATE
|
211
|
+
end
|
212
|
+
|
213
|
+
script += "\nservice spot-watcher start" if spot_price and ec2_sns_arn
|
214
|
+
|
215
|
+
raise "ec2: ingress option needs to be an array" if !ingress.is_a? Array
|
216
|
+
raise "ec2: egress option needs to be an array" if !egress.is_a? Array
|
217
|
+
|
218
|
+
web_sec_group = make "AWS::EC2::SecurityGroup" do
|
219
|
+
GroupDescription "Security group for layer: #{layer}"
|
220
|
+
SecurityGroupIngress ingress
|
221
|
+
SecurityGroupEgress egress
|
222
|
+
VpcId network.vpc
|
223
|
+
end
|
224
|
+
|
225
|
+
wait_handle = make "AWS::CloudFormation::WaitConditionHandle"
|
226
|
+
|
227
|
+
user_data = initscript(wait_handle, name, script)
|
228
|
+
|
229
|
+
role_policy_doc = {
|
230
|
+
"Version" => "2012-10-17",
|
231
|
+
"Statement" => [{
|
232
|
+
"Effect" => "Allow",
|
233
|
+
"Principal" => {"Service" => ["ec2.amazonaws.com"]},
|
234
|
+
"Action" => ["sts:AssumeRole"]
|
235
|
+
}]
|
236
|
+
}
|
237
|
+
|
238
|
+
asg_role = make "AWS::IAM::Role" do
|
239
|
+
AssumeRolePolicyDocument role_policy_doc
|
240
|
+
Path "/"
|
241
|
+
Policies [{
|
242
|
+
"PolicyName" => "root",
|
243
|
+
"PolicyDocument" => {
|
244
|
+
"Version" => "2012-10-17",
|
245
|
+
"Statement" => [{
|
246
|
+
"Effect" => "Allow",
|
247
|
+
"Action" => ["sns:Publish"],
|
248
|
+
"Resource" => "*"
|
249
|
+
},
|
250
|
+
{
|
251
|
+
"Effect" => "Allow",
|
252
|
+
"Action" => ["s3:DeleteObject", "s3:GetObject", "s3:PutObject"],
|
253
|
+
"Resource" => "arn:aws:s3:::#{bucket_name}/uploads/*"
|
254
|
+
},
|
255
|
+
{
|
256
|
+
"Effect" => "Allow",
|
257
|
+
"Action" => [
|
258
|
+
"ec2:AllocateAddress",
|
259
|
+
"ec2:AssociateAddress",
|
260
|
+
"ec2:DescribeAddresses",
|
261
|
+
"ec2:DisassociateAddress"
|
262
|
+
],
|
263
|
+
"Resource" => "*"
|
264
|
+
},
|
265
|
+
{
|
266
|
+
"Effect" => "Allow",
|
267
|
+
"Action" => [
|
268
|
+
"ecs:DeregisterContainerInstance",
|
269
|
+
"ecs:DiscoverPollEndpoint",
|
270
|
+
"ecs:Poll",
|
271
|
+
"ecs:RegisterContainerInstance",
|
272
|
+
"ecs:StartTelemetrySession",
|
273
|
+
"ecs:Submit*",
|
274
|
+
"ecr:GetAuthorizationToken",
|
275
|
+
"ecr:BatchCheckLayerAvailability",
|
276
|
+
"ecr:GetDownloadUrlForLayer",
|
277
|
+
"ecr:BatchGetImage",
|
278
|
+
"logs:CreateLogStream",
|
279
|
+
"logs:PutLogEvents"
|
280
|
+
],
|
281
|
+
"Resource": "*"
|
282
|
+
}]
|
283
|
+
}
|
284
|
+
}]
|
285
|
+
end
|
286
|
+
|
287
|
+
asg_profile = make "AWS::IAM::InstanceProfile" do
|
288
|
+
Path "/"
|
289
|
+
Roles [ asg_role ]
|
290
|
+
end
|
291
|
+
|
292
|
+
launch_config = make "AWS::AutoScaling::LaunchConfiguration" do
|
293
|
+
AssociatePublicIpAddress has_public_ips
|
294
|
+
KeyName keypair
|
295
|
+
SecurityGroups [ web_sec_group ]
|
296
|
+
ImageId ami_name
|
297
|
+
UserData user_data
|
298
|
+
InstanceType type
|
299
|
+
IamInstanceProfile asg_profile
|
300
|
+
SpotPrice spot_price if spot_price
|
301
|
+
BlockDeviceMappings [{
|
302
|
+
"DeviceName" => ebs_root_device,
|
303
|
+
"Ebs" => {
|
304
|
+
"VolumeType" => vol_type,
|
305
|
+
"VolumeSize" => vol_size,
|
306
|
+
}
|
307
|
+
}]
|
308
|
+
end
|
309
|
+
|
310
|
+
zones_used = network.azs
|
311
|
+
subnet_ids = network.subnets[layer].map { |x| x[:name] }
|
312
|
+
|
313
|
+
if zone
|
314
|
+
# if we only specified a single zone, then we have to do some processing
|
315
|
+
res = define_custom_resource(name: "SubnetIdentifierCodeFor#{name}", code: <<-CODE
|
316
|
+
var ids = {};
|
317
|
+
var zones = request.ResourceProperties.SubnetZones;
|
318
|
+
for (var i=0;i<zones.length;i++)
|
319
|
+
{
|
320
|
+
ids[zones[i]] = request.ResourceProperties.SubnetIds[i];
|
321
|
+
}
|
322
|
+
|
323
|
+
Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, "Success", ids[request.ResourceProperties.Zone]);
|
324
|
+
CODE
|
325
|
+
)
|
326
|
+
|
327
|
+
identifier = make_custom res, name: "SubnetIdentifierFor#{name}" do
|
328
|
+
SubnetIds network.subnets[layer].map { |x| x[:name] }
|
329
|
+
SubnetZones network.subnets[layer].map { |x| x[:zone] }
|
330
|
+
Zone zone
|
331
|
+
end
|
332
|
+
|
333
|
+
zones_used = [ zone ]
|
334
|
+
subnet_ids = [ identifier ]
|
335
|
+
end
|
336
|
+
|
337
|
+
|
338
|
+
asg = make "AWS::AutoScaling::AutoScalingGroup", name: name do
|
339
|
+
depends_on network.attachment
|
340
|
+
|
341
|
+
AvailabilityZones zones_used
|
342
|
+
|
343
|
+
Cooldown 30
|
344
|
+
MinSize min_size
|
345
|
+
MaxSize max_size
|
346
|
+
|
347
|
+
VPCZoneIdentifier subnet_ids
|
348
|
+
|
349
|
+
LaunchConfigurationName launch_config
|
350
|
+
LoadBalancerNames [ elb ] if elb
|
351
|
+
|
352
|
+
NotificationConfigurations [
|
353
|
+
{
|
354
|
+
"NotificationTypes" => [
|
355
|
+
"autoscaling:EC2_INSTANCE_LAUNCH",
|
356
|
+
"autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
|
357
|
+
"autoscaling:EC2_INSTANCE_TERMINATE",
|
358
|
+
"autoscaling:EC2_INSTANCE_TERMINATE_ERROR",
|
359
|
+
"autoscaling:TEST_NOTIFICATION"
|
360
|
+
],
|
361
|
+
"TopicARN" => ec2_sns_arn
|
362
|
+
}
|
363
|
+
] if ec2_sns_arn
|
364
|
+
|
365
|
+
file "/etc/aws_region", content: "{{ region }}", context: {
|
366
|
+
region: ref("AWS::Region")
|
367
|
+
}
|
368
|
+
|
369
|
+
if ec2_sns_arn
|
370
|
+
file "/etc/sns_arn", content: "{{ sns_arn }}", context: {
|
371
|
+
sns_arn: ec2_sns_arn
|
372
|
+
}
|
373
|
+
end
|
374
|
+
|
375
|
+
if eip
|
376
|
+
file "/etc/eip_allocation_id", content: "{{ id }}", context: {
|
377
|
+
id: eip.AllocationId
|
378
|
+
}
|
379
|
+
end
|
380
|
+
|
381
|
+
if spot_price and ec2_sns_arn
|
382
|
+
watcher = File.read( File.join( Gem.datadir("sumomo"), "sources", "spot-watcher.sh" ) )
|
383
|
+
poller = File.read( File.join( Gem.datadir("sumomo"), "sources", "spot-watcher-poller.sh" ) )
|
384
|
+
|
385
|
+
file "/etc/init.d/spot-watcher", content: watcher, mode: "000700"
|
386
|
+
file "/bin/spot-watcher", content: poller, mode: "000700", context: {
|
387
|
+
sns_arn: ec2_sns_arn,
|
388
|
+
region: ref("AWS::Region")
|
389
|
+
}
|
390
|
+
end
|
391
|
+
|
392
|
+
if ecs_cluster
|
393
|
+
ecs_config = <<-CONFIG
|
394
|
+
ECS_CLUSTER={{cluster_name}}
|
395
|
+
ECS_ENGINE_AUTH_TYPE=docker
|
396
|
+
ECS_ENGINE_AUTH_DATA={"https://index.docker.io/v1/":{"username":"{{docker_username}}","password":"{{docker_password}}","email":"{{docker_email}}"}}
|
397
|
+
CONFIG
|
398
|
+
|
399
|
+
file "/ecs.config", content: ecs_config, context: {
|
400
|
+
cluster_name: ecs_cluster,
|
401
|
+
docker_username: docker_username,
|
402
|
+
docker_password: docker_password,
|
403
|
+
docker_email: docker_email
|
404
|
+
}
|
405
|
+
end
|
406
|
+
|
407
|
+
tag "Name", machine_tag, propagate_at_launch: true
|
408
|
+
end
|
409
|
+
|
410
|
+
asg
|
411
|
+
|
412
|
+
end
|
413
|
+
end
|
414
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require "active_support/inflector"
|
2
|
+
require "hashie"
|
3
|
+
|
4
|
+
module Sumomo
|
5
|
+
module Stack
|
6
|
+
|
7
|
+
def make_network(layers: [])
|
8
|
+
|
9
|
+
zones = get_azs()
|
10
|
+
|
11
|
+
region = @region
|
12
|
+
|
13
|
+
vpc = make "AWS::EC2::VPC" do
|
14
|
+
CidrBlock "10.0.0.0/16"
|
15
|
+
EnableDnsSupport true
|
16
|
+
EnableDnsHostnames true
|
17
|
+
tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
|
18
|
+
end
|
19
|
+
|
20
|
+
gateway = make "AWS::EC2::InternetGateway" do
|
21
|
+
tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
|
22
|
+
end
|
23
|
+
|
24
|
+
attachment = make "AWS::EC2::VPCGatewayAttachment" do
|
25
|
+
VpcId vpc
|
26
|
+
InternetGatewayId gateway
|
27
|
+
end
|
28
|
+
|
29
|
+
inet_route_table = make "AWS::EC2::RouteTable" do
|
30
|
+
depends_on attachment
|
31
|
+
VpcId vpc
|
32
|
+
tag "Name", call("Fn::Join", "-", ["public", ref("AWS::StackName")])
|
33
|
+
end
|
34
|
+
|
35
|
+
make "AWS::EC2::Route" do
|
36
|
+
RouteTableId inet_route_table
|
37
|
+
DestinationCidrBlock "0.0.0.0/0"
|
38
|
+
GatewayId gateway
|
39
|
+
end
|
40
|
+
|
41
|
+
subnets = {}
|
42
|
+
|
43
|
+
layers.product(zones).each_with_index do |e, subnet_number|
|
44
|
+
layer = e[0]
|
45
|
+
zone = e[1]
|
46
|
+
|
47
|
+
zone_letter = zone.sub("#{region}", "")
|
48
|
+
cidr = "10.0.#{subnet_number}.0/24"
|
49
|
+
|
50
|
+
subnet = make "AWS::EC2::Subnet", name: "SubnetFor#{layer.camelize}Layer#{zone_letter.upcase}" do
|
51
|
+
AvailabilityZone zone
|
52
|
+
VpcId vpc
|
53
|
+
CidrBlock cidr
|
54
|
+
|
55
|
+
tag("Name", call("Fn::Join", "-", [ ref("AWS::StackName"), "#{layer}", zone_letter] ) )
|
56
|
+
end
|
57
|
+
|
58
|
+
make "AWS::EC2::SubnetRouteTableAssociation", name: "SubnetRTAFor#{layer.camelize}Layer#{zone_letter.upcase}" do
|
59
|
+
SubnetId subnet
|
60
|
+
RouteTableId inet_route_table
|
61
|
+
end
|
62
|
+
|
63
|
+
subnets[layer] ||= []
|
64
|
+
subnets[layer] << {name: subnet, cidr: cidr, zone: zone}
|
65
|
+
end
|
66
|
+
|
67
|
+
Hashie::Mash.new vpc: vpc, subnets: subnets, azs: zones, attachment: attachment
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|