sumomo 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,414 @@
1
+
2
+ module Sumomo
3
+ module Stack
4
+
5
+ def get_azs
6
+ resp = @ec2.describe_availability_zones
7
+
8
+ Array(resp.availability_zones.map do |x|
9
+ x.zone_name
10
+ end)
11
+ end
12
+
13
+ def allow(thing)
14
+ if (thing == :all)
15
+ {
16
+ "IpProtocol" => "-1",
17
+ "ToPort" => 65535,
18
+ "FromPort" => 0,
19
+ "CidrIp" => "0.0.0.0/0"
20
+ }
21
+ elsif thing.is_a? Integer and thing > 0 and thing < 65536
22
+ # its a port!
23
+ {
24
+ "IpProtocol" => "tcp",
25
+ "ToPort" => thing,
26
+ "FromPort" => thing,
27
+ "CidrIp" => "0.0.0.0/0"
28
+ }
29
+ elsif thing.is_a? String and /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\/[0-9]+/.match(thing)
30
+ # its a cidr!
31
+ {
32
+ "IpProtocol" => "tcp",
33
+ "ToPort" => 65535,
34
+ "FromPort" => 0,
35
+ "CidrIp" => thing
36
+ }
37
+ elsif thing.is_a? Hash
38
+ # more shit
39
+ {
40
+ "IpProtocol" => thing[:protocol] || "tcp",
41
+ "ToPort" => thing[:port] || thing[:end_port] || 0,
42
+ "FromPort" => thing[:port] || thing[:start_port] || 65535,
43
+ "CidrIp" => thing[:cidr] || "0.0.0.0/0"
44
+ }
45
+ else
46
+ raise "utils.rb allow: please allow something"
47
+ end
48
+ end
49
+
50
+ def http_listener(port: 80, instance_port: port)
51
+ {
52
+ "LoadBalancerPort" => port,
53
+ "InstancePort" => instance_port,
54
+ "Protocol" => "HTTP"
55
+ }
56
+ end
57
+
58
+ def https_listener(cert_arn:, instance_port: 80, port: 443)
59
+ res = http_listener(instance_port)
60
+ res["LoadBalancerPort"] = lb_port
61
+ res["Protocol"] = "HTTPS"
62
+ res["SSLCertificateId"] = cert_arn
63
+
64
+ return res
65
+ end
66
+
67
+ def elb_tcp_health_check(port: 80, healthy_threshold: 2, interval: 10, timeout: 5, unhealthy_threshold: 10, path: "/")
68
+ elb_health_check(port: port,
69
+ healthy_threshold: healthy_threshold,
70
+ interval: interval,
71
+ timeout: timeout,
72
+ unhealthy_threshold: unhealthy_threshold,
73
+ path: path,
74
+ check_type: "TCP")
75
+ end
76
+
77
+ def elb_health_check(port: 80,
78
+ healthy_threshold: 2,
79
+ interval: 10,
80
+ timeout: 5,
81
+ unhealthy_threshold: 10,
82
+ path: "/",
83
+ check_type: "HTTP")
84
+
85
+ options[:path] = "/#{options[:path]}"
86
+ options[:path].gsub!(/^[\/]+/, "/")
87
+ {
88
+ "HealthyThreshold" => options[:healthy_threshold] || 2,
89
+ "Interval" => options[:interval] || 10,
90
+ "Target" => "#{check_type}:#{port}#{options[:path]}",
91
+ "Timeout" => options[:timeout] || 5,
92
+ "UnhealthyThreshold" => options[:unhealthy_threshold] || 10
93
+ }
94
+ end
95
+
96
+ def initscript(wait_handle, asgname, script)
97
+
98
+ call("Fn::Base64",
99
+ call("Fn::Join", "", [
100
+
101
+ "#!/bin/bash -v\n",
102
+ "yum update -y aws-cfn-bootstrap\n",
103
+
104
+ "# Helper function\n",
105
+ "function error_exit\n",
106
+ "{\n",
107
+ " /opt/aws/bin/cfn-signal -e 1 -r \"$1\" \"", wait_handle, "\"\n",
108
+ " exit 1\n",
109
+ "}\n",
110
+
111
+ "# Run init meta\n",
112
+ "/opt/aws/bin/cfn-init -s ", ref("AWS::StackId"), " -r ", asgname, " ",
113
+ " --region ", ref("AWS::Region"), " || error_exit 'Failed to run cfn-init'\n",
114
+
115
+ "# Run script\n",
116
+ script,
117
+
118
+ "\n",
119
+
120
+ "# All is well so signal success\n",
121
+ "/opt/aws/bin/cfn-signal -e 0 -r \"Setup complete\" \"", wait_handle, "\"\n"
122
+ ]))
123
+ end
124
+
125
+ class EC2Tasks
126
+ def initialize(bucket_name, &block)
127
+ @script = ""
128
+ @bucket_name = bucket_name
129
+ instance_eval(&block) if block
130
+ end
131
+
132
+ def mkdir(name)
133
+ @script += <<-SNIPPET
134
+ mkdir -p #{name}
135
+ SNIPPET
136
+ end
137
+
138
+ def download_file(name, local_path)
139
+ @script += <<-SNIPPET
140
+ aws s3 cp s3://#{@bucket_name}/uploads/#{name} #{local_path}
141
+ SNIPPET
142
+ end
143
+
144
+ def script
145
+ @script
146
+ end
147
+ end
148
+
149
+ def make_autoscaling_group(
150
+ network:,
151
+ layer:,
152
+ zone:nil,
153
+ type:"m3.medium",
154
+ name:nil,
155
+ elb:nil,
156
+ min_size:1,
157
+ max_size:min_size,
158
+ vol_size:10,
159
+ vol_type:"gp2",
160
+ keypair:@master_key_name,
161
+ has_public_ips:true,
162
+ ingress:nil,
163
+ egress:nil,
164
+ machine_tag:nil,
165
+ ec2_sns_arn:nil,
166
+ ami_name:,
167
+ ebs_root_device:,
168
+ spot_price:nil,
169
+ script: nil,
170
+ ecs_cluster: nil,
171
+ docker_username:"",
172
+ docker_email:"",
173
+ docker_password: "",
174
+ eip:nil,
175
+ &block)
176
+
177
+ tasks = EC2Tasks.new(@bucket_name, &block).script
178
+
179
+ ingress ||= [ allow(:all) ]
180
+ egress ||= [ allow(:all) ]
181
+ machine_tag ||= ref("AWS::StackName")
182
+ name ||= make_default_resource_name("AutoScalingGroup")
183
+ script ||= ""
184
+
185
+ bucket_name = @bucket_name
186
+
187
+ script += "\n#{tasks}\n"
188
+
189
+ if ecs_cluster
190
+ script += <<-ECS_START
191
+
192
+ yum update
193
+ yum groupinstall "Development Tools"
194
+ yum install -y python screen git gcc-c++ ecs-init
195
+ curl -sSL https://get.docker.com/ | sh
196
+
197
+ cp /ecs.config /etc/ecs/ecs.config
198
+
199
+ service docker start
200
+ start ecs
201
+
202
+ curl http://localhost:51678/v1/metadata > /home/ec2-user/ecs_info
203
+
204
+ ECS_START
205
+ end
206
+
207
+ if eip
208
+ script += <<-EIP_ALLOCATE
209
+ aws ec2 associate-address --region `cat /etc/aws_region` --instance-id `curl http://169.254.169.254/latest/meta-data/instance-id` --allocation-id `cat /etc/eip_allocation_id`
210
+ EIP_ALLOCATE
211
+ end
212
+
213
+ script += "\nservice spot-watcher start" if spot_price and ec2_sns_arn
214
+
215
+ raise "ec2: ingress option needs to be an array" if !ingress.is_a? Array
216
+ raise "ec2: egress option needs to be an array" if !egress.is_a? Array
217
+
218
+ web_sec_group = make "AWS::EC2::SecurityGroup" do
219
+ GroupDescription "Security group for layer: #{layer}"
220
+ SecurityGroupIngress ingress
221
+ SecurityGroupEgress egress
222
+ VpcId network.vpc
223
+ end
224
+
225
+ wait_handle = make "AWS::CloudFormation::WaitConditionHandle"
226
+
227
+ user_data = initscript(wait_handle, name, script)
228
+
229
+ role_policy_doc = {
230
+ "Version" => "2012-10-17",
231
+ "Statement" => [{
232
+ "Effect" => "Allow",
233
+ "Principal" => {"Service" => ["ec2.amazonaws.com"]},
234
+ "Action" => ["sts:AssumeRole"]
235
+ }]
236
+ }
237
+
238
+ asg_role = make "AWS::IAM::Role" do
239
+ AssumeRolePolicyDocument role_policy_doc
240
+ Path "/"
241
+ Policies [{
242
+ "PolicyName" => "root",
243
+ "PolicyDocument" => {
244
+ "Version" => "2012-10-17",
245
+ "Statement" => [{
246
+ "Effect" => "Allow",
247
+ "Action" => ["sns:Publish"],
248
+ "Resource" => "*"
249
+ },
250
+ {
251
+ "Effect" => "Allow",
252
+ "Action" => ["s3:DeleteObject", "s3:GetObject", "s3:PutObject"],
253
+ "Resource" => "arn:aws:s3:::#{bucket_name}/uploads/*"
254
+ },
255
+ {
256
+ "Effect" => "Allow",
257
+ "Action" => [
258
+ "ec2:AllocateAddress",
259
+ "ec2:AssociateAddress",
260
+ "ec2:DescribeAddresses",
261
+ "ec2:DisassociateAddress"
262
+ ],
263
+ "Resource" => "*"
264
+ },
265
+ {
266
+ "Effect" => "Allow",
267
+ "Action" => [
268
+ "ecs:DeregisterContainerInstance",
269
+ "ecs:DiscoverPollEndpoint",
270
+ "ecs:Poll",
271
+ "ecs:RegisterContainerInstance",
272
+ "ecs:StartTelemetrySession",
273
+ "ecs:Submit*",
274
+ "ecr:GetAuthorizationToken",
275
+ "ecr:BatchCheckLayerAvailability",
276
+ "ecr:GetDownloadUrlForLayer",
277
+ "ecr:BatchGetImage",
278
+ "logs:CreateLogStream",
279
+ "logs:PutLogEvents"
280
+ ],
281
+ "Resource": "*"
282
+ }]
283
+ }
284
+ }]
285
+ end
286
+
287
+ asg_profile = make "AWS::IAM::InstanceProfile" do
288
+ Path "/"
289
+ Roles [ asg_role ]
290
+ end
291
+
292
+ launch_config = make "AWS::AutoScaling::LaunchConfiguration" do
293
+ AssociatePublicIpAddress has_public_ips
294
+ KeyName keypair
295
+ SecurityGroups [ web_sec_group ]
296
+ ImageId ami_name
297
+ UserData user_data
298
+ InstanceType type
299
+ IamInstanceProfile asg_profile
300
+ SpotPrice spot_price if spot_price
301
+ BlockDeviceMappings [{
302
+ "DeviceName" => ebs_root_device,
303
+ "Ebs" => {
304
+ "VolumeType" => vol_type,
305
+ "VolumeSize" => vol_size,
306
+ }
307
+ }]
308
+ end
309
+
310
+ zones_used = network.azs
311
+ subnet_ids = network.subnets[layer].map { |x| x[:name] }
312
+
313
+ if zone
314
+ # if we only specified a single zone, then we have to do some processing
315
+ res = define_custom_resource(name: "SubnetIdentifierCodeFor#{name}", code: <<-CODE
316
+ var ids = {};
317
+ var zones = request.ResourceProperties.SubnetZones;
318
+ for (var i=0;i<zones.length;i++)
319
+ {
320
+ ids[zones[i]] = request.ResourceProperties.SubnetIds[i];
321
+ }
322
+
323
+ Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, "Success", ids[request.ResourceProperties.Zone]);
324
+ CODE
325
+ )
326
+
327
+ identifier = make_custom res, name: "SubnetIdentifierFor#{name}" do
328
+ SubnetIds network.subnets[layer].map { |x| x[:name] }
329
+ SubnetZones network.subnets[layer].map { |x| x[:zone] }
330
+ Zone zone
331
+ end
332
+
333
+ zones_used = [ zone ]
334
+ subnet_ids = [ identifier ]
335
+ end
336
+
337
+
338
+ asg = make "AWS::AutoScaling::AutoScalingGroup", name: name do
339
+ depends_on network.attachment
340
+
341
+ AvailabilityZones zones_used
342
+
343
+ Cooldown 30
344
+ MinSize min_size
345
+ MaxSize max_size
346
+
347
+ VPCZoneIdentifier subnet_ids
348
+
349
+ LaunchConfigurationName launch_config
350
+ LoadBalancerNames [ elb ] if elb
351
+
352
+ NotificationConfigurations [
353
+ {
354
+ "NotificationTypes" => [
355
+ "autoscaling:EC2_INSTANCE_LAUNCH",
356
+ "autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
357
+ "autoscaling:EC2_INSTANCE_TERMINATE",
358
+ "autoscaling:EC2_INSTANCE_TERMINATE_ERROR",
359
+ "autoscaling:TEST_NOTIFICATION"
360
+ ],
361
+ "TopicARN" => ec2_sns_arn
362
+ }
363
+ ] if ec2_sns_arn
364
+
365
+ file "/etc/aws_region", content: "{{ region }}", context: {
366
+ region: ref("AWS::Region")
367
+ }
368
+
369
+ if ec2_sns_arn
370
+ file "/etc/sns_arn", content: "{{ sns_arn }}", context: {
371
+ sns_arn: ec2_sns_arn
372
+ }
373
+ end
374
+
375
+ if eip
376
+ file "/etc/eip_allocation_id", content: "{{ id }}", context: {
377
+ id: eip.AllocationId
378
+ }
379
+ end
380
+
381
+ if spot_price and ec2_sns_arn
382
+ watcher = File.read( File.join( Gem.datadir("sumomo"), "sources", "spot-watcher.sh" ) )
383
+ poller = File.read( File.join( Gem.datadir("sumomo"), "sources", "spot-watcher-poller.sh" ) )
384
+
385
+ file "/etc/init.d/spot-watcher", content: watcher, mode: "000700"
386
+ file "/bin/spot-watcher", content: poller, mode: "000700", context: {
387
+ sns_arn: ec2_sns_arn,
388
+ region: ref("AWS::Region")
389
+ }
390
+ end
391
+
392
+ if ecs_cluster
393
+ ecs_config = <<-CONFIG
394
+ ECS_CLUSTER={{cluster_name}}
395
+ ECS_ENGINE_AUTH_TYPE=docker
396
+ ECS_ENGINE_AUTH_DATA={"https://index.docker.io/v1/":{"username":"{{docker_username}}","password":"{{docker_password}}","email":"{{docker_email}}"}}
397
+ CONFIG
398
+
399
+ file "/ecs.config", content: ecs_config, context: {
400
+ cluster_name: ecs_cluster,
401
+ docker_username: docker_username,
402
+ docker_password: docker_password,
403
+ docker_email: docker_email
404
+ }
405
+ end
406
+
407
+ tag "Name", machine_tag, propagate_at_launch: true
408
+ end
409
+
410
+ asg
411
+
412
+ end
413
+ end
414
+ end
@@ -0,0 +1,10 @@
1
+
2
+ module Momo
3
+ class Resource
4
+ def exec_role
5
+ res = Momo::Resource.new("AWS::IAM::Role", "LambdaFunctionExecutionRole", @stack)
6
+ res.complete!
7
+ res
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,6 @@
1
+
2
+ module Momo
3
+ class Stack
4
+ alias_method :stack_make, :make
5
+ end
6
+ end
@@ -0,0 +1,70 @@
1
+ require "active_support/inflector"
2
+ require "hashie"
3
+
4
+ module Sumomo
5
+ module Stack
6
+
7
+ def make_network(layers: [])
8
+
9
+ zones = get_azs()
10
+
11
+ region = @region
12
+
13
+ vpc = make "AWS::EC2::VPC" do
14
+ CidrBlock "10.0.0.0/16"
15
+ EnableDnsSupport true
16
+ EnableDnsHostnames true
17
+ tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
18
+ end
19
+
20
+ gateway = make "AWS::EC2::InternetGateway" do
21
+ tag "Name", call("Fn::Join", "-", [ref("AWS::StackName")])
22
+ end
23
+
24
+ attachment = make "AWS::EC2::VPCGatewayAttachment" do
25
+ VpcId vpc
26
+ InternetGatewayId gateway
27
+ end
28
+
29
+ inet_route_table = make "AWS::EC2::RouteTable" do
30
+ depends_on attachment
31
+ VpcId vpc
32
+ tag "Name", call("Fn::Join", "-", ["public", ref("AWS::StackName")])
33
+ end
34
+
35
+ make "AWS::EC2::Route" do
36
+ RouteTableId inet_route_table
37
+ DestinationCidrBlock "0.0.0.0/0"
38
+ GatewayId gateway
39
+ end
40
+
41
+ subnets = {}
42
+
43
+ layers.product(zones).each_with_index do |e, subnet_number|
44
+ layer = e[0]
45
+ zone = e[1]
46
+
47
+ zone_letter = zone.sub("#{region}", "")
48
+ cidr = "10.0.#{subnet_number}.0/24"
49
+
50
+ subnet = make "AWS::EC2::Subnet", name: "SubnetFor#{layer.camelize}Layer#{zone_letter.upcase}" do
51
+ AvailabilityZone zone
52
+ VpcId vpc
53
+ CidrBlock cidr
54
+
55
+ tag("Name", call("Fn::Join", "-", [ ref("AWS::StackName"), "#{layer}", zone_letter] ) )
56
+ end
57
+
58
+ make "AWS::EC2::SubnetRouteTableAssociation", name: "SubnetRTAFor#{layer.camelize}Layer#{zone_letter.upcase}" do
59
+ SubnetId subnet
60
+ RouteTableId inet_route_table
61
+ end
62
+
63
+ subnets[layer] ||= []
64
+ subnets[layer] << {name: subnet, cidr: cidr, zone: zone}
65
+ end
66
+
67
+ Hashie::Mash.new vpc: vpc, subnets: subnets, azs: zones, attachment: attachment
68
+ end
69
+ end
70
+ end