sumomo 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19a7b13c51abbddeff61c5b0b23eb1163dd70bd8
-  data.tar.gz: fbe8f61d32f2d6acd41b6437fa8f8ac84773f4c3
+  metadata.gz: 1840367c723c373796e9a822b5c5e148a35bd609
+  data.tar.gz: f9a20e064f13018ce49868289243eb95c893dd96
 SHA512:
-  metadata.gz: a12275c78d5ee20bcd911f678607003f6c1681570243d1a8772d702dc1950b8180474cfdfbd10440813d05d8d370b260c6c235c476d15de96c16dbf1dc225b2e
-  data.tar.gz: a13a1e7840147f9e70948256b81ecd8f0baf25b7bc58bc53e938e641663a8040e5ca00d949dae36de3333a0f3fc54b99da0c592bb95f33323a03d78940194265
+  metadata.gz: cdfdedee63e23f076103f0de8a0e7c2f945830d254c57587668d8a0065d04ff7894205210b0c45f54c941303f6e3e5af75f8b44e0a9b8bfa47a1c8ce2a9ff087
+  data.tar.gz: 56333dd662ec58d5c426d40a8b2d1837aa78c0547f523bd175cc611fbdf79d59f5bd54fecb86ecfd3e7d6634fe675faff12bb09e1490c22f6a89afd7b08067a6

data/data/sumomo/custom_resources/DeployTime.js

@@ -1,4 +1,3 @@
-
 if (request.RequestType == "Create")
 {
   Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, "Success", String((new Date).getTime()));
@@ -12,4 +11,4 @@ if (request.RequestType == "Update")
 if (request.RequestType == "Delete")
 {
   Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, "Success", String((new Date).getTime()));
-}
+}

data/data/sumomo/custom_resources/OriginAccessIdentity.js

@@ -0,0 +1,115 @@
+
+var cloudfront = new aws.CloudFront({region: request.ResourceProperties.Region});
+
+function randomIntInc (low, high)
+{
+    return Math.floor(Math.random() * (high - low + 1) + low);
+}
+
+function randomNumber()
+{
+  var numbers = new Array(10);
+  for (var i = 0; i < numbers.length; i++)
+  {
+      numbers[i] = randomIntInc(1,10);
+  }
+  return numbers.join("");
+}
+
+function request_identity(ref, onSuccess, onError)
+{
+  var params = {
+    CloudFrontOriginAccessIdentityConfig: {
+      CallerReference: ref,
+      Comment: request.ResourceProperties.Comment || "No Comment Specified"
+    }
+  }
+
+  cloudfront.createCloudFrontOriginAccessIdentity(params, function(err, data) {
+    if (err)
+    {
+      onError(err);
+    }
+    else
+    {
+      onSuccess(data);
+    }
+  });
+}
+
+if (request.RequestType == "Create")
+{
+  var ref = randomNumber();
+  store.put("oai_ref", ref, function() {
+    request_identity(ref, 
+      function(data)
+      {
+        console.log(data);
+        Cloudformation.send(request, context, Cloudformation.SUCCESS, {
+          Id: data.CloudFrontOriginAccessIdentity.Id,
+          S3CanonicalUserId: data.CloudFrontOriginAccessIdentity.S3CanonicalUserId
+        }, "Success", "origin-access-identity/cloudfront/" + data.CloudFrontOriginAccessIdentity.Id);
+
+      }, function(err)
+      {
+        console.log(err, err.stack);
+        Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Error: " + err);
+      });
+  }, function(err) {
+    Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Cannot Store OAI Reference");
+  });
+}
+
+if (request.RequestType == "Update")
+{
+  store.get("oai_ref", function(ref) {
+    request_identity(ref, 
+      function(data)
+      {
+        console.log(data);
+        Cloudformation.send(request, context, Cloudformation.SUCCESS, {
+          Id: data.CloudFrontOriginAccessIdentity.Id,
+          S3CanonicalUserId: data.CloudFrontOriginAccessIdentity.S3CanonicalUserId
+        }, "Success", "origin-access-identity/cloudfront/" + data.CloudFrontOriginAccessIdentity.Id);
+
+      }, function(err)
+      {
+        console.log(err, err.stack);
+        Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Error: " + err);
+      });
+  }, function(err) {
+    Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Cannot Load OAI Reference");
+  });
+}
+
+if (request.RequestType == "Delete")
+{
+  store.get("oai_ref", function(ref) {
+    request_identity(ref, 
+      function(data)
+      {
+        var params = {
+          Id: data.CloudFrontOriginAccessIdentity.Id,
+        };
+        cloudfront.deleteCloudFrontOriginAccessIdentity(params, function(err, data) {
+          if (err)
+          {
+            console.log(err, err.stack);
+            Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Error: " + err);
+          }
+          else
+          {
+            Cloudformation.send(request, context, Cloudformation.SUCCESS, {}, "Success", "" );
+          }
+        });
+
+      }, function(err)
+      {
+        console.log(err, err.stack);
+        Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Error: " + err);
+      });
+
+  }, function(err) {
+    Cloudformation.send(request, context, Cloudformation.FAILED, {}, "Cannot Load OAI Reference");
+  });
+}

data/exe/sumomo

@@ -13,10 +13,16 @@ global_opts = Trollop::options do
   USAGE
 
   opt :region, "AWS region to use", type: :string, default: "ap-northeast-1"
+  opt :profile, "AWS credential profile to use", type: :string, default: "default"
 
   stop_on SUB_COMMANDS
 end
 
+ENV["AWS_PROFILE"] = global_opts[:profile]
+
+puts "Using profile:"
+p ENV["AWS_PROFILE"]
+
 cmd = ARGV.shift # get the subcommand
 
 cmd_opts = case cmd

data/lib/sumomo.rb

@@ -91,6 +91,8 @@ module Sumomo
 
 		end.templatize
 
+		
+
 		#puts JSON.parse(template).to_yaml
 		
 		store.set_raw("cloudformation/template", template)

data/lib/sumomo/ec2.rb

@@ -10,7 +10,7 @@ module Sumomo
 			end)
 		end
 
-		def allow(thing)
+		def allow_port(thing)
 			if (thing == :all)
 				{
 					"IpProtocol" => "-1",
@@ -270,6 +270,7 @@ sudo aws s3 cp s3://#{@bucket_name}/uploads/#{name} #{local_path}
 			docker_email:"",
 			docker_password: "",
 			eip:nil,
+			policies:[],
 			&block)
 
 			if ami_name == nil
@@ -290,8 +291,8 @@ sudo aws s3 cp s3://#{@bucket_name}/uploads/#{name} #{local_path}
 
 			task_script = tasks.script
 
-			ingress ||= [ allow(:all) ]
-			egress ||= [ allow(:all) ]
+			ingress ||= [ allow_port(:all) ]
+			egress ||= [ allow_port(:all) ]
 			machine_tag ||= ref("AWS::StackName")
 			name ||= make_default_resource_name("AutoScalingGroup")
 			script ||= ""
@@ -393,7 +394,7 @@ aws ec2 associate-address --region `cat /etc/aws_region` --instance-id `curl htt
 								"logs:PutLogEvents"
 							],
 							"Resource": "*"
-						}]
+						}] + policies
 					}
 				}]
 			end

data/lib/sumomo/ecs.rb

@@ -6,21 +6,67 @@ module Sumomo
 			str.gsub(/[^0-9a-zA-Z]/, "_")
 		end
 
-		def make_ecs_cluster(name:"ECSCluster",services:[],machine_config:{},log_retention:30)
+		def make_ecs_role
+			make "AWS::IAM::Role", name: "ECSServiceRole" do
+
+				role_policy_doc = {
+					"Version" => "2012-10-17",
+					"Statement" => [{
+						"Effect" => "Allow",
+						"Principal" => {"Service" => ["ecs.amazonaws.com"]},
+						"Action" => ["sts:AssumeRole"]
+					}]
+				}
 
-			ecs = make "AWS::ECS::Cluster", name: "#{name}"
+				AssumeRolePolicyDocument role_policy_doc
+				Path "/"
+				Policies [
+					{
+						"PolicyName" => "ecs-service",
+						"PolicyDocument" => {
+							"Version" => "2012-10-17",
+							"Statement" => [{
+								"Effect" => "Allow",
+								"Action" => [
+									"ec2:AuthorizeSecurityGroupIngress",
+									"ec2:Describe*",
+									"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+									"elasticloadbalancing:DeregisterTargets",
+									"elasticloadbalancing:Describe*",
+									"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+									"elasticloadbalancing:RegisterTargets"
+        						],
+								"Resource" => "*"
+							}]
+						}
+					}
+				]
+			end
+		end
+
+		def make_ecs_cluster(name:make_default_resource_name("ECSCluster"),services:[],machine_config:{},network:,log_retention:30,dependencies:[])
+
+			ecs = make "AWS::ECS::Cluster", name: "#{name}" do
+				dependencies.each do |x|
+					depends_on x
+				end
+			end
 
 			volumes = []
 			machine_volume_locations = {}
 
-			service_count = 0
+			service_number = 0
 
 			services.each do |service|
 
-				service_count += 1
+				alb = service[:alb]
+				certificate = service[:certificate]
+				alb_ports = {}
+
+				service_number += 1
 
 				containers = service[:containers]
-				service_name = service[:name] || "Service#{service_count}"
+				service_name = service[:name] || "Service#{service_number}"
 				service_count = service[:count] || 1
 
 				container_defs = containers.map do |container|
@@ -29,7 +75,7 @@ module Sumomo
 					definition["Name"] = "#{sluggify(container[:image]).camelize}"
 					definition["Name"] = container[:name] if container[:name]
 
-					definition["Memory"] = 1024
+					definition["Memory"] = container[:memory] || 1024
 
 					loggroup = make "AWS::Logs::LogGroup", name: "#{name}#{definition["Name"]}Logs" do
 						LogGroupName "#{definition["Name"].underscore}_logs"
@@ -68,11 +114,68 @@ module Sumomo
 					end
 
 					if container[:ports]
-						definition["PortMappings"] = container[:ports].map do |from_port, to_port|
-							{
-								"ContainerPort" => from_port,
-								"HostPort" => to_port
-							}
+						if !alb
+							definition["PortMappings"] = container[:ports].map do |from_port, to_port|
+								{
+									"ContainerPort" => from_port,
+									"HostPort" => to_port
+								}
+							end
+
+						else
+							definition["PortMappings"] = container[:ports].map do |from_port, to_port|
+								{
+									"ContainerPort" => from_port
+								}
+							end
+
+							container[:ports].each do |container_port, host_port|
+
+								if alb_ports[host_port.to_i]
+									raise "Container #{alb_ports[host_port][:name]} is already using #{host_port}"
+								end
+
+								alb_target = make "AWS::ElasticLoadBalancingV2::TargetGroup", name: "#{name}#{definition["Name"]}Target" do
+									HealthCheckIntervalSeconds 60
+									UnhealthyThresholdCount 10
+									HealthCheckPath "/"
+									Name "#{name}Port#{host_port}Target"
+									Port container_port
+									Protocol "HTTP"
+									VpcId network[:vpc]
+								end
+
+								alb_action = {
+									"Type" => "forward",
+									"TargetGroupArn" => alb_target
+								}
+
+								if certificate
+									alb_listener = make "AWS::ElasticLoadBalancingV2::Listener", name: "#{name}#{definition["Name"]}Listener" do
+										Certificates [{ CertificateArn: certificate }]
+										DefaultActions [ alb_action ]
+										LoadBalancerArn alb
+										Port host_port
+										Protocol "HTTPS"
+									end
+								else
+									alb_listener = make "AWS::ElasticLoadBalancingV2::Listener", name: "#{name}#{definition["Name"]}Listener" do
+										DefaultActions [ alb_action ]
+										LoadBalancerArn alb
+										Port host_port
+										Protocol "HTTP"
+									end
+								end
+
+								alb_ports[host_port.to_i] = {
+									listener: alb_listener,
+									target: alb_target,
+									port: container_port,
+									name: definition["Name"]
+								}
+
+							end
+
 						end
 						container.delete(:ports)
 					end
@@ -87,13 +190,12 @@ module Sumomo
 						container.delete(:envvars)
 					end
 
-
 					container.each do |key, value|
 						definition["#{key}".camelize] = value
 					end
 
 					definition
-				end
+				end 
 
 
 				deployment_config = {
@@ -106,13 +208,31 @@ module Sumomo
 					Volumes volumes
 				end
 
+				stack = self
+
 				ecs_service = make "AWS::ECS::Service", name: "#{name}#{service_name}" do
+					alb_ports.each do |host_port, info|
+						depends_on info[:listener]
+					end
+
 					Cluster ecs
 					DesiredCount service_count
 					TaskDefinition ecs_task
 					DeploymentConfiguration deployment_config
+
+					if alb_ports.keys.count != 0
+						Role stack.make_ecs_role
+						LoadBalancers alb_ports.values.map { |info|
+							{
+								"TargetGroupArn" => info[:target],
+								"ContainerPort" => info[:port],
+								"ContainerName" => info[:name]
+							}
+						}
+					end
+
 				end
-			end
+			end #services
 
 			machine_config[:methods].each do |method_name|
 				parameters = {ecs_cluster: ecs}
@@ -121,6 +241,8 @@ module Sumomo
 					parameters[param[1]] = machine_config[param[1]] if (param[0] == :keyreq or param[0] == :key) and machine_config[param[1]]
 				end
 
+				parameters[:network] = network if !parameters[:network]
+
 				method(method_name).call(parameters) do
 					machine_volume_locations.each do |s3_loc, machine_loc|
 						mkdir File.dirname(machine_loc)

data/lib/sumomo/network.rb

@@ -38,9 +38,47 @@ module Sumomo
 				GatewayId gateway
 			end
 
+			last_unused_number = 0
+
+			subnet_numbers = []
+
+			#load current config
+			number_hash = {}
+			subnet_hash = {}
+
+			ec2 = Aws::EC2::Client.new(region: @region)
+			ec2_subnets = ec2.describe_subnets().subnets
+			ec2_subnets.each do |subnet|
+				if subnet.tags.select {|x| x.key == "aws:cloudformation:stack-name" && x.value == @bucket_name}.length == 1
+					layer = /^#{@bucket_name}-(?<layer_name>.+)-[a-z]+$/.match(subnet.tags.select{|x| x.key == "Name"}.first.value)[:layer_name]
+					zone = subnet.availability_zone
+					number = /^10.0.(?<num>[0-9]+).0/.match(subnet.cidr_block)[:num].to_i
+
+					key = "#{layer}/#{zone}"
+					number_hash[number] = key
+					subnet_hash[key] = number
+				end
+			end
+
+			# assign numbers to unassigned subnets
+			layers.product(zones).each do |e|
+				key = "#{e[0]}/#{e[1]}"
+				if !subnet_hash.has_key?(key)
+					loop do
+						break if !number_hash.has_key?(last_unused_number)
+						last_unused_number += 1
+					end
+					number_hash[last_unused_number] = key
+					subnet_hash[key] = last_unused_number
+					subnet_numbers << [e, last_unused_number]
+				else
+					subnet_numbers << [e, subnet_hash[key]]
+				end
+			end
+
 			subnets = {}
 
-			layers.product(zones).each_with_index do |e, subnet_number|
+			subnet_numbers.each do |e, subnet_number|
 				layer = e[0]
 				zone = e[1]
 

data/lib/sumomo/stack.rb

@@ -17,6 +17,7 @@ module Sumomo
 		end
 
 		def upload_file(name,content)
+			puts "Uploaded #{name}"
 			@store.set_raw("uploads/#{name}", content)
 		end
 
@@ -139,6 +140,11 @@ module Sumomo
 									"Effect" => "Allow",
 									"Action" => ["s3:DeleteObject", "s3:GetObject", "s3:PutObject"],
 									"Resource" => "arn:aws:s3:::#{bucket_name}/*"
+								},
+								{
+									"Effect" => "Allow",
+									"Action" => ["cloudfront:CreateCloudFrontOriginAccessIdentity", "cloudfront:DeleteCloudFrontOriginAccessIdentity"],
+									"Resource" => "*"
 								}]
 							}
 						}

data/lib/sumomo/version.rb

@@ -1,3 +1,3 @@
 module Sumomo
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sumomo
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - David Siaw
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-02 00:00:00.000000000 Z
+date: 2017-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - data/sumomo/custom_resources/AvailabilityZones.js
 - data/sumomo/custom_resources/CloudflareCNAME.js
 - data/sumomo/custom_resources/DeployTime.js
+- data/sumomo/custom_resources/OriginAccessIdentity.js
 - data/sumomo/custom_resources/SelectSpot.js
 - data/sumomo/sources/spot-watcher-poller.sh
 - data/sumomo/sources/spot-watcher.sh