rubycfn 0.4.10 → 0.5.0

data/templates/lib/stacks/ecs_stack/ecs_cluster.rb

@@ -0,0 +1,344 @@
+module EcsStack
+  module EcsCluster
+    extend ActiveSupport::Concern
+    included do
+      parameter :vpc,
+                description: "VPC ID"
+      parameter :subnets,
+                description: "Subnets for ECS"
+      parameter :ecs_ami,
+                description: "ECS-Optimized AMI ID",
+                type: "AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>",
+                default: "/aws/service/ecs/optimized-ami/amazon-linux/recommended/image_id"
+
+      variable :cluster_size,
+               default: "1",
+               value: infra_config["environments"][environment]["cluster_size"]
+
+      variable :cluster_instance_type,
+               default: "t2.micro",
+               value: infra_config["environments"][environment]["cluster_instance_type"]
+
+      resource :ecs_cluster,
+               type: "AWS::ECS::Cluster"
+
+      resource :ecs_auto_scaling_group,
+               type: "AWS::AutoScaling::AutoScalingGroup" do |r|
+        r.property(:vpc_zone_identifier) { :subnets.ref.fnsplit(",") }
+        r.property(:launch_configuration_name) { :ecs_launch_configuration.ref }
+        r.property(:min_size) { cluster_size.to_i }
+        r.property(:max_size) { cluster_size.to_i }
+        r.property(:desired_capacity) { cluster_size.to_i }
+        r.property(:tags) do
+          [
+            {
+              "Key": "Name",
+              "Value": "#{environment} ECS host",
+              "PropagateAtLaunch": true
+            }
+          ]
+        end
+      end
+
+      resource :ecs_launch_configuration,
+               metadata: {
+                 "AWS::CloudFormation::Init": {
+                   "config": {
+                     "packages": {
+                       "yum": {
+                         "collectd": []
+                       }
+                     },
+                     "commands": {
+                       "01_add_instance_to_cluster": {
+                         "command": "echo ECS_CLUSTER=${SfsEcsCluster} >> /etc/ecs/ecs.config".fnsub
+                       },
+                       "02_enable_cloudwatch_agent": {
+                         "command": "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c ssm:${EcsCloudWatchParameter} -s".fnsub
+                       }
+                     },
+                     "files": {
+                       "/etc/cfn/cfn-hup.conf": {
+                         "mode": 256,
+                         "owner": "root",
+                         "group": "root",
+                         "content": "[main]\nstack=${AWS::StackId}\nregion=${AWS::Region}\n".fnsub
+                       },
+                       "/etc/cfn/hooks.d/cfn-auto-reloader.conf": {
+                         "content": "[cfn-auto-reloader-hook]\ntriggers=post.update\npath=Resources.EcsLaunchConfiguration.Metadata.AWS::CloudFormation::Init\naction=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource EcsLaunchConfiguration\n".fnsub
+                       }
+                     },
+                     "services": {
+                       "sysvinit": {
+                         "cfn-hup": {
+                           "enabled": true,
+                           "ensureRunning": true,
+                           "files": [
+                             "/etc/cfn/cfn-hup.conf",
+                             "/etc/cfn/hooks.d/cfn-auto-reloader.conf"
+                           ]
+                         }
+                       }
+                     }
+                   }
+                 }
+               },
+               type: "AWS::AutoScaling::LaunchConfiguration" do |r|
+        r.property(:image_id) { :ecs_ami.ref }
+        r.property(:key_name) { "sfs-keypair-#{environment}" }
+        r.property(:instance_type) { cluster_instance_type }
+        r.property(:security_groups) do
+          [
+            :ecs_host_security_group.ref,
+            :load_balancer_security_group.ref # Not sure if necessary
+          ]
+        end
+        r.property(:iam_instance_profile) { :ecs_instance_profile.ref }
+        r.property(:user_data) { "#!/bin/bash\nyum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm\nyum install -y https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm\nyum install -y aws-cfn-bootstrap hibagent \n/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource EcsLaunchConfiguration\n/opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource EcsAutoScalingGroup\n/usr/bin/enable-ec2-spot-hibernation\n".fnsub.fnbase64 }
+      end
+
+      ecs_ssm_parameter_value = {
+        "logs": {
+          "force_flush_interval": 5,
+          "logs_collected": {
+            "files": {
+              "collect_list": [
+                {
+                  "file_path": "/var/log/messages",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/messages",
+                  "log_stream_name": "{instance_id}",
+                  "timestamp_format": "%b %d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/dmesg",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/dmesg",
+                  "log_stream_name": "{instance_id}"
+                },
+                {
+                  "file_path": "/var/log/docker",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/docker",
+                  "log_stream_name": "{instance_id}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
+                },
+                {
+                  "file_path": "/var/log/ecs/ecs-init.log",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/ecs/ecs-init.log",
+                  "log_stream_name": "{instance_id}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ"
+                },
+                {
+                  "file_path": "/var/log/ecs/ecs-agent.log.*",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/ecs/ecs-agent.log",
+                  "log_stream_name": "{instance_id}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ"
+                },
+                {
+                  "file_path": "/var/log/ecs/audit.log",
+                  "log_group_name": "${SfsEcsCluster}-/var/log/ecs/audit.log",
+                  "log_stream_name": "{instance_id}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ"
+                }
+              ]
+            }
+          }
+        },
+        "metrics": {
+          "append_dimensions": {
+            "AutoScalingGroupName": "${!aws:AutoScalingGroupName}",
+            "InstanceId": "${!aws:InstanceId}",
+            "InstanceType": "${!aws:InstanceType}"
+          },
+          "metrics_collected": {
+            "collectd": {
+              "metrics_aggregation_interval": 60
+            },
+            "disk": {
+              "measurement": [
+                "used_percent"
+              ],
+              "metrics_collection_interval": 60,
+              "resources": [
+                "/"
+              ]
+            },
+            "mem": {
+              "measurement": [
+                "mem_used_percent"
+              ],
+              "metrics_collection_interval": 60
+            },
+            "statsd": {
+              "metrics_aggregation_interval": 60,
+              "metrics_collection_interval": 10,
+              "service_address": ":8125"
+            }
+          }
+        }
+      }
+
+      resource :ecs_cloud_watch_parameter,
+               type: "AWS::SSM::Parameter" do |r|
+        r.property(:description) { "ECS" }
+        r.property(:name) { "AmazonCloudWatch-${SfsEcsCluster}-ECS".fnsub }
+        r.property(:type) { "String" }
+        r.property(:value) { JSON.pretty_generate(ecs_ssm_parameter_value).fnsub }
+      end
+
+      resource :ecs_host_security_group,
+               type: "AWS::EC2::SecurityGroup" do |r|
+        r.property(:vpc_id) { :vpc.ref }
+        r.property(:group_description) { "Access to the ECS hosts and the tasks/containers that run on them" }
+        r.property(:security_group_ingress) do
+          [
+            {
+              "SourceSecurityGroupId": :load_balancer_security_group.ref,
+              "IpProtocol": -1
+            }
+          ]
+        end
+        r.property(:tags) do
+          [
+            {
+              "Key": "Name",
+              "Value": "#{environment}-ECS-Hosts"
+            }
+          ]
+        end
+      end
+
+      resource :load_balancer_security_group,
+               type: "AWS::EC2::SecurityGroup" do |r|
+        r.property(:vpc_id) { :vpc.ref }
+        r.property(:group_description) { "Access to the load balancer that sits in front of ECS" }
+        r.property(:security_group_ingress) do
+          [
+            {
+              "CidrIp": "0.0.0.0/0",
+              "IpProtocol": -1
+            }
+          ]
+        end
+        r.property(:tags) do
+          [
+            {
+              "Key": "Name",
+              "Value": "#{environment}-ECS-LoadBalancers"
+            }
+          ]
+        end
+      end
+
+      ecs_role_assume_role_policy_document = {
+        "Statement": [
+          {
+            "Action": "sts:AssumeRole",
+            "Effect": "Allow",
+            "Principal": {
+              "Service": "ec2.amazonaws.com"
+            }
+          }
+        ]
+      }
+
+      ecs_role_policy_document = {
+        "Statement": [{
+          "Effect": "Allow",
+          "Action": [
+            "ecs:CreateCluster",
+            "ecs:DeregisterContainerInstance",
+            "ecs:DiscoverPollEndpoint",
+            "ecs:Poll",
+            "ecs:RegisterContainerInstance",
+            "ecs:StartTelemetrySession",
+            "ecs:Submit*",
+            "ecr:BatchCheckLayerAvailability",
+            "ecr:BatchGetImage",
+            "ecr:GetDownloadUrlForLayer",
+            "ecr:GetAuthorizationToken"
+          ],
+          "Resource": "*"
+        }]
+      }
+
+      resource :ecs_role,
+               type: "AWS::IAM::Role" do |r|
+        r.property(:path) { "/" }
+        r.property(:role_name) { "#{environment}-ECSRole-${AWS::Region}".fnsub }
+        r.property(:assume_role_policy_document) { JSON.pretty_generate(ecs_role_assume_role_policy_document) }
+        r.property(:managed_policy_arns) do
+          [
+            "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM",
+            "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy",
+            "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
+          ]
+        end
+        r.property(:policies) do
+          [
+            {
+              "PolicyName": "ecs-service",
+              "PolicyDocument": JSON.pretty_generate(ecs_role_policy_document)
+            }
+          ]
+        end
+      end
+
+      resource :ecs_instance_profile,
+               type: "AWS::IAM::InstanceProfile" do |r|
+        r.property(:path) { "/" }
+        r.property(:roles) do
+          [
+            :ecs_role.ref
+          ]
+        end
+      end
+
+      resource :ecs_service_auto_scaling_role,
+               type: "AWS::IAM::Role" do |r|
+        r.property(:assume_role_policy_document) do
+          {
+            "Version": "2012-10-17",
+            "Statement": {
+              "Action": [
+                "sts:AssumeRole"
+              ],
+              "Effect": "Allow",
+              "Principal": {
+                "Service": [
+                  "application-autoscaling.amazonaws.com"
+                ]
+              }
+            }
+          }
+        end
+        r.property(:path) { "/" }
+        r.property(:policies) do
+          [
+            {
+              "PolicyName": "ecs-service-autoscaling",
+              "PolicyDocument": {
+                "Statement": {
+                  "Effect": "Allow",
+                  "Action": [
+                    "application-autoscaling:*",
+                    "cloudwatch:DescribeAlarms",
+                    "cloudwatch:PutMetricAlarm",
+                    "ecs:DescribeServices",
+                    "ecs:UpdateService"
+                  ],
+                  "Resource": "*"
+                }
+              }
+            }
+          ]
+        end
+      end
+
+      output :ecs_cluster,
+             value: :ecs_cluster.ref
+      output :ecs_cluster_arn,
+             value: :ecs_cluster.ref(:arn)
+      output :ecs_auto_scaling_role_arn,
+             value: :ecs_service_auto_scaling_role.ref(:arn)
+    end
+  end
+end

data/templates/lib/stacks/ecs_stack/lifecycle_hook.rb

@@ -0,0 +1,188 @@
+module EcsStack
+  module LifecycleHook
+    extend ActiveSupport::Concern
+    ## Drains instances on termination
+    included do
+      resource :ecs_lifecycle_notification_topic,
+               depends_on: :lifecycle_handler_function,
+               type: "AWS::SNS::Topic" do |r|
+        r.property(:subscription) do
+          [
+            {
+              "Endpoint": :lifecycle_handler_function.ref(:arn),
+              "Protocol": "lambda"
+            }
+          ]
+        end
+      end
+
+      resource :instance_terminating_hook,
+               depends_on: :ecs_lifecycle_notification_topic,
+               type: "AWS::AutoScaling::LifecycleHook" do |r|
+        r.property(:auto_scaling_group_name) { :ecs_auto_scaling_group.ref }
+        r.property(:default_result) { "ABANDON" }
+        r.property(:heartbeat_timeout) { 900 }
+        r.property(:lifecycle_transition) { "autoscaling:EC2_INSTANCE_TERMINATING" }
+        r.property(:notification_target_arn) { :ecs_lifecycle_notification_topic.ref }
+        r.property(:role_arn) { :autoscaling_notification_role.ref(:arn) }
+      end
+
+      resource :autoscaling_notification_role,
+               type: "AWS::IAM::Role" do |r|
+        r.property(:assume_role_policy_document) do
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Principal": {
+                  "Service": [
+                    "autoscaling.amazonaws.com"
+                  ]
+                },
+                "Action": [
+                  "sts:AssumeRole"
+                ]
+              }
+            ]
+          }
+        end
+        r.property(:managed_policy_arns) { ["arn:aws:iam::aws:policy/service-role/AutoScalingNotificationAccessRole"] }
+      end
+
+      resource :lambda_execution_role,
+               type: "AWS::IAM::Role" do |r|
+        r.property(:policies) do
+          [
+            {
+              "PolicyName": "lambda-inline",
+              "PolicyDocument": {
+                "Version": "2012-10-17",
+                "Statement": [
+                  {
+                    "Effect": "Allow",
+                    "Action": [
+                      "autoscaling:CompleteLifecycleAction",
+                      "logs:CreateLogGroup",
+                      "logs:CreateLogStream",
+                      "logs:PutLogEvents",
+                      "ec2:DescribeInstances",
+                      "ec2:DescribeInstanceAttribute",
+                      "ec2:DescribeInstanceStatus",
+                      "ec2:DescribeHosts",
+                      "ecs:ListContainerInstances",
+                      "ecs:SubmitContainerStateChange",
+                      "ecs:SubmitTaskStateChange",
+                      "ecs:DescribeContainerInstances",
+                      "ecs:UpdateContainerInstancesState",
+                      "ecs:ListTasks",
+                      "ecs:DescribeTasks",
+                      "sns:Publish",
+                      "sns:ListSubscriptions"
+                    ],
+                    "Resource": "*"
+                  }
+                ]
+              }
+            }
+          ]
+        end
+        r.property(:assume_role_policy_document) do
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Principal": {
+                  "Service": [
+                    "lambda.amazonaws.com"
+                  ]
+                },
+                "Action": [
+                  "sts:AssumeRole"
+                ]
+              }
+            ]
+          }
+        end
+        r.property(:managed_policy_arns) do
+          [
+            "arn:aws:iam::aws:policy/service-role/AutoScalingNotificationAccessRole"
+          ]
+        end
+      end
+
+      resource :lambda_invoke_permission,
+               type: "AWS::Lambda::Permission" do |r|
+        r.property(:function_name) { :lifecycle_handler_function.ref }
+        r.property(:action) { "lambda:InvokeFunction" }
+        r.property(:principal) { "sns.amazonaws.com" }
+        r.property(:source_arn) { :ecs_lifecycle_notification_topic.ref }
+      end
+
+      resource :lifecycle_handler_function,
+               type: "AWS::Lambda::Function" do |r|
+        r.property(:environment) do
+          {
+            "Variables": {
+              "CLUSTER": :sfs_ecs_cluster.ref
+            }
+          }
+        end
+        r.property(:code) do
+          {
+            "ZipFile": {
+              "Fn::Join": [
+                "\n",
+                [
+                  "import boto3,json,os,time",
+                  "ec2Client = boto3.client('ec2')",
+                  "ecsClient = boto3.client('ecs')",
+                  "autoscalingClient = boto3.client('autoscaling')",
+                  "snsClient = boto3.client('sns')",
+                  "lambdaClient = boto3.client('lambda')",
+                  "def publishSNSMessage(snsMessage,snsTopicArn):",
+                  "    response = snsClient.publish(TopicArn=snsTopicArn,Message=json.dumps(snsMessage),Subject='reinvoking')",
+                  "def setContainerInstanceStatusToDraining(ecsClusterName,containerInstanceArn):",
+                  "    response = ecsClient.update_container_instances_state(cluster=ecsClusterName,containerInstances=[containerInstanceArn],status='DRAINING')",
+                  "def tasksRunning(ecsClusterName,ec2InstanceId):",
+                  "    ecsContainerInstances = ecsClient.describe_container_instances(cluster=ecsClusterName,containerInstances=ecsClient.list_container_instances(cluster=ecsClusterName)['containerInstanceArns'])['containerInstances']",
+                  "    for i in ecsContainerInstances:",
+                  "        if i['ec2InstanceId'] == ec2InstanceId:",
+                  "            if i['status'] == 'ACTIVE':",
+                  "                setContainerInstanceStatusToDraining(ecsClusterName,i['containerInstanceArn'])",
+                  "                return 1",
+                  "            if (i['runningTasksCount']>0) or (i['pendingTasksCount']>0):",
+                  "                return 1",
+                  "            return 0",
+                  "    return 2",
+                  "def lambda_handler(event, context):",
+                  "    ecsClusterName=os.environ['CLUSTER']",
+                  "    snsTopicArn=event['Records'][0]['Sns']['TopicArn']",
+                  "    snsMessage=json.loads(event['Records'][0]['Sns']['Message'])",
+                  "    lifecycleHookName=snsMessage['LifecycleHookName']",
+                  "    lifecycleActionToken=snsMessage['LifecycleActionToken']",
+                  "    asgName=snsMessage['AutoScalingGroupName']",
+                  "    ec2InstanceId=snsMessage['EC2InstanceId']",
+                  "    checkTasks=tasksRunning(ecsClusterName,ec2InstanceId)",
+                  "    if checkTasks==0:",
+                  "        try:",
+                  "            response = autoscalingClient.complete_lifecycle_action(LifecycleHookName=lifecycleHookName,AutoScalingGroupName=asgName,LifecycleActionToken=lifecycleActionToken,LifecycleActionResult='CONTINUE')",
+                  "        except BaseException as e:",
+                  "            print(str(e))",
+                  "    elif checkTasks==1:",
+                  "        time.sleep(5)",
+                  "        publishSNSMessage(snsMessage,snsTopicArn)"
+                ]
+              ]
+            }
+          }
+        end
+        r.property(:handler) { "index.lambda_handler" }
+        r.property(:role) { :lambda_execution_role.ref(:arn) }
+        r.property(:runtime) { "python3.6" }
+        r.property(:timeout) { 10 }
+      end
+    end
+  end
+end

data/templates/lib/stacks/ecs_stack/load_balancer.rb

@@ -0,0 +1,68 @@
+module EcsStack
+  module LoadBalancer
+    extend ActiveSupport::Concern
+    included do
+      parameter :public_subnets,
+                description: "List of Public EC2 Subnets for the ALB"
+
+      resource :ecs_load_balancer,
+               type: "AWS::ElasticLoadBalancingV2::LoadBalancer" do |r|
+        r.property(:name) { environment }
+        r.property(:subnets) { :public_subnets.ref.fnsplit(",") }
+        r.property(:security_groups) do
+          [
+            :ecs_host_security_group.ref,
+            :load_balancer_security_group.ref # Not sure if necessary
+          ]
+        end
+        r.property(:tags) do
+          [
+            {
+              "Key": "Name",
+              "Value": "#{environment}_ecs_loadbalancer"
+            }
+          ]
+        end
+      end
+
+      resource :load_balancer_listener,
+               type: "AWS::ElasticLoadBalancingV2::Listener" do |r|
+        r.property(:load_balancer_arn) { :ecs_load_balancer.ref }
+        r.property(:port) { 80 }
+        r.property(:protocol) { "HTTP" }
+        r.property(:default_actions) do
+          [
+            {
+              "Type": "forward",
+              "TargetGroupArn": :default_target_group.ref
+            }
+          ]
+        end
+      end
+
+      resource :default_target_group,
+               type: "AWS::ElasticLoadBalancingV2::TargetGroup" do |r|
+        r.property(:name) { "#{environment}-default" }
+        r.property(:vpc_id) { :vpc.ref }
+        r.property(:port) { 80 }
+        r.property(:protocol) { "HTTP" }
+      end
+
+      output :ecs_load_balancer,
+             description: "ECS Application Load Balancer",
+             value: :ecs_load_balancer.ref
+
+      output :ecs_load_balancer_url,
+             description: "URL of the ECS ALB",
+             value: :ecs_load_balancer.ref("DNSName")
+
+      output :ecs_load_balancer_listener,
+             description: "ECS Port 80 listener",
+             value: :load_balancer_listener.ref
+
+      output :ecs_load_balancer_hosted_zone_id,
+             description: "Canonical Hosted Zone ID of the ALB",
+             value: :ecs_load_balancer.ref("CanonicalHostedZoneID")
+    end
+  end
+end

data/templates/{ecs_stack.rb.erb → lib/stacks/ecs_stack/main.rb}

@@ -1,11 +1,12 @@
 module EcsStack
   extend ActiveSupport::Concern
   include Rubycfn
-
   included do
     include Concerns::GlobalVariables
     include Concerns::SharedMethods
     include EcsStack::EcsCluster
+    include EcsStack::LifecycleHook
+    include EcsStack::LoadBalancer
 
     description generate_stack_description("EcsStack")
   end

data/templates/{project_stack.rb.erb → lib/stacks/parent_stack/main.rb}

@@ -1,11 +1,11 @@
-module <%= name %>Stack
+module InfraStack
   extend ActiveSupport::Concern
   include Rubycfn
 
   included do
     include Concerns::GlobalVariables
     include Concerns::SharedMethods
-    include <%= name %>Stack::Parent
+    include InfraStack::Parent
 
     description generate_stack_description("ParentStack")
   end

data/templates/lib/stacks/parent_stack/parent.rb

@@ -0,0 +1,18 @@
+require_relative "../../core/applications"
+
+module InfraStack
+  module Parent
+    extend ActiveSupport::Concern
+    included do
+      generate_bootstrap_parameters
+
+      resource :vpc_stack,
+               type: "AWS::CloudFormation::Stack" do |r|
+        r.property(:template_url) { "vpcstack" }
+        r.property(:tags) { default_tags }
+      end
+
+      create_applications
+    end
+  end
+end