awsdsl 0.0.1

data/lib/awsdsl/cfn_builder.rb

@@ -0,0 +1,360 @@
+require 'cfndsl'
+require 'netaddr'
+require 'awsdsl/cfn_helpers'
+
+module AWSDSL
+  class CfnBuilder
+    include CfnHelpers
+
+    def initialize(stack)
+      @stack = stack
+    end
+
+    def self.build(stack)
+      CfnBuilder.new(stack).build
+    end
+
+    def build
+      @t = CfnDsl::CloudFormationTemplate.new
+      stack = @stack
+      @t.declare do
+        Description stack.description
+      end
+      AWS.memoize do
+        build_vpcs
+        build_elasticaches
+        build_roles
+      end
+      @t
+    end
+
+    def build_roles
+      stack = @stack
+      stack.roles.each do |role|
+        role_name = role.name.capitalize
+        role_vpc = resolve_vpc(role.vpc)
+
+        # Create ELBs and appropriate security groups etc.
+        role.load_balancers.each do |lb|
+          listeners = lb.listeners.map { |l| format_listener(l) }
+          health_check = health_check_defaults(lb.health_check) if lb.health_check
+
+          lb_name = "#{role_name}#{lb.name.capitalize}ELB"
+          subnets = lb.subnets.empty? ? role.subnets : lb.subnets
+          lb_subnets = resolve_subnets(role.vpc, subnets)
+
+          # ELB
+          @t.declare do
+            LoadBalancer lb_name do
+              Listeners listeners
+              ConnectionSettings lb.connection_settings if lb.connection_settings
+              HealthCheck health_check if health_check
+              CrossZone true
+              Subnets lb_subnets
+              SecurityGroups [Ref("#{lb_name}SG")]
+            end
+          end
+
+          # ELB SG
+          @t.declare do
+            EC2_SecurityGroup "#{lb_name}SG" do
+              GroupDescription "#{lb.name.capitalize} ELB Security Group"
+              VpcId role_vpc
+              listeners.map { |l| l[:LoadBalancerPort] }.each do |port|
+                SecurityGroupIngress IpProtocol: 'tcp',
+                                     FromPort: port,
+                                     ToPort: port,
+                                     CidrIp: '0.0.0.0/0'
+              end
+            end
+          end
+
+          # ELB DNS records
+          lb.dns_records.each do |record|
+            zone = record[:zone] || get_zone_for_record(record[:name]).id
+            record_name = record[:name].split('.').map(&:capitalize).join
+            @t.declare do
+              RecordSet record_name do
+                HostedZoneId zone
+                Name record
+                Type 'A'
+                AliasTarget HostedZoneId: FnGetAtt(lb_name, 'CanonicalHostedZoneNameID'),
+                            DNSName: FnGetAtt(lb_name, 'CanonicalHostedZoneName')
+              end
+            end
+          end
+        end # end load_balancers
+
+        # IAM Role
+        @t.declare do
+          IAM_Role "#{role_name}Role" do
+            AssumeRolePolicyDocument Statement: [{
+              Effect: 'Allow',
+              Principal: {
+                Service: ['ec2.amazonaws.com']
+              },
+              Action: ['sts:AssumeRole']
+            }]
+            Path '/'
+          end
+        end
+
+        # Policy
+        statements = role.policy_statements.map { |s| format_policy_statement(s) }
+        if statements.count > 1
+          policy_name = "#{role_name}Policy"
+          @t.declare do
+            Policy policy_name do
+              PolicyName policy_name
+              PolicyDocument Statement: statements
+              Roles [Ref("#{role_name}Role")]
+            end
+          end
+        end
+
+        # Instance Profile
+        @t.declare do
+          InstanceProfile "#{role_name}InstanceProfile" do
+            Path '/'
+            Roles [Ref("#{role_name}Role")]
+          end
+        end
+
+        # Autoscaling Group
+        update_policy = update_policy_defaults(role)
+        lb_names = role.load_balancers.map { |lb| "#{role_name}#{lb.name.capitalize}ELB" }
+        subnets = resolve_subnets(role.vpc, role.subnets)
+        min = role.min_size || 0
+        max = role.max_size || 1
+        tgt = role.tgt_size || 1
+        @t.declare do
+          AutoScalingGroup "#{role_name}ASG" do
+            LaunchConfigurationName Ref("#{role.name.capitalize}LaunchConfig")
+            UpdatePolicy 'AutoScalingRollingUpdate', update_policy if update_policy
+            MinSize min
+            MaxSize max
+            DesiredCapacity tgt
+            LoadBalancerNames lb_names.map { |name| Ref(name) }
+            VPCZoneIdentifier subnets
+            AvailabiltityZones FnGetAZs('')
+          end
+        end
+
+        # Launch Configuration
+        security_groups = resolve_security_groups(role.vpc, role.security_groups)
+        block_devices = format_block_devices(role.block_devices)
+        @t.declare do
+          LaunchConfiguration "#{role_name}LaunchConfig" do
+            ImageId role.ami
+            # TODO(jpg): Should support NAT at some stage even though it's nasty on AWS
+            AssociatePublicIpAddress true
+            InstanceType role.instance_type
+            # TODO(jpg): Need to resolve this to IDs or Refs as necessary
+            SecurityGroups [Ref("#{role_name}SG")] + security_groups
+            IamInstanceProfile Ref("#{role_name}InstanceProfile")
+            BlockDeviceMappings block_devices
+            KeyName role.key_pair if role.key_pair
+          end
+        end
+
+        lb_ingress_rules = role.load_balancers.map do |lb|
+          lb.listeners.map do |l|
+            h = listener_defaults(l)
+            h[:sg] = "#{role_name}#{lb.name.capitalize}ELBSG"
+            h
+          end
+        end.flatten
+        # Security Group
+        @t.declare do
+          EC2_SecurityGroup "#{role_name}SG" do
+            GroupDescription "#{role_name} Security Group"
+            VpcId role_vpc
+            # TODO(jpg): Better way of offering up defaults
+            SecurityGroupIngress IpProtocol: 'tcp',
+                                 FromPort: 22,
+                                 ToPort: 22,
+                                 CidrIp: '0.0.0.0/0'
+
+            # Access from configured load_balancers
+            lb_ingress_rules.each do |rule|
+              SecurityGroupIngress IpProtocol: 'tcp',
+                                   FromPort: rule[:instance_port],
+                                   ToPort: rule[:instance_port],
+                                   SourceSecurityGroupId: Ref(rule[:sg])
+            end
+
+            # Access from other roles
+            # TODO(jpg): catch undefined roles before template generation
+            role.allows.select { |r| r[:role] != role.name }.each do |rule|
+              ports = rule[:ports].is_a?(Array) ? rule[:ports] : [rule[:ports]]
+              ports.each do |port|
+                SecurityGroupIngress IpProtocol: rule[:proto] || 'tcp',
+                                     FromPort: port,
+                                     ToPort: port,
+                                     SourceSecurityGroupId: Ref("#{rule[:role].capitalize}SG")
+              end
+            end
+          end
+
+          # Intracluster communication
+          role.allows.select { |r| r[:role] == role.name }.each do |rule|
+            ports = rule[:ports].is_a?(Array) ? rule[:ports] : [rule[:ports]]
+            proto = rule[:proto] || 'tcp'
+            ports.each do |port|
+              EC2_SecurityGroupIngress "#{role_name}SG#{proto.upcase}#{port}" do
+                GroupId Ref("#{role_name}SG")
+                IpProtocol proto
+                FromPort port
+                ToPort port
+                SourceSecurityGroupId Ref("#{role_name}SG")
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def build_elasticaches
+      default_ports = {
+        'redis' => 6379,
+        'memcached' => 11211
+      }
+      stack = @stack
+      stack.elasticaches.each do |cache|
+        # Default to Redis, also set default port if unset.
+        engine ||= 'redis'
+        port ||= default_ports[engine]
+        num_nodes ||= 1
+        cache_vpc = resolve_vpc(cache.vpc)
+        cache_name = "#{cache.name.capitalize}Cache"
+
+        # SG
+        @t.declare do
+          EC2_SecurityGroup "#{cache_name}SG" do
+            GroupDescription "#{cache.name.capitalize} Cache Security Group"
+            VpcId cache_vpc
+            cache.allows.each do |rule|
+              SecurityGroupIngress IpProtocol: 'tcp',
+                                   FromPort: port,
+                                   ToPort: port,
+                                   SourceSecurityGroupId: Ref("#{rule[:role].capitalize}SG")
+            end
+          end
+        end
+
+        # ElastiCacheSubnetGroup
+        cache_subnets = resolve_subnets(cache.vpc, cache.subnets)
+        @t.declare do
+          ElastiCache_SubnetGroup "#{cache_name}SubnetGroup" do
+            Description "SubnetGroup for #{cache_name}"
+            SubnetIds cache_subnets
+          end
+        end
+
+        # CacheCluster
+        @t.declare do
+          CacheCluster cache_name do
+            CacheNodeType cache.node_type
+            NumCacheNodes num_nodes
+            Engine engine
+            Port port
+            CacheSubnetGroupName Ref("#{cache_name}SubnetGroup")
+            VpcSecurityGroupIds [FnGetAtt("#{cache_name}SG", 'GroupId')]
+          end
+        end
+
+        # Add additional policy to each Role that can access this Cache
+        # This will allow said Role to discover the Cache nodes
+        cache.allows.each do |rule|
+          role = stack.roles.find { |r| r.name = rule[:role] }
+          role.policy_statement effect: 'Allow',
+                                action: 'elasticache:Describe*',
+                                resource: '*'
+        end
+      end
+    end
+
+    def build_vpcs
+      stack = @stack
+      stack.vpcs.each do |vpc|
+        igw = vpc.igw || true
+        dns = vpc.dns || true
+        cidr = vpc.cidr || '10.0.0.0/16'
+        subnet_bits = vpc.subnet_bits || 24
+        dns_hostnames = vpc.dns_hostnames || true
+
+        cidr = NetAddr::CIDR.create(cidr)
+        subnets = cidr.subnet(Bits: subnet_bits).to_enum
+
+        # VPC
+        vpc_name = "#{vpc.name.capitalize}VPC"
+        @t.declare do
+          VPC vpc_name do
+            CidrBlock cidr
+            EnableDnsSupport dns
+            EnableDnsHostnames dns_hostnames
+          end
+        end
+
+        if igw # Don't create internet facing stuff if igw is not enabled
+          igw_name = "#{vpc.name.capitalize}IGW"
+
+          # IGW
+          @t.declare do
+            InternetGateway igw_name
+          end
+
+          # Attach to VPC
+          @t.declare do
+            VPCGatewayAttachment "#{vpc.name.capitalize}GWAttachment" do
+              VpcId Ref(vpc_name)
+              InternetGatewayId Ref(igw_name)
+            end
+          end
+
+          # RouteTable
+          rt_name = "#{vpc.name.capitalize}RouteTable"
+          @t.declare do
+            RouteTable rt_name do
+              VpcId Ref(vpc_name)
+            end
+          end
+
+          # Default route for RouteTable
+          @t.declare do
+            Route "#{vpc.name.capitalize}DefaultRoute" do
+              RouteTableId Ref(rt_name)
+              DestinationCidrBlock '0.0.0.0/0'
+              GatewayId Ref(igw_name)
+              # TODO(jpg): DependsOn rt_name
+            end
+          end
+        end
+
+        vpc.subnets.each do |subnet|
+          subnet_igw = subnet.igw || igw
+          azs = subnet.azs || fetch_availability_zones(vpc.region)
+          subnet_name = "#{vpc.name.capitalize}#{subnet.name.capitalize}Subnet"
+          azs.each do |az|
+            subnet_name_az = "#{subnet_name}#{az.capitalize}"
+            @t.declare do
+              Subnet subnet_name_az do
+                AvailabilityZone "#{vpc.region}#{az}"
+                CidrBlock subnets.next
+                VpcId Ref(vpc_name)
+              end
+
+              if subnet_igw
+                SubnetRouteTableAssociation "#{subnet_name_az}DefaultRTAssoc" do
+                  SubnetId Ref(subnet_name_az)
+                  RouteTableId Ref(rt_name)
+                  # TODO(jpg): DependsOn rt_name
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/awsdsl/cfn_helpers.rb

@@ -0,0 +1,131 @@
+require 'aws'
+require 'cfndsl'
+
+module AWSDSL
+  module CfnHelpers
+    include CfnDsl::Functions
+
+    def listener_defaults(listener)
+      listener[:proto] ||= 'HTTP'
+      listener[:instance_port] ||= listener[:port]
+      listener[:loadbalancer_port] ||= listener[:port]
+      listener
+    end
+
+    def format_listener(listener)
+      listener = listener_defaults(listener)
+      hash = {
+        LoadBalancerPort: listener[:loadbalancer_port],
+        InstancePort: listener[:instance_port],
+        Protocol: listener[:proto]
+      }
+      hash[:SSLCertificateId] = listener[:cert] if listener[:cert]
+      hash
+    end
+
+    def health_check_defaults(health_check)
+      hash = {
+        Target: health_check[:target]
+      }
+      hash[:HealthyThreshold] = health_check[:healthy_threshold] || 3
+      hash[:UnhealthyThreshold] = health_check[:unhealthy_threshold] || 5
+      hash[:Interval] = health_check[:interval] || 90
+      hash[:Timeout] = health_check[:timeout] || 60
+      hash
+    end
+
+    def update_policy_defaults(role)
+      update_policy = role.update_policy || {}
+      return nil if update_policy[:disable] == true
+      hash = {}
+      hash[:MaxBatchSize] = update_policy[:max_batch] || 1
+      hash[:MinInstancesInService] = update_policy[:min_inservice] || role.min_size
+      hash[:PauseTime] = update_policy[:pause_time] if update_policy[:pause_time]
+    end
+
+    def format_policy_statement(policy_statement)
+      Hash[policy_statement.map { |k, v| [k.to_s.capitalize.to_sym, v] }]
+    end
+
+    def format_block_devices(devices)
+      devices.map do |dev|
+        h = { DeviceName: dev[:name] }
+        if dev[:ephemeral]
+          h[:VirtualName] = "ephemeral#{dev[:ephemeral]}"
+        else
+          h[:Ebs] = {
+            VolumeSize: dev[:size],
+            VolumeType: dev[:type] || 'gp2'
+          }
+        end
+        h
+      end
+    end
+
+    def get_zone_for_record(name)
+      r53 = AWS::Route53.new
+      zones = r53.hosted_zones.sort_by { |z| z.name.split('.').count }.reverse
+      zones.find do |z|
+        name.split('.').reverse.take(z.name.split('.').count) == z.name.split('.').reverse
+      end
+    end
+
+    def get_vpc_by_name(vpc)
+      ec2 = AWS::EC2.new
+      ec2.vpcs.with_tag('Name', vpc).first
+    end
+
+    def resolve_vpc(vpc)
+      return vpc if vpc.start_with?('vpc-')
+      return Ref("#{vpc.capitalize}VPC") if vpc_defined?(vpc)
+      get_vpc_by_name(vpc).id
+    end
+
+    def resolve_subnets(vpc, subnets)
+      subnets.map do |subnet|
+        resolve_subnet(vpc, subnet)
+      end.flatten(1)
+    end
+
+    def subnet_refs(vpc, subnet)
+      vpc = @stack.vpcs.find {|v| v.name == vpc }
+      subnet = vpc.subnets.find {|s| s.name == subnet}
+      subnet_name = "#{vpc.name.capitalize}#{subnet.name.capitalize}Subnet"
+      azs = subnet.azs || fetch_availability_zones(vpc.region)
+      azs.map do |az|
+        Ref("#{subnet_name}#{az.capitalize}")
+      end
+    end
+
+    def subnet_defined?(vpc, subnet)
+      @stack.vpcs.find {|v| v.name == vpc }.subnets.map(&:name).include?(subnet)
+    end
+
+    def vpc_defined?(vpc)
+      @stack.vpcs.map(&:name).include?(vpc)
+    end
+
+    def resolve_subnet(vpc, subnet)
+      return [subnet] if subnet.start_with?('subnet-')
+      return subnet_refs(vpc, subnet) if subnet_defined?(vpc, subnet)
+      ec2 = AWS::EC2.new
+      v = ec2.vpcs[vpc] if vpc.start_with?('vpc-')
+      v ||= get_vpc_by_name(vpc)
+      v.subnets.with_tag('Name', subnet).map(&:id)
+    end
+
+    def resolve_security_groups(vpc, security_groups)
+      security_groups.map do |sg|
+        resolve_security_group(vpc, sg)
+      end.flatten
+    end
+
+    def resolve_security_group(vpc, sg)
+      return [sg] if sg.start_with?('sg-')
+      ec2 = AWS::EC2.new
+      v = ec2.vpcs[vpc] if vpc.start_with?('vpc-')
+      v ||= get_vpc_by_name(vpc)
+      v.security_groups.with_tag('Name', sg).map(&:id)
+    end
+  end
+end

data/lib/awsdsl/command_line.rb

@@ -0,0 +1,48 @@
+require 'clamp'
+require 'awsdsl'
+
+module AWSDSL
+  class CommandLine < Clamp::Command
+    option '--stackfile',
+      'STACKFILE',
+      'Path to Stackfile',
+      default: 'Stackfile'
+
+    subcommand 'build', 'Build Stack AMIs' do
+      def execute
+        stack = Loader.load(stackfile)
+        AMIBuilder.build(stack)
+      end
+    end
+
+    subcommand 'create', 'Create Stack' do
+      def execute
+        stack = Loader.load(stackfile)
+        stack = AMIBuilder.latest_amis(stack)
+        template = CfnBuilder.build(stack)
+        cfm = AWS::CloudFormation.new
+        cfm.stacks.create(stack.name.capitalize, template,
+                          capabilities: ['CAPABILITY_IAM'])
+      end
+    end
+
+    subcommand 'update', 'Update Stack' do
+      def execute
+        stack = Loader.load(stackfile)
+        stack = AMIBuilder.latest_amis(stack)
+        template = CfnBuilder.build(stack)
+        cfm = AWS::CloudFormation.new
+        cfm.stacks[stack.name.capitalize].update(template: template,
+                                                 capabilities: ['CAPABILITY_IAM'])
+      end
+    end
+
+    subcommand 'delete', 'Delete Stack' do
+      def execute
+        stack = Loader.load(stackfile)
+        cfm = AWS::CloudFormation.new
+        cfm.stacks[stack.name.capitalize].delete
+      end
+    end
+  end
+end

data/lib/awsdsl/dsl/elasticache.rb

@@ -0,0 +1,7 @@
+module AWSDSL
+  class Elasticache
+    include DSL
+    multi_attributes :allow, :subnet
+    attributes :engine, :node_type, :port, :num_nodes, :vpc
+  end
+end

data/lib/awsdsl/dsl/load_balancer.rb

@@ -0,0 +1,7 @@
+module AWSDSL
+  class LoadBalancer
+    include DSL
+    multi_attributes :listener, :dns_record, :security_group, :subnet
+    attributes :health_check, :internal, :connection_settings
+  end
+end

data/lib/awsdsl/dsl/role.rb

@@ -0,0 +1,26 @@
+module AWSDSL
+  class Role
+    include DSL
+    attr_accessor :ami
+
+    sub_components :load_balancer
+    multi_attributes :policy_statement,
+                     :include_profile,
+                     :security_group,
+                     :block_device,
+                     :file_provisioner,
+                     :chef_provisioner,
+                     :ansible_provisioner,
+                     :subnet,
+                     :allow
+    attributes :min_size,
+               :max_size,
+               :tgt_size,
+               :update_policy,
+               :instance_type,
+               :vpc,
+               :base_ami,
+               :vars,
+               :key_pair
+  end
+end

data/lib/awsdsl/dsl/role_profile.rb

@@ -0,0 +1,31 @@
+module AWSDSL
+  class RoleProfile < Role
+    include DSL
+    attr_accessor :block
+
+    sub_components :load_balancer
+    multi_attributes :policy_statement,
+                     :include_profile,
+                     :security_group,
+                     :block_device,
+                     :file_provisioner,
+                     :chef_provisioner,
+                     :ansible_provisioner,
+                     :subnet,
+                     :allow
+    attributes :min_size,
+               :max_size,
+               :tgt_size,
+               :update_policy,
+               :instance_type,
+               :vpc,
+               :base_ami,
+               :vars,
+               :key_pair
+
+    def initialize(name, &block)
+      @block = block if block_given?
+      super
+    end
+  end
+end

data/lib/awsdsl/dsl/stack.rb

@@ -0,0 +1,16 @@
+module AWSDSL
+  class Stack
+    include DSL
+    attributes :description, :vars
+    sub_components :role, :role_profile, :elasticache, :vpc
+
+    def mixin_profiles
+      @roles.each do |role|
+        role.include_profiles.each do |profile|
+          role_profile = @role_profiles.find { |rp| rp.name == profile }
+          role_profile.block.bind(role).call
+        end
+      end
+    end
+  end
+end

data/lib/awsdsl/dsl/subnet.rb

@@ -0,0 +1,7 @@
+module AWSDSL
+  class Subnet
+    include DSL
+    attributes :igw
+    multi_attributes :az
+  end
+end

data/lib/awsdsl/dsl/vpc.rb

@@ -0,0 +1,7 @@
+module AWSDSL
+  class Vpc
+    include DSL
+    sub_components :subnet
+    attributes :cidr, :dns, :dns_hostnames, :igw, :region, :subnet_bits
+  end
+end