convection 0.0.1 → 0.2.1

data/example/security-groups.rb

@@ -0,0 +1,40 @@
+require_relative '../lib/convection'
+
+module Convection
+  module Demo
+    SECURITY_GROUPS = Convection.template do
+      description 'Demo Security Groups'
+
+      ec2_security_group 'FoobarELB' do
+        vpc stack.get('vpc', 'id')
+        description 'Foobar ELB Ingress'
+
+        ingress_rule(:tcp, 80, '0.0.0.0/0')
+        ingress_rule(:tcp, 443, '0.0.0.0/0')
+
+        tag 'Name', "sg-foobar-elb-#{ stack.cloud }"
+        tag 'Service', 'foobar'
+        tag 'Resource', 'ELB'
+        tag 'Scope', 'public'
+        tag 'Stack', stack.cloud
+
+        with_output
+      end
+
+      ec2_security_group 'Foobar' do
+        vpc stack.get('vpc', 'id')
+        description 'Foobar Ingress'
+
+        ingress_rule(:tcp, 8080) { source_group fn_ref('FoobarELB') }
+
+        tag 'Name', "sg-foobar-#{ stack.cloud }"
+        tag 'Service', 'foobar'
+        tag 'Resource', 'EC2'
+        tag 'Scope', 'private'
+        tag 'Stack', stack.cloud
+
+        with_output
+      end
+    end
+  end
+end

data/example/trust_cloudtrail.rb

@@ -0,0 +1,24 @@
+require 'convection'
+
+module CLOUDTRAIL
+ #IAM role to create a log stream & put events
+  iam_role 'role' do
+    path "/"
+    #defines trust relationship
+    trust_cloudtrail
+
+    policy 'CreateStreamPolicy' do
+      allow do
+        resource 'arn:aws:logs:*:*:*'
+        action 'logs:CreateLogStream'
+      end
+    end
+
+    policy 'PutEventsPolicy' do
+      allow do
+        resource 'arn:aws:logs:*:*:*'
+        action 'logs:PutLogEvents'
+      end
+    end
+  end
+end

data/example/vpc.rb

@@ -1,85 +1,67 @@
-#!/usr/bin/env ruby
-require 'convection'
-
-test_template = Convection.template do
-  description 'This is a test stack generated with Convection'
-
-  parameter 'InstanceSize' do
-    type 'String'
-    description 'Instance Size'
-    default 'm3.medium'
-
-    allow 'm3.medium'
-    allow 'm3.large'
-    allow 'm3.xlarge'
-  end
-
-  mapping 'RegionalAMIs' do
-    item 'us-east-1', 'hvm', 'ami-76e27e1e'
-    item 'us-west-1', 'hvm', 'ami-d5180890'
-    item 'us-east-1', 'pv', 'ami-64e27e0c'
-    item 'us-west-1', 'pv', 'ami-c5180880'
-  end
-
-  mapping 'RegionalKeys' do
-    item 'us-east-1', 'test', 'cf-test-keys'
-    item 'us-west-1', 'test', 'cf-test-keys'
-  end
-
-  ## Define the VPC
-  ec2_vpc 'TargetVPC' do
-    network '100.65.0.0/18'
-    subnet_length 25
-
-    ## Add an InternetGateway
-    add_internet_gateway
-
-    ## Add a default routing table
-    public_table = add_route_table('Public', :gateway_route => true)
-
-    ## Define Subnets and Insatnces in each availability zone
-    stack.availability_zones do |zone, i|
-      add_subnet "Test#{ i }" do
-        availability_zone zone
-        associate_route_table public_table
-
-        tag 'Service', 'Foo'
+require_relative '../lib/convection'
+
+module Convection
+  module Demo
+    VPC = Convection.template do
+      description 'Demo VPC'
+
+      ## Define the VPC
+      ec2_vpc 'TargetVPC' do
+        network stack['subnet']
+        subnet_length 24
+        enable_dns true
+
+        tag 'Name', stack.cloud
+        tag 'Stack', stack.cloud
+        with_output 'id'
+
+        ## Add an InternetGateway
+        add_internet_gateway
+
+        public_acl = add_network_acl 'Public' do
+          entry 'AllowAllIngress' do
+            action 'allow'
+            number 100
+            network '0.0.0.0/0'
+            protocol :any
+            range :From => 0,
+                  :To => 65_535
+          end
+
+          entry 'AllowAllEgress' do
+            action 'allow'
+            number 100
+            egress true
+            network '0.0.0.0/0'
+            protocol :any
+            range :From => 0,
+                  :To => 65_535
+          end
+
+          tag 'Name', "acl-public-#{ stack.cloud }"
+          tag 'Stack', stack.cloud
+        end
+
+        public_table = add_route_table 'Public', :gateway_route => true do
+          tag 'Name', "routes-public-#{ stack.cloud }"
+          tag 'Stack', stack.cloud
+        end
+
+        stack.availability_zones do |zone, i|
+          add_subnet "Public#{ i }" do
+            availability_zone zone
+            acl public_acl
+            route_table public_table
+
+            with_output
+
+            immutable_metadata "public-#{ stack.cloud }"
+            tag 'Name', "subnet-public-#{ stack.cloud }-#{ zone }"
+            tag 'Stack', stack.cloud
+            tag 'Service', 'Public'
+          end
+        end
       end
     end
-
-    tag 'Name', join('-', 'cf-test-vpc', fn_ref('AWS::StackName'))
-  end
-
-  ec2_security_group 'BetterSecurityGroup' do
-    ingress_rule do
-      cidr_ip '0.0.0.0/0'
-      from 22
-      to 22
-      protocol 'TCP'
-    end
-    egress_rule do
-      cidr_ip '0.0.0.0/0'
-      from 0
-      to 65_535
-      protocol(-1)
-    end
-
-    description 'Allow SSH traffic from all of the places'
-    vpc_id fn_ref('TargetVPC')
-
-    tag 'Name', join('-', fn_ref('AWS::StackName'), 'BetterSecurityGroup')
   end
 end
-
-# puts test_template.render
-# puts test_template.to_json
-
-# stack_e1 = Convection.stack('TestStackE1B1', test_template, :region => 'us-east-1')
-stack_w1 = Convection.stack('TestStackW1B2', test_template, :region => 'us-west-1')
-
-# puts stack_e1.status
-# puts stack_e1.apply
-puts stack_w1.to_json
-
-puts "Status #{ stack_w1.status }"
-# puts stack_w1.apply

data/ext/resource_generator.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Given content like the following passed as the first argument
+# this should build the methods needed to write most of the convection
+# code for you.  You will have to do some massaging of the method names,
+# but this takes most of the grunt work out.
+# 
+# Example:
+# % . ./resource_generator.sh
+# or add it to your .profile
+# 
+# % resource_generator ""Count" : String,
+#"Handle" : String,
+#"Timeout" : String"
+#echo $1 | sed "s/\"//g" | cut -d ':' -f 1
+#echo $1 | sed "s/\"//g" | cut -d ':' -f 1 | tr -d ' ' | xargs -I {} printf "def {}\(value\)\n  property\(\'{}\', value\)\nend\n\n"
+
+
+resource_generator() {
+    echo $1 | sed "s/\"//g" | cut -d ':' -f 1 | tr -d ' ' | xargs -I {} printf "def {}\(value\)\n  property\(\'{}\', value\)\nend\n\n"
+}

data/lib/convection.rb

@@ -3,16 +3,17 @@
 ##
 module Convection
   class << self
-    def template(&block)
-      Model::Template.new(&block)
+    def template(*args, &block)
+      Model::Template.new(*args, &block)
     end
 
-    def stack(name, template, options = {})
-      Control::Stack.new(name, template, options)
+    def stack(*args)
+      Control::Stack.new(*args)
     end
   end
 end
 
 require_relative 'convection/version'
+require_relative 'convection/model/attributes'
 require_relative 'convection/model/template'
 require_relative 'convection/control/stack'

data/lib/convection/control/cloud.rb

@@ -0,0 +1,59 @@
+require_relative '../model/cloudfile'
+require_relative '../model/event'
+
+module Convection
+  module Control
+    ##
+    # Control tour Clouds
+    ##
+    class Cloud
+      def configure(cloudfile)
+        @cloudfile = Model::Cloudfile.new(cloudfile)
+      end
+
+      def stacks
+        @cloudfile.stacks
+      end
+
+      def deck
+        @cloudfile.deck
+      end
+
+      def converge(to_stack, &block)
+        unless to_stack.nil? || stacks.include?(to_stack)
+          block.call(Model::Event.new(:error, "Stack #{ to_stack } is not defined", :error)) if block
+          return
+        end
+
+        deck.each do |stack|
+          block.call(Model::Event.new(:converge, "Stack #{ stack.name }", :info)) if block
+          stack.apply(&block)
+
+          if stack.error?
+            block.call(Model::Event.new(:error, "Error converging stack #{ stack.name }", :error), stack.errors) if block
+            break
+          end
+
+          ## Stop on converge error
+          break unless stack.success?
+
+          ## Stop here
+          break if !to_stack.nil? && stack.name == to_stack
+          sleep rand @cloudfile.splay || 2
+        end
+      end
+
+      def diff(&block)
+        @cloudfile.deck.each do |stack|
+          block.call(Model::Event.new(:compare, "Compare local state of stack #{ stack.name } (#{ stack.cloud_name }) with remote template", :info))
+          sleep rand @cloudfile.splay || 2
+
+          difference = stack.diff
+          next block.call(Model::Event.new(:unchanged, "Stack #{ stack.cloud_name } Has no changes", :info)) if difference.empty?
+
+          difference.each { |diff| block.call(diff) }
+        end
+      end
+    end
+  end
+end

data/lib/convection/control/stack.rb

@@ -7,139 +7,338 @@ module Convection
     # Instantiation of a template in an account/region
     ##
     class Stack
+      attr_reader :id
       attr_reader :name
       attr_accessor :template
+      attr_reader :exist
+      attr_reader :status
+      alias_method :exist?, :exist
 
+      attr_reader :attributes
+      attr_reader :errors
+      attr_reader :options
+      attr_reader :resources
+      attr_reader :attribute_mapping_values
+      attr_reader :outputs
+
+      ## AWS-SDK
       attr_accessor :region
+      attr_accessor :cloud
+      attr_reader :capabilities
       attr_accessor :credentials
       attr_reader :parameters
       attr_reader :tags
       attr_accessor :on_failure
-      attr_reader :capabilities
-      attr_reader :options
 
       ## Valid Stack Statuses
-      CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
-      CREATE_FAILED = 'CREATE_FAILED'
       CREATE_COMPLETE = 'CREATE_COMPLETE'
-      ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
-      ROLLBACK_FAILED = 'ROLLBACK_FAILED'
-      ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
-      DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
-      DELETE_FAILED = 'DELETE_FAILED'
+      CREATE_FAILED = 'CREATE_FAILED'
+      CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
       DELETE_COMPLETE = 'DELETE_COMPLETE'
-      UPDATE_IN_PROGRESS = 'UPDATE_IN_PROGRESS'
-      UPDATE_COMPLETE_CLEANUP_IN_PROGRESS = 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'
+      DELETE_FAILED = 'DELETE_FAILED'
+      DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
+      ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
+      ROLLBACK_FAILED = 'ROLLBACK_FAILED'
+      ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
       UPDATE_COMPLETE = 'UPDATE_COMPLETE'
-      UPDATE_ROLLBACK_IN_PROGRESS = 'UPDATE_ROLLBACK_IN_PROGRESS'
-      UPDATE_ROLLBACK_FAILED = 'UPDATE_ROLLBACK_FAILED'
-      UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS = 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS'
+      UPDATE_COMPLETE_CLEANUP_IN_PROGRESS = 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'
+      UPDATE_FAILED = 'UPDATE_FAILED'
+      UPDATE_IN_PROGRESS = 'UPDATE_IN_PROGRESS'
       UPDATE_ROLLBACK_COMPLETE = 'UPDATE_ROLLBACK_COMPLETE'
+      UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS = 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS'
+      UPDATE_ROLLBACK_FAILED = 'UPDATE_ROLLBACK_FAILED'
+      UPDATE_ROLLBACK_IN_PROGRESS = 'UPDATE_ROLLBACK_IN_PROGRESS'
 
       ## Internal status
       NOT_CREATED = 'NOT_CREATED'
 
       def initialize(name, template, options = {})
         @name = name
-        @template = template
+        @template = template.clone(self)
+        @errors = []
 
+        @cloud = options.delete(:cloud)
+        @cloud_name = options.delete(:cloud_name)
         @region = options.delete(:region) { |_| 'us-east-1' }
         @credentials = options.delete(:credentials)
         @parameters = options.delete(:parameters) { |_| {} } # Default empty hash
         @tags = options.delete(:tags) { |_| {} } # Default empty hash
-
-        ## There can be only one...
+        options.delete(:disable_rollback) # There can be only one...
         @on_failure = options.delete(:on_failure) { |_| 'DELETE' }
-        options.delete(:disable_rollback)
-
         @capabilities = options.delete(:capabilities) { |_| ['CAPABILITY_IAM'] }
+
+        @attributes = options.delete(:attributes) { |_| Model::Attributes.new }
         @options = options
 
-        @ec2_client = AWS::EC2::Client.new(:region => region,
-                                           :credentials => @credentials)
-        @cf_client = AWS::CloudFormation::Client.new(:region => region,
-                                                     :credentials => @credentials)
+        client_options = {}.tap do |opt|
+          opt[:region] = @region
+          opt[:credentials] = @credentials unless @credentials.nil?
+        end
+        @ec2_client = Aws::EC2::Client.new(client_options)
+        @cf_client = Aws::CloudFormation::Client.new(client_options)
+
+        ## Remote state
+        @exist = false
+        @status = NOT_CREATED
+        @id = nil
+        @outputs = {}
+        @resources = {}
+        @current_template = {}
+        @last_event_seen = nil
+
+        ## Get initial state
+        get_status(cloud_name)
+        return unless exist?
+
+        get_resources
+        get_template
+        resource_attributes
+        get_events(1) # Get the latest page of events (Set @last_event_seen before starting)
+      rescue Aws::Errors::ServiceError => e
+        @errors << e
       end
 
-      def stacks
-        @stacks || cf_get_stacks
+      def cloud_name
+        return @cloud_name unless @cloud_name.nil?
+        return name if cloud.nil?
+        "#{ cloud }-#{ name }"
       end
 
-      def status
-        stacks[name].stack_status rescue NOT_CREATED
+      ##
+      # Attribute Accessors
+      ##
+      def include?(stack, key = nil)
+        return @attributes.include?(name, stack) if key.nil?
+        @attributes.include?(stack, key)
       end
 
-      def exist?
-        stacks.include?(name)
+      def [](key)
+        @attributes.get(name, key)
       end
 
-      def complete?
-        [CREATE_COMPLETE, UPDATE_COMPLETE].include?(status)
+      def []=(key, value)
+        @attributes.set(name, key, value)
       end
 
-      def rollback?
-        [ROLLBACK_IN_PROGRESS, ROLLBACK_FAILED, ROLLBACK_COMPLETE,
-         UPDATE_ROLLBACK_IN_PROGRESS, UPDATE_ROLLBACK_FAILED,
-         UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS,
-         UPDATE_ROLLBACK_COMPLETE].include?(status)
+      def get(*args)
+        @attributes.get(*args)
+      end
+
+      ##
+      # Stack State
+      ##
+      def in_progress?
+        [CREATE_IN_PROGRESS, ROLLBACK_IN_PROGRESS, DELETE_IN_PROGRESS,
+         UPDATE_IN_PROGRESS, UPDATE_COMPLETE_CLEANUP_IN_PROGRESS,
+         UPDATE_ROLLBACK_IN_PROGRESS,
+         UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
+        ].include?(status)
+      end
+
+      def complete?
+        [CREATE_COMPLETE, ROLLBACK_COMPLETE, UPDATE_COMPLETE, UPDATE_ROLLBACK_COMPLETE].include?(status)
       end
 
       def fail?
         [CREATE_FAILED, ROLLBACK_FAILED, DELETE_FAILED, UPDATE_ROLLBACK_FAILED].include?(status)
       end
 
+      def error?
+        !errors.empty?
+      end
+
+      def success?
+        !error? && complete?
+      end
+
+      ##
+      # Rendderers
+      ##
       def render
-        template.render(self)
+        @template.render
+      end
+
+      def to_json(pretty = false)
+        @template.to_json(nil, pretty)
       end
 
-      def to_json
-        template.to_json(self)
+      def diff
+        @template.diff(@current_template)
       end
 
-      def apply
-        cf_get_stacks ## force-update status
+      ##
+      # Controllers
+      ##
+      def apply(&block)
         request_options = @options.clone.tap do |o|
-          o[:stack_name] = name
           o[:template_body] = to_json
           o[:parameters] = cf_parameters
           o[:capabilities] = capabilities
         end
 
-        return @cf_client.update_stack(request_options) if exist?
-        @cf_client.create_stack(request_options.tap do |o|
-          o[:tags] = cf_tags
-          o[:on_failure] = @on_failure
-        end)
-      rescue AWS::CloudFormation::Errors::ValidationError => e
-        ## TODO Return something sane
-        ## SDK throws this as an error >.<
-        raise e unless e.message == 'No updates are to be performed.'
+        if exist?
+          if diff.empty? ## No Changes. Just get resources and move on
+            block.call(Model::Event.new(:complete, "Stack #{ name } has no changes", :info)) if block
+            get_status
+            return
+          end
+
+          ## Update
+          @cf_client.update_stack(request_options.tap do |o|
+            o[:stack_name] = id
+          end)
+        else
+          ## Create
+          @cf_client.create_stack(request_options.tap do |o|
+            o[:stack_name] = cloud_name
+
+            o[:tags] = cf_tags
+            o[:on_failure] = on_failure
+          end)
+
+          get_status(cloud_name) # Get ID of new stack
+        end
+
+        watch(&block) if block # Block execution on stack status
+      rescue Aws::Errors::ServiceError => e
+        @errors << e
       end
 
-      def delete
+      def delete(&block)
         @cf_client.delete_stack(
-          :stack_name => name
+          :stack_name => id
         )
+
+        ## Block execution on stack status
+        watch(&block) if block
+
+        get_status
+      rescue Aws::Errors::ServiceError => e
+        @errors << e
+      end
+
+      def watch(poll = 2, &block)
+        get_status
+
+        loop do
+          get_events.reverse_each do |event|
+            block.call(Model::Event.from_cf(event))
+          end if block
+
+          break unless in_progress?
+
+          sleep poll
+          get_status
+        end
+      rescue Aws::Errors::ServiceError => e
+        @errors << e
       end
 
       def availability_zones(&block)
         @availability_zones ||=
-          @ec2_client.describe_availability_zones.availability_zone_info.map(&:zone_name).sort
+          @ec2_client.describe_availability_zones.availability_zones.map(&:zone_name).sort
 
         @availability_zones.each_with_index(&block) if block
         @availability_zones
       end
 
+      def validate
+        result = @cf_client.validate_template(:template_body => template.to_json)
+        fail result.context.http_response.inspect unless result.successful?
+        puts "\nTemplate validated successfully"
+      end
+
       private
 
-      def cf_get_stacks
-        @stacks = {}.tap do |col|
-          cf_stacks = @cf_client.list_stacks.stack_summaries rescue []
-          cf_stacks.each do |s|
-            next if s.stack_status == DELETE_COMPLETE
-            col[s.stack_name] = s
+      def get_status(stack_name = id)
+        cf_stack = @cf_client.describe_stacks(:stack_name => stack_name).stacks.first
+
+        @id = cf_stack.stack_id
+        @status = cf_stack.stack_status
+        @exist = true
+
+        ## Parse outputs
+        @outputs = {}.tap do |collection|
+          cf_stack.outputs.each do |output|
+            collection[output[:output_key].to_s] = (JSON.parse(output[:output_value]) rescue output[:output_value])
           end
         end
+
+        ## Add outputs to attribute set
+        @attributes.load_outputs(self)
+      rescue Aws::CloudFormation::Errors::ValidationError # Stack does not exist
+        @exist = false
+        @status = NOT_CREATED
+        @id = nil
+        @outputs = {}
+      end
+
+      ## Fetch current resources
+      def get_resources
+        @resources = {}.tap do |collection|
+          @cf_client.list_stack_resources(:stack_name => @id).each do |page|
+            page.stack_resource_summaries.each do |resource|
+              collection[resource[:logical_resource_id]] = resource
+            end
+          end
+        end
+      rescue Aws::CloudFormation::Errors::ValidationError # Stack does not exist
+        @resources = {}
+      end
+
+      def get_template
+        @current_template = JSON.parse(@cf_client.get_template(:stack_name => id).template_body)
+      rescue Aws::CloudFormation::Errors::ValidationError # Stack does not exist
+        @current_template = {}
+      end
+
+      ## Fetch new stack events
+      def get_events(pages = nil, stack_name = id)
+        return [] unless exist?
+
+        [].tap do |collection|
+          @cf_client.describe_stack_events(:stack_name => stack_name).each do |page|
+            pages -= 1 unless pages.nil?
+
+            page.stack_events.each do |event|
+              if @last_event_seen == event.event_id
+                pages = 0 # Break page loop
+                break
+              end
+
+              collection << event
+            end
+
+            break if pages == 0
+          end
+
+          @last_event_seen = collection.first.event_id unless collection.empty?
+        end
+      rescue Aws::CloudFormation::Errors::ValidationError # Stack does not exist
+      end
+
+      ## TODO No. This will become unnecessary as current_state is fleshed out
+      def resource_attributes
+        @attribute_mapping_values = {}
+        @template.execute ## Populate mappings fro the template
+
+        @resources.each do |logical, resource|
+          next unless @template.attribute_mappings.include?(logical)
+
+          attribute_map = @template.attribute_mappings[logical]
+          case attribute_map[:type].to_sym
+          when :string
+            @attribute_mapping_values[attribute_map[:name]] = resource[:physical_resource_id]
+          when :array
+            @attribute_mapping_values[attribute_map[:name]] = [] unless @attribute_mapping_values[attribute_map[:name]].is_a?(Array)
+            @attribute_mapping_values[attribute_map[:name]].push(resource[:physical_resource_id])
+          else
+            fail TypeError, "Attribute Mapping must be defined with type `string` or `array`, not #{ type }"
+          end
+        end
+
+        ## Add mapped resource IDs to attributes
+        @attributes.load_resources(self)
       end
 
       def cf_parameters
@@ -163,3 +362,5 @@ module Convection
     end
   end
 end
+
+require_relative '../model/event'