jackal-cfn 0.2.8 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f9789fbb8ee01ff50a47c77185056c6e52c8937
-  data.tar.gz: 179c2cc85930686d222e3928f05d6e1ec6408a01
+  metadata.gz: 9fddaa1ca5447da20d6c856d2ccdf3ba4d3c89d9
+  data.tar.gz: acddccbce932860d09c048ba04627df5744cd4b3
 SHA512:
-  metadata.gz: 6f9ea369c29581843ea001e10b38747e3b76e12f1cd3756c38f5c5115615380a9922ddb6201bf99ab384da638e3daae677a433482f90b783923eddb96fc86712
-  data.tar.gz: 93ce81bd66869a378ae651b5fe1ccc6dbce264d52535a9ce66e995834de57722ba8aad980f9b7675786fc3f24a0e433daaa385b24d0325ce4e12e82636e61919
+  metadata.gz: 6d5f2187fab6e8d42da3ab3351b53720b1cd351a95df76ffe40390e50786a4f457fef306f6b00b93de1dabba807bcdb1f87e1c0c5b08b67d3f9dd8985d6c5b86
+  data.tar.gz: 9b5b9c770c12c9158bdc2ad3bf03041b7200b3aacdaf35f2bbec8c561673646d0b1a2ec63d1acea196e8b246b29468397497291f6b7fb006e6ac96359e2447f6

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v0.2.10
+* Properly handle no parameters in properties
+* Add new JackalStack resource
+
 # v0.2.8
 * Fix optional instance halt to wait for ami AVAILABLE state
 

data/README.md

@@ -1,23 +1,319 @@
 # Jackal CFN
 
-CFN proxy to the jackal subsystem.
+Provides jackal integration for AWS CloudFormation custom
+resources and stack event notifications.
 
-## Supported payload generation
+## Requirements
 
-### Events
+This library currently uses the `patron` gem for sending
+notifications to AWS S3. It requires the curl development
+libraries to be available, so ensure it is installed.
 
-Stack events are injected with the following
-payload structure:
+## Usage
+
+There are two ways to use this library. The first is to process
+events and resources into proper payloads and inject them into the
+pipeline. The other is to re-process a formatted payload.
+
+### Pipeline Injection
+
+Configuration for pipeline injection of resource and event notifications
+is very straightforward:
+
+```json
+{
+  "jackal": {
+    "cfn": {
+      "config": {
+      },
+      "sources": {
+        "input": {
+          "type": "sqs",
+          "args": {
+            SQS_CONFIG
+          }
+        },
+        "output": {
+          OUTPUT_SOURCE
+        }
+      },
+      "callbacks": [
+        "Jackal::Cfn::Resource",
+        "Jackal::Cfn::Event"
+      ]
+    }
+  }
+}
+```
+
+With this configuration in place resources and events will be received,
+formatted, and delivered to the `OUTPUT_SOURCE`. It will be the job of
+a later service to handle reply notifications (for resources) in this
+style of usage.
+
+### Message Reprocessing
+
+When using the re-processing configuration, messages do not continue
+down the pipeline. Instead, the original message is received, formatted,
+and then re-delivered to the originating source (generally an SQS queue).
+The message will be fetched again (this time properly formatted) and
+any matching callbacks will be executed. A sample configuration may look
+something like this:
+
+```json
+{
+  "jackal": {
+    "cfn": {
+      "config": {
+        "reprocess": true,
+        "ami": {
+          "credentials": {
+            CREDENTIALS
+          }
+        }
+      },
+      "sources": {
+        "input": {
+          "type": "sqs",
+          "args": {
+            SQS_CONFIG
+          }
+        },
+        "output": {
+          OUTPUT_SOURCE
+        }
+      },
+      "callbacks": [
+        "Jackal::Cfn::Resource",
+        "Jackal::Cfn::AmiRegister"
+      ]
+    }
+  }
+}
+```
+
+The important item to note is the `"reprocess": true` which enables the
+automatic re-processing of messages.
+
+## Custom Resources
+
+This library provides support for creating new custom resources. Creation
+is as simple as subclassing:
+
+```ruby
+module Jackal
+  module Cfn
+    class LocalPrinter < Jackal::Cfn::Resource
+
+      def execute(message)
+        failure_wrap(message) do |payload|
+          cfn_resource = rekey_hash(payload.get(:data, :cfn_resource))
+          cfn_response = build_response(cfn_resource)
+          info "CFN Resource: #{cfn_resource.inspect}"
+          info "CFN Response: #{cfn_response.inspect}"
+          respond_to_stack(cfn_response, cfn_resource[:response_url])
+        end
+      end
+
+    end
+  end
+end
+```
+
+This will match `Custom::LocalPrinter` resource requests that are received. It
+will print information into the log about the request and then send a successful
+response back to the stack.
+
+### Builtin Custom Resources
+
+This library provides a few custom resources builtin. These can be used directly or
+as examples for building new custom resources.
+
+#### `Jackal::Cfn::AmiRegister`
+
+Generates and registers an AMI based on an EC2 resource. This allows for building a
+single EC2 resource within a stack that is fully configured, creating and registering
+an AMI based on that EC2 resource, and using the new AMI for instances created within
+an ASG. It is an integrated way to speed up ASG instance launches, but keep all resources
+for the stack fully managed.
+
+Resource usage:
 
 ```json
+{
+  "Type": "Custom::AmiRegister",
+  "Properties": {
+    "Parameters": {
+      "Name": String,
+      "InstanceId": String,
+      "Description": String,
+      "NoReboot": Boolean,
+      "BlockDeviceMappings": Array,
+      "HaltInstance": Boolean,
+      "Region": String
+    }
+  }
+}
 ```
 
-### Resources
+Resource Response:
+
+```json
+{
+  "AmiId": String
+}
+```
+
+Configuration:
+
+```json
+{
+  "jackal": {
+    "cfn": {
+      "config": {
+        "ami": {
+          "credentials": {
+            "compute": {
+              FOG_CREDENTIALS
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+#### `Jackal::Cfn::AmiManager`
+
+This resource is a simplification of the `AmiRegister` resource. The
+`AmiManager` is used to ensure an AMI is removed from the system when
+a stack is destroyed. This allows for customized AMI generation to be
+integrated and ensure that once the stack is destroyed all custom AMIs
+for that stack are destroyed as well.
+
+Resource usage:
+
+```json
+{
+  "Type": "Custom::AmiManager",
+  "Properties": {
+    "Parameters": {
+      "AmiId": "",
+      "Region": ""
+    }
+  }
+}
+```
+
+Resource Response:
+
+```json
+{
+}
+```
+
+Configuration:
+
+```json
+{
+  "jackal": {
+    "cfn": {
+      "config": {
+        "ami": {
+          "credentials": {
+            "compute": {
+              FOG_CREDENTIALS
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+#### `Jackal::Cfn::HashExtractor`
+
+This resource will extract a nested hash value from a JSON string. Useful
+for when a result may be serialized JSON and a value from that structure
+is required elsewhere.
+
+Resource usage:
+
+```json
+{
+  "Type": "Custom::HashExtractor",
+  "Properties": {
+    "Parameters": {
+      "Key": "path.to.value.in.hash",
+      "Value": JSON
+    }
+  }
+}
+```
+
+Resource Response:
+
+```json
+{
+  "Payload": VALUE
+}
+```
+
+#### `Jackal::Cfn::JackalStack`
+
+This resource provides an integration point for building stacks
+on remote endpoints. Remote end points are provided via configuration
+and referenced via the `Location` property in the custom resource.
+
+Resource usage:
+
+```json
+{
+  "Type": "Custom::JackalStack",
+  "Properties": {
+    "Parameters": {
+      STACK_PARAMETERS
+    },
+    "Location": LOCATION,
+    "TemplateURL": URL
+  }
+}
+```
+
+Resource Response:
+
+The outputs of the stack will be proxied:
+
+```json
+{
+  "Outputs.OUTPUT_NAME": "OUTPUT_VALUE"
+}
+```
 
-Custom stack resources are injected with the
-following structure:
+Configuration:
 
 ```json
+{
+  "jackal": {
+    "cfn": {
+      "config": {
+        "jackal_stack": {
+          "credentials": {
+            "storage": {
+              TEMPLATE_S3_CREDENTIALS
+            },
+            LOCATION: {
+              "provider": NAME,
+              MIASMA_CREDENTIALS
+            }
+          }
+        }
+      }
+    }
+  }
+}
 ```
 
 ## Info

data/jackal-cfn.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |s|
   s.require_path = 'lib'
   s.license = 'Apache 2.0'
   s.add_dependency 'jackal'
+  s.add_dependency 'miasma'
   s.add_dependency 'patron'
   s.files = Dir['lib/**/*'] + %w(jackal-cfn.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
 end

data/lib/jackal-cfn/resource/ami_manager.rb

@@ -9,9 +9,7 @@ module Jackal
     #     "Type": "Custom::AmiManager",
     #     "Properties": {
     #       "Parameters": {
-    #         "InstanceId": "",
     #         "AmiId": "",
-    #         "Action": "",
     #         "Region": ""
     #       }
     #     }

data/lib/jackal-cfn/resource/hash_extractor.rb

@@ -10,7 +10,7 @@ module Jackal
     #     "Properties": {
     #       "Parameters": {
     #         "Key": "path.to.value.in.hash",
-    #         "Value": Hash_or_JSON_string
+    #         "Value": JSON
     #       }
     #     }
     #   }

data/lib/jackal-cfn/resource/jackal_stack.rb

@@ -0,0 +1,248 @@
+require 'jackal-cfn'
+
+module Jackal
+  module Cfn
+    # Manage AMI Resources
+    #
+    # Expected resource:
+    #   {
+    #     "Type": "Custom::JackalStack",
+    #     "Properties": {
+    #       "Parameters": {
+    #         STACK_PARAMETERS
+    #       },
+    #       "Location": LOCATION,
+    #       "TemplateURL": "URL"
+    #     }
+    #   }
+    #
+    # Required configuration:
+    #   {
+    #     "config": {
+    #       "jackal_stack": {
+    #         "credentials": {
+    #           "storage": {
+    #             AWS_CREDENTIALS
+    #           },
+    #           LOCATION: {
+    #             "provider": "NAME",
+    #             MIAMSA_CREDENTIALS
+    #           }
+    #         }
+    #       }
+    #     }
+    #   }
+    class JackalStack < Jackal::Cfn::Resource
+
+      # Load miasma for stack building
+      def setup(*_)
+        require 'miasma'
+      end
+
+      # Perform requested stack action
+      #
+      # @param message [Carnivore::Message]
+      def execute(message)
+        failure_wrap(message) do |payload|
+          cfn_resource = rekey_hash(payload.get(:data, :cfn_resource))
+          properties = rekey_hash(cfn_resource[:resource_properties])
+          parameters = rekey_hash(properties[:parameters])
+          cfn_response = build_response(cfn_resource)
+          case cfn_resource[:request_type].to_sym
+          when :create
+            create_stack(cfn_response, cfn_resource, properties, parameters, message)
+          when :update
+            update_stack(cfn_response, cfn_resource, properties, parameters, message)
+          when :delete
+            destroy_stack(cfn_response, cfn_resource, message)
+          else
+            error "Unknown request type received: #{cfn_resource[:request_type].inspect}"
+            cfn_response['Status'] = 'FAILED'
+            cfn_response['Reason'] = 'Unknown request type received'
+          end
+          respond_to_stack(cfn_response, cfn_resource[:response_url])
+          job_completed(:jackal_cfn, payload, message)
+        end
+      end
+
+      # Build API connection to base template storage bucket
+      #
+      # @param bucket_region [String] location of bucket
+      # @return [Miasma::Models::Storage]
+      def storage_api(bucket_region)
+        Miasma.api(
+          :type => :storage,
+          :provider => :aws,
+          :credentials => config.get(:jackal_stack, :credentials, :storage).merge(
+            :aws_bucket_region => bucket_region
+          )
+        )
+      end
+
+      # Build orchestration API connection for provided location
+      #
+      # @param location [String, Symbol]
+      # @return [Miasma::Models::Orchestration]
+      def remote_api(location)
+        l_config = config.get(:jackal_stack, :credentials, location)
+        if(l_config)
+          Miasma.api(
+            :type => :orchestration,
+            :provider => l_config[:provider],
+            :credentials => l_config
+          )
+        else
+          raise ArgumentError.new "Unknown target location provided `#{location}`!"
+        end
+      end
+
+      # Fetch a template from a storage bucket
+      #
+      # @param endpoint [String] URL to template
+      # @return [Hash] loaded template data
+      def fetch_template(endpoint)
+        url = URI.parse(endpoint)
+        region = url.host.split('.').first.split('-', 2).last
+        if(region == 's3')
+          region = 'us-east-1'
+        end
+        bucket, path = url.path.sub('/', '').split('/', 2)
+        MultiJson.load(
+          storage_api(region).buckets.get(
+            bucket.sub('/', '')
+          ).files.get(path).body.read
+        )
+      end
+
+      # Create a new stack and update the response values
+      #
+      # @param response [Hash] response data of action
+      # @param resource [Hash] request resource
+      # @param properties [Hash] properties of request resource
+      # @param parameters [Hash] parmeters provided via properties
+      # @param message [Carnivore::Message] original message
+      # @return [TrueClass, FalseClass]
+      def create_stack(response, resource, properties, parameters, message)
+        stack = remote_api(properties[:location]).stacks.build(
+          :name => [
+            'JackalStack',
+            resource[:logical_resource_id],
+            resource[:stack_id].split('/').last
+          ].join('-'),
+          :template => properties.fetch(:stack, fetch_template(properties[:template_url])),
+          :parameters => Hash[parameters.map{|k,v| [Bogo::Utility.camel(k), v] }]
+        )
+        stack.save
+        until(stack.state.to_s.end_with?('complete'))
+          message.touch!
+          debug "Waiting for created stack to reach completion..."
+          sleep 5
+          stack.reload
+        end
+        if(stack.state.to_s.end_with?('complete') || stack.state.to_s.end_with?('failed'))
+          stack.outputs.each do |output|
+            response['Data']["Outputs.#{output.key}"] = output.value
+          end
+          response['PhysicalResourceId'] = "#{properties[:location]}-#{stack.id}"
+          true
+        else
+          response['Status'] = 'FAILED'
+          response['Reason'] = 'Stack creation failed!'
+          stack.destroy
+          false
+        end
+      end
+
+      # Update an existing stack and update the response values
+      #
+      # @param response [Hash] response data of action
+      # @param resource [Hash] request resource
+      # @param properties [Hash] properties of request resource
+      # @param parameters [Hash] parmeters provided via properties
+      # @param message [Carnivore::Message] original message
+      # @return [TrueClass, FalseClass]
+      def update_stack(response, resource, properties, parameters, message)
+        c_location, stack_id = resource[:physical_resource_id].split('-', 2)
+        if(c_location != properties[:location])
+          warn "Stack resource has changed location! #{c_location} -> #{properties[:location]}"
+          warn "Starting destruction of existing resource: #{stack_id}"
+          if(destroy_stack(response, resource, message))
+            info "Destruction of stack `#{stack_id}` complete. Creating replacement stack."
+            create_stack(response, resource, properties, parameters, message)
+          else
+            error "Failed to destroy existing stack for replacement `#{stack_id}`"
+          end
+        else
+          stack = remote_api(c_location).stacks.get(stack_id)
+          if(stack)
+            info "Stack resource update on: #{stack_id}"
+            stack.template = fetch_template(properties['TemplateURL'])
+            stack.parameters = Hash[parameters.map{|k,v| [Bogo::Utility.camel(k), v] }]
+            stack.save
+            until(stack.state.to_s.end_with?('complete') || stack.state.to_s.end_with?('failed'))
+              debug "Waiting for created stack to reach completion..."
+              sleep 5
+              stack.reload
+            end
+            if(stack.state.to_s.end_with?('complete'))
+              stack.outputs.each do |output|
+                response['Data']["Outputs.#{output.key}"] = output.value
+              end
+              response['PhysicalResourceId'] = stack.id
+            else
+              response['Status'] = 'FAILED'
+              response['Reason'] = 'Stack update failed!'
+            end
+          else
+            response['Status'] = 'FAILED'
+            response['Reason'] = "No stack was found matching request: #{stack_id}"
+          end
+        end
+      end
+
+      # Destroy the stack
+      #
+      # @param response [Hash] response data of action
+      # @param resource [Hash] request resource
+      # @param message [Carnivore::Message] original message
+      def destroy_stack(response, resource, message)
+        stack = request_destroy(resource[:physical_resource_id])
+        if(stack)
+          until(stack.state.nil? || stack.state.to_s.end_with?('complete') || stack.state.to_s.end_with?('failed'))
+            info "Waiting for stack destruction (#{stack.name})..."
+            message.touch!
+            sleep 5
+            stack.reload
+          end
+          if(stack.state.to_s.end_with?('failed'))
+            response['Status'] = 'FAILED'
+            response['Reason'] = 'Failed to delete remote stack!'
+          end
+        end
+      end
+
+      # Send a stack delete request
+      #
+      # @param stack_resource_id [String] physical resource ID
+      # @return [Miasma::Models::Orchestration::Stack, FalseClass]
+      def request_destroy(stack_resource_id)
+        location, stack_id = stack_resource_id.split('-', 2)
+        if(stack_id)
+          begin
+            info "Sending stack destruction request to: #{stack_id} in: #{location}"
+            stack = remote_api(location).stacks.get(stack_id)
+            stack.destroy
+            stack
+          rescue => e
+            error "Stack destruction request failed! #{e.class}: #{e.message}"
+            false
+          end
+        else
+          warn "No stack ID registered in resource. Skipping destroy: #{stack_resource_id}"
+          false
+        end
+      end
+
+    end
+  end
+end

data/lib/jackal-cfn/resource.rb

@@ -175,12 +175,12 @@ module Jackal
           payload = unpack(message)
           yield payload
         rescue => e
-          error "Unexpected error encountered processing custom resource - #{e.class}: #{e}"
+          error "Unexpected error encountered processing custom resource - #{e.class}: #{e.message}"
           debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
           cfn_resource = payload.get(:data, :cfn_resource)
           cfn_response = build_response(cfn_resource)
           cfn_response['Status'] = 'FAILED'
-          cfn_response['Reason'] = "Unexpected error encountered [#{e}]"
+          cfn_response['Reason'] = "Unexpected error encountered [#{e.message}]"
           respond_to_stack(cfn_response, cfn_resource[:response_url])
           message.confirm!
         end

data/lib/jackal-cfn/utils.rb

@@ -13,7 +13,7 @@ module Jackal
       # @return [Hash] new hash with snake cased toplevel keys
       def transform_parameters(params)
         Smash.new.tap do |new_hash|
-          params.each do |key, value|
+          (params || []).each do |key, value|
             new_hash[snakecase(key)] = value
           end
         end

data/lib/jackal-cfn/version.rb

@@ -1,6 +1,6 @@
 module Jackal
   module Cfn
     # Current version
-    VERSION = Gem::Version.new('0.2.8')
+    VERSION = Gem::Version.new('0.2.10')
   end
 end

data/lib/jackal-cfn.rb

@@ -12,5 +12,6 @@ module Jackal
     autoload :HashExtractor, 'jackal-cfn/resource/hash_extractor'
     autoload :AmiManager, 'jackal-cfn/resource/ami_manager'
     autoload :AmiRegister, 'jackal-cfn/resource/ami_register'
+    autoload :JackalStack, 'jackal-cfn/resource/jackal_stack'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jackal-cfn
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.10
 platform: ruby
 authors:
 - Chris Roberts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-21 00:00:00.000000000 Z
+date: 2015-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jackal
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: miasma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: patron
   requirement: !ruby/object:Gem::Requirement
@@ -56,6 +70,7 @@ files:
 - lib/jackal-cfn/resource/ami_manager.rb
 - lib/jackal-cfn/resource/ami_register.rb
 - lib/jackal-cfn/resource/hash_extractor.rb
+- lib/jackal-cfn/resource/jackal_stack.rb
 - lib/jackal-cfn/utils.rb
 - lib/jackal-cfn/utils/fog.rb
 - lib/jackal-cfn/utils/http.rb