cfnpp 0.2.0

data/bin/cfnpp

@@ -0,0 +1,396 @@
+#!/usr/bin/env ruby
+
+unless Kernel.respond_to?(:require_relative)
+  module Kernel
+    def require_relative(path)
+      require File.join(File.dirname(caller[0]), path.to_str)
+    end
+  end
+end
+
+require 'rubygems'
+require 'bundler/setup'
+require 'aws-sdk'
+require 'securerandom'
+require 'json'
+require 'cfnpp/transform'
+require 'cfnpp/uploader'
+require 'awesome_print'
+require 'yaml'
+require 'dogapi'
+require 'ploy/yamlreader'
+require 'optparse'
+
+def main()
+  usage = <<eot
+  Usage: ./launch create|update|print STACK_TYPE ENVIRONMENT_TYPE [ENVIRONMENT_NAME]
+    STACK_TYPE found in stacks, i.e manta-site
+    ENVIRONMENT_TYPE is e.g. smoketest, used as a key into configuration
+    ENVIRONMENT_NAME is your unique name, i.e stsmith
+  Launch a cloudformation template.
+eot
+
+  cli_opts = {}
+  optparse = OptionParser.new do|opts|
+    opts.on('-h', '--help', 'Display this screen') do
+      puts usage
+      exit
+    end
+    opts.on('-d', '--deployconfig URL', 'stackhub URL for deployconfig') do |url|
+      cli_opts[:deployconfig] = url
+    end
+    opts.on('-F', '--force', 'Force stack update and disable cancel-bad-rolling-updates from executing. (Affects only this update)') do |url|
+      cli_opts[:force] = 1
+    end
+    opts.on('-b', '--basepath PATH', 'base path for stacks; defaults to ./stacks') do |basepath|
+      cli_opts[:basepath] = basepath
+    end
+    opts.on('-f', '--file PATH', 'use PATH instead of $basepath/main.yml for main template') do |filepath|
+      cli_opts[:filepath] = filepath
+    end
+    opts.on('-c', '--conf PATH', 'use PATH instead of $basepath/conf/$environment_type for conf file') do |confpath|
+      cli_opts[:confpath] = confpath
+    end
+  end
+  optparse.parse!
+
+  if cli_opts[:deployconfig]
+    yr = Ploy::YamlReader.new
+    conf = yr.from_http(cli_opts[:deployconfig])
+    if (conf['locked'])
+      puts "skipping autoupdate: deployconfig is locked"
+      exit
+    end
+    unless (conf['stack_autoupdate'])
+      puts "skipping autoupdate: autoupdate is false"
+      exit
+    end
+  end
+
+  whoami = `whoami`
+  whoami.chomp!
+  
+  case whoami
+  when "judd"
+     whoami="jmontgomery"
+  when "juddmontgomery"
+     whoami="jmontgomery"
+  end
+
+  basepath = cli_opts[:basepath] || "stacks"
+  
+  action = ARGV.shift
+  stack_type = ARGV.shift
+  template_path = "#{basepath}/#{stack_type}/main.yml"
+  template_path = cli_opts[:filepath] if cli_opts[:filepath]
+  environment_type = ARGV.shift
+  environment_name = ARGV.shift || whoami
+  
+  cfn = AWS::CloudFormation.new
+  stack_name = "#{stack_type}--#{environment_name}--#{environment_type}"
+  conf_path = cli_opts[:confpath] || "#{basepath}/#{stack_type}/conf/#{environment_type}.yml"
+
+  stack = cfn.stacks[stack_name]
+  if stack.exists?
+    status = stack.status
+    if status =~ /_IN_PROGRESS$/
+      puts "can't update stack (#{stack_name}) while status is (#{status})"
+      exit 0
+    end
+  end
+ 
+  allowed_actions = ['create', 'update', 'print', 'validate'] 
+  if !action || !allowed_actions.include?(action) || !template_path || !File.file?(template_path) || !File.file?(conf_path)
+    if action && ! allowed_actions.include?(action)
+      puts "unknown action (#{action})"
+    end
+  
+    if conf_path
+      if conf_path && !File.file?(conf_path)
+        puts "ERROR: file (#{conf_path}) not found for stack (#{environment_type})"
+      end
+    end
+  
+    if stack_type
+      if template_path && !File.file?(template_path)
+        puts "ERROR: file (#{template_path}) not found for stack (#{stack_type})"
+      end
+    end
+  
+    if environment_type && !/(^smoketest$)/.match(environment_type)
+      puts "ERROR: unknown environment_type (#{environment_type})"
+    end
+  
+    puts usage
+    exit 1
+  end
+  
+  conf_yaml = File.read(conf_path)
+  opts = YAML::load(conf_yaml)
+  
+  timestamp = Time.now.utc.strftime("%Y-%m-%dT%H.%M.%S.%LZ")
+  template_info = get_template_info()
+  opts['LastUpdateTimestamp'] = timestamp
+  opts['TemplateGitRevision'] = template_info[:gitrev]
+  opts['TemplateGitStatus'] = template_info[:gitstatus]
+  opts['TemplateSource'] = template_info[:source]
+  opts['LaunchInstanceId'] = template_info[:instanceid]
+  opts['StackType'] = stack_type
+  opts['StackName'] = stack_name
+  opts['EnvironmentName'] = environment_name
+  opts['EnvironmentType'] = environment_type
+
+  if cli_opts[:force]
+    opts['Force'] = 'true'
+  else
+    opts['Force'] = 'true'
+  end
+ 
+  uploader = CfnPP::Uploader.new('manta-cloudformation', stack_name)
+
+  template_result = CfnPP::Transform.load_file(template_path, opts, "main", "https://manta-cloudformation.s3.amazonaws.com/#{uploader.s3_path}") 
+
+  if action == "print" #hack. TODO: dispatch actions sanely
+    puts JSON.pretty_generate(template_result.data)
+    exit 0
+  end
+
+  upload_results = do_upload(uploader, template_result, opts)
+
+  if action == 'validate'
+    puts 'uploading and validating only; no updates applied'
+    exit 0
+  end
+
+  if action == 'create'
+    puts "creating stack (#{stack_name}) from template (#{template_path}) with config (#{conf_path})"
+    stack = cfn.stacks.create(stack_name,
+                              upload_results[:url],
+                              :parameters => upload_results[:opts_parameters],
+                              :capabilities => ['CAPABILITY_IAM'])
+  end
+  
+  if action ==  'update'
+    current_resource_count = 0
+    current_resource_count = stack.resources.count()
+    stack.resources.each_batch do |batch|
+      current_resource_count += batch.count
+    end
+    puts "updating stack (#{stack_name}) from template (#{template_path}) with config (#{conf_path})"
+    puts "stack currently contains (#{current_resource_count}) resources"
+
+    # Send an event to Datadog:
+    # disabled while decoupling cfnpp; TODO: enable
+    if false
+      api_key = ''
+      dog = Dogapi::Client.new(api_key)
+      dd_event_title = "stack update"
+      dd_event_text = "#{stack_name} stack was updated by #{whoami}"
+      dd_event_tags = "deployment, stackname:#{stack_name}"
+      dog.emit_event(Dogapi::Event.new("#{dd_event_text}", :msg_title => "#{dd_event_title}", :tags => "#{dd_event_tags}"))
+    end
+
+    # Update the stack
+    stack.update(:template => upload_results[:url],
+                         :parameters => upload_results[:opts_parameters],
+                         :capabilities => ['CAPABILITY_IAM'])
+  end
+  
+  after_update(stack, timestamp)
+end
+
+def do_upload(uploader, template_result, opts)
+
+  template_result.substacks.each do |substack_result|
+    do_upload(uploader, substack_result, opts)
+  end
+
+  upload_res = uploader.upload_template(template_result, opts)
+
+  if upload_res[:error]
+    puts "error for #{upload_res[:url]}"
+    if upload_res[:error] == 'validation_error'
+      vs = upload_res[:validation_status]
+      puts "validation error code: #{vs[:code]}"
+      puts "validation error message: #{vs[:message]}"
+    elsif upload_res[:error] == 'already_updated'
+      prev_opts = upload_res[:prev_opts]
+      last_update_timestamp = prev_opts['LastUpdateTimestamp']
+      template_git_revision = prev_opts['TemplateGitRevision']
+      instanceid = prev_opts['LaunchInstanceId']
+      puts "Update to git revision #{template_git_revision} already attempted at #{last_update_timestamp} from instance (#{instanceid})."
+    end
+    exit 1
+  else
+    puts "uploaded #{upload_res[:url]}"
+  end
+
+  return upload_res
+
+end
+
+def after_update(stack, timestamp)
+  stack_name = stack.name
+  #cmd = "watch -n 10 cfn-describe-stack-events #{stack_name}"
+  #cmd = "watch -n 10 bin/get_stack_events.rb #{stack_name}"
+  #puts cmd
+  if STDIN.tty? && STDOUT.tty?
+    puts "STDIN and STDOUT are ttys!"
+    
+    begin
+      stackEvents = Set.new()
+      while true
+        get_stack_events( stack.name).reverse_each do |event|
+          if stackEvents.add?( event)
+            puts event
+          end
+        end
+        get_sub_stacks( stack.name).each do |stack|
+          get_stack_events( stack).reverse.each do |event|
+            if stackEvents.add?(event)
+              puts "NESTED: #{event}"
+            end
+          end
+        end
+        sleep 5
+      end
+    rescue SystemExit, Interrupt
+      exit 0
+    end
+  end
+  
+  # This is all working, tested code, just not sure it belongs here. --stsmith
+  #stack_timestamp = ''
+  #status = ''
+  #begin
+  #  loop do
+  #    # During an in-progress update, the parameters report the new values.
+  #    # If rollback happens, the parameters revert to their previous values.
+  #    # This behavior makes sense and is helpful.
+  #    stack_timestamp = stack.parameters['LastUpdateTimestamp']
+  #    status = stack.status
+  #    puts "#{stack_name} #{status} #{stack_timestamp}"
+  #    break unless status =~ /(CREATE|UPDATE)_IN_PROGRESS$/ && stack_timestamp == timestamp
+  #    sleep 5
+  #  end
+  #
+  #rescue AWS::CloudFormation::Errors::Throttling
+  #  puts "API request throttled."
+  #  retry
+  #end
+  #
+  #exit_status = 1
+  #if status =~ /(CREATE|UPDATE)_COMPLETE/ && stack_timestamp == timestamp
+  #    exit_status = 0
+  #end
+  #exit exit_status
+end
+
+def get_sub_stacks( stack_name)
+  stacks = Array.new()
+  AWS.memoize do
+    cfm = AWS::CloudFormation.new()
+    stack = cfm.stacks[stack_name]
+    stack.resource_summaries.each do |resource|
+      if resource[:resource_type] == 'AWS::CloudFormation::Stack'
+        stacks.push( resource[:physical_resource_id] )
+      end
+    end
+  end
+  return stacks
+end
+
+def get_stack_events( stack_name)
+
+  events = Array.new()
+
+	AWS.memoize do
+	  cfm = AWS::CloudFormation.new()
+	  stack = cfm.stacks[stack_name]
+	
+	  table = {
+	    'columns' => [ 'timestamp', 'resource_type', 'logical_resource_id', 'resource_status', 'resource_status_reason' ],
+	    'column_maxlengths' => {},
+	    'rows' => []
+	  }
+	
+	  stack.events.each do |event|
+	    row = {}
+	    table['columns'].each do |column_name|
+	      maxlength = table['column_maxlengths'][column_name] || 0
+	      value = sprintf("%s", event.send(column_name))
+	      row[column_name] = value
+	      maxlength = (value.length > maxlength) ? value.length : maxlength
+	      table['column_maxlengths'][column_name] = maxlength
+	    end
+	    table['rows'].push(row)
+	  end
+	
+	  header_row = {}
+	  table['columns'].each do |column_name|
+	    header_row[column_name] = column_name
+	  end
+	
+	  table['rows'].unshift(header_row)
+	  table['rows'].each do |row|
+	    row_text = ''
+	    last_column_name = table['columns'].last
+	    table['columns'].each do |column_name|
+	      if column_name != last_column_name
+	        maxlength = table['column_maxlengths'][column_name] || 0
+	        row_text += sprintf("%-*s", maxlength, row[column_name])
+	        row_text += ' '
+	      else
+	        row_text += sprintf("%s", row[column_name])
+	      end
+	    end
+	    #puts row_text
+      events.push( row_text)
+	  end
+	end
+  return events
+end
+
+
+def get_template_info()
+  gitrev = ''
+  gitstatus = ''
+  source = '' # ploy or git
+  
+  pwd = Dir.pwd
+
+  # Try from ploy package.
+  package = `dpkg -S #{pwd} 2>/dev/null`
+  if package && package =~ /^cloudformation-tools:/
+    gitrev = `dpkg-query -W -f='${gitrev}' cloudformation-tools`
+  end
+
+  #If stin and stdout or tty then someone is running this from command line.
+  if STDIN.tty? && STDOUT.tty?
+    source = 'cmd'
+  else
+    source = 'ploy'
+  end
+
+  # Try from git.
+  if gitrev == ''
+    gitrev = `/usr/bin/git log --format='%H' -n 1`
+    gitrev.chomp!
+    if gitrev
+      source = 'git'
+      gitstatus = `git status --porcelain`
+    end
+  end
+
+  instanceid = `/bin/netcat 169.254.169.254 80 -w 1 -q 0 </dev/null && /usr/bin/ec2metadata --instance-id`
+  instanceid.chomp!
+
+  template = {
+    :gitrev => gitrev,
+    :gitstatus => gitstatus,
+    :source => source,
+    :instanceid => instanceid
+  }
+end
+
+main()

data/lib/cfnpp/replacer.rb

@@ -0,0 +1,83 @@
+require 'json'
+require 'yaml'
+require 'erb'
+require 'pp'
+
+module CfnPP
+	# Implements logic for having textual templates that get turned
+	# into CloudFormation Fn::Join invocations. This makes userdata scripts
+	# and other things much easier to read.
+	#
+	# The templates are ERB syntax, with three functions available for
+	# inserting CloudFormation references: +cfn_raw+, +cfn_ref+ and +cfn_getatt+.
+	#
+	# == Example Template
+	#     #!/bin/bash
+	#     yum update -y aws-cfn-bootstrap
+	#     /opt/aws/bin/cfn-init -s <%= cfn_ref("AWS::StackId") %> -r LaunchConfig --region <%= cfn_ref("AWS::Region") %>
+	#     /opt/aws/bin/cfn-signal -e $? <%= cfn_ref("WaitHandle") %>
+	#     # Setup correct file ownership
+	#     chown -R apache:apache /var/www/html/wordpress
+	#
+	# == Example use of CfnPP::Replacer
+	#     template_text = "..."
+	#     cfn_hash = CfnReplacer.new.process(template_text)
+	class Replacer
+		# Turns input text into a Hash appropriate for inserting into a
+		# CloudFormation structure.
+    def initialize(text, vars = {}, opts = {})
+      @text = text
+      @r_vars = vars
+      @opts = opts
+    end
+
+		def process()
+			tmpl = ERB.new @text
+			joinparts = []
+			tmpl.result(self.get_binding).split('@@@').each do |chunk|
+				if chunk.match(/^\{/)
+					chunk = JSON.parse(chunk)
+				end
+				joinparts.push(chunk)
+			end
+			return { "Fn::Join" => [ '', joinparts ] }
+		end
+
+    def process_basic()
+      tmpl = ERB.new @text
+      return tmpl.result(self.get_binding)
+    end
+
+    def cfn_render(path)
+			txt = File.read File.join(File.dirname('.'), path)
+			tmpl = ERB.new txt
+			return tmpl.result(self.get_binding)
+    end
+
+    # for local refs (instead of cfn_ref)
+    def cfn_cfnpp_ref(s)
+      return @opts[s]
+    end
+
+		# called in the template to include any arbitrary CloudFormation code.
+		def cfn_raw(h)
+			txt = JSON.dump h
+			return "@@@#{txt}@@@"
+		end
+
+		# shortcut for inserting a cfn "Ref"; just pass the "Ref" value
+		def cfn_ref(s)
+			return cfn_raw({ "Ref" => s })
+		end
+
+		# shortcut for inserting a cfn "Fn::GetAtt"; just pass a two element
+		# array of the key and value to get
+		def cfn_getatt(k,v)
+			return cfn_raw({ "Fn::GetAtt" => [k, v]})
+		end
+
+		def get_binding
+			binding
+		end
+	end
+end

data/lib/cfnpp/templateresult.rb

@@ -0,0 +1,28 @@
+require 'digest'
+
+module CfnPP
+  class TemplateResult
+    attr_accessor :substacks
+    attr_accessor :data
+
+    def initialize(name, data, stack_url_base="", substacks = [])
+      @base_name = name
+      @data = data
+      @stack_url_base = stack_url_base
+      @substacks = substacks
+    end
+
+    def url
+      return "#{@stack_url_base}/#{name}/template.json"
+    end   
+
+    def name
+      return "#{@base_name}-#{checksum}"
+    end
+
+    def checksum
+      return Digest::SHA256.hexdigest(@data.to_json)
+    end
+  end
+end
+

data/lib/cfnpp/transform.rb

@@ -0,0 +1,273 @@
+require 'json'
+require 'yaml'
+require_relative 'replacer'
+require_relative 'templateresult'
+require 'set'
+require 'erb'
+
+# CfnPP is a module with various methods to make working with CloudFormation
+# templates easier. Particularly, it has facilities for writing templates in
+# YAML, with added features for modularizing templates, easily writing
+# user-data blocks, etc.
+module CfnPP
+  # This class has methods to read in cloudformation templates in YAML
+  # format with some Manta-specific extensions to make things easier.
+  class Transform
+    # returns a ruby hash, from a file at +path+
+    #
+    # This is the easiest way to load things. It takes care of
+    # setting reasonable file base for includes, etc., and gives
+    # you back a hash ready for use.
+    def self.load_file(path, opts = {}, name = "main", stack_url_base="http://example.com")
+      return self.load_yaml(File.read(path), path, opts, name, stack_url_base)
+    end
+
+    # returns a ruby hash, from unparsed YAML input text.
+    #
+    def self.load_yaml(yaml_txt, filebase=".", opts={}, name = "main", stack_url_base="http://example.com")
+      h = YAML::load(yaml_txt)
+      return self.new(h, filebase, opts, name, stack_url_base).as_template_result
+    end
+
+    # CfnPP::Transform is initialized with a hash and an optional file base
+    # parameter. The file base will default to ".", which may or may
+    # not be what you want.
+    def initialize(in_hash, filebase=".", opts={}, name="main", stack_url_base="http://example.com")
+      @name = name
+      @opts = opts
+      @filebase = filebase
+      @stack_url_base = stack_url_base
+      @in_hash = { :root => in_hash }
+      @tops = self.class.stdtops()
+      trans_hash(@in_hash)
+      @in_hash = @in_hash[:root]
+      @substacks = grab_stacks(@in_hash)
+      lift
+      @in_hash = apply_opts(@in_hash, opts)
+      prune(@in_hash)
+    end
+
+    # Return the parsed, processed CfnPP YAML file as a ruby hash
+    def as_hash
+      return @in_hash
+    end
+
+    def as_template_result
+      return CfnPP::TemplateResult.new(@name, @in_hash, @stack_url_base, @substacks)
+    end
+
+    private
+
+    # which keys always get lifted to the top
+    def self.stdtops
+      return Set.new ["Parameters", "Mappings", "Resources", "Outputs", "Conditions"]
+    end
+
+    # classic recursion!
+    def apply_opts(thing, opts)
+      if thing.is_a? Hash
+        if thing.has_key? 'CfnPPRef'
+          if not opts.has_key? thing['CfnPPRef']
+            raise "missing value for '#{thing['CfnPPRef']}'"
+          end
+          return "#{opts[thing['CfnPPRef']] || ''}"
+        else
+          r = {}
+          thing.each { |k,v| r[k] = apply_opts(v, opts) }
+          return r
+        end
+      elsif thing.is_a? Array
+        return thing.collect { |t| apply_opts(t, opts) }
+      else
+        return thing
+      end
+    end
+
+    # give the contents of a named file; handle any directory weirdness
+    # here
+    def read_ext(name)
+      return File.read File.join(File.dirname(@filebase), name)
+    end
+
+    # how the results of a CfnPPTemplate are put back into the tree
+    def sub_merge(h, key, v)
+      if v.is_a? Hash then
+        h[key].merge! v
+      else
+        h[key] = v
+      end
+    end
+
+    # sniff the type of filter based on the string contents
+    def auto_filter(txt)
+      if txt =~ /^---/
+        return "erb-yaml"
+      else
+        return "replacer"
+      end
+    end
+
+    # given the multiple ways a template can be specified, create
+    # a single predictable Hash to be used elsewhere
+    def norm_tmplspec(v)
+      if v.is_a? String
+        rec = {
+          "filter" => "",
+          "txt"    => "",
+          "vars"   => {},
+        }
+        if File.exists? File.join(File.dirname(@filebase), v)
+          rec["txt"] = read_ext v
+        else
+          rec["txt"] = v
+        end
+        rec["filter"] = auto_filter(rec["txt"])
+        return rec
+      elsif v.is_a? Hash
+        if v["path"]
+          v["txt"] = read_ext v["path"]
+        end
+        if not v.has_key? "filter"
+          v["filter"] = auto_filter(v["txt"])
+        end
+        return v
+      end
+    end
+
+    # Apply the given filter to the text content, with given
+    # vars.
+    # the overloading of Replacer here is super weird and
+    # ugly. Sorry. Will fix... soon?
+    def proc_tmplspec(rec)
+      replacer = Replacer.new(rec["txt"], rec["vars"], @opts)
+      if rec["filter"] == 'replacer'
+        return replacer.process
+      elsif rec["filter"] == 'erb-yaml'
+        res = replacer.process_basic
+        #puts "TEXT: #{res}"
+        #puts "#{res}"
+        #ERB.new rec["txt"]
+        return YAML::load(res)
+      end
+    end
+
+    # trans/trans_hash/trans_array walk the tree
+    def trans(e)
+      if e.is_a? Hash
+        trans_hash(e)
+      end
+      if e.is_a? Array
+        trans_array(e)
+      end
+    end
+
+    def trans_hash(h)
+      h.keys.each do |key|
+        if (h[key].is_a? Hash) and (h[key].has_key? "CfnPPTemplate")
+          rec = norm_tmplspec h[key]["CfnPPTemplate"]
+          v = proc_tmplspec(rec)
+          trans(v)
+          sub_merge(h, key, v)
+        elsif (h[key].is_a? Hash) and (h[key].has_key? "CfnPPStack")
+          rec = h[key]["CfnPPStack"]
+          if rec.has_key? "inline"
+            inline = rec["inline"]
+            name = rec["name"]
+            rec.delete("inline")
+            sub_params = rec["params"]
+            rec["result"] = self.class.new(inline, @filebase, sub_params.merge(@opts), name, @stack_url_base).as_template_result
+            rec["Resources"] = {} if not rec["Resources"]
+            rec["Resources"][name] = {
+              "Type" => "AWS::CloudFormation::Stack",
+              "Properties" => {
+                "TemplateURL" => rec["result"].url,
+                "Parameters" => sub_params,
+                "TimeoutInMinutes" => 60
+              }
+            }
+          end
+        else
+          trans(h[key])
+        end
+      end
+    end
+
+    def trans_array(a)
+      a.each do |e|
+        trans(e)
+      end
+    end
+
+    # recursively remove any keys we want cleaned up. due to lifting they
+    # can leave junk laying around
+    def prune(h)
+      prunes = ["CfnPPTemplate", "MantaTemplateInclude", "MantaTemplate", "MantaInclude", "CfnPPSection", "CfnPPStack"]
+      if h.is_a? Hash then
+        h.keys.each do |k|
+          if prunes.include? k then
+            h.delete(k)
+          else
+            prune(h[k])
+          end
+        end
+      elsif h.is_a? Array then
+        h.each { |e| prune(e) }
+      end
+    end
+
+    # return all of the embedded stack objects. super, super
+    # ugly
+    def grab_stacks(h)
+      stacks = []
+      if h.is_a? Hash
+        h.keys.each do |k|
+          if k == "CfnPPStack" and h[k].has_key? "result"
+            stacks.push(h[k]["result"])
+          else
+            stacks.concat(grab_stacks(h[k]))
+          end
+        end
+      end
+      return stacks
+    end
+
+    # given some defined top keys, find them everywhere, cut them out,
+    # and put them back in at the top level. Weirdly fiddly code.
+    def lift
+      def lifter(h, tops, store)
+        # this guard is a super ugly hacky
+        if h.has_key? 'Type' and h['Type'] == 'AWS::CloudFormation::Stack'
+          return
+        end
+        h.keys.each do |key|
+          if h[key].is_a? Hash
+            lifter(h[key], tops, store)
+          elsif h[key].is_a? Array
+            h[key].each do |e|
+              if e.is_a? Hash
+                lifter(e, tops, store)
+              end
+            end
+          end
+          if tops.include? key
+            if (not store[key])
+              store[key] = []
+            end
+            store[key].push h.delete(key)
+          end
+        end
+      end
+
+      h = @in_hash
+      store = {}
+      lifter(h, @tops, store)
+      store.keys.each do |k|
+        n = {}
+        store[k].each do |se|
+          n = n.merge se
+        end
+        h[k] = n
+      end
+    end
+  end
+end

data/lib/cfnpp/uploader.rb

@@ -0,0 +1,85 @@
+require 'aws-sdk'
+require 'json'
+require 'yaml'
+
+module CfnPP
+  class Uploader
+    def initialize(bucketname, stackname)
+      @bucket = AWS::S3.new.buckets[bucketname]
+      @stackname = stackname
+      @timestamp = new_timestamp
+      @s3_path = "testing/stacks/#{@stackname}/#{@timestamp}"
+      @cfn = AWS::CloudFormation.new
+    end
+
+    def new_timestamp
+      return Time.now.utc.strftime("%Y-%m-%dT%H.%M.%S.%LZ")
+    end
+
+    def s3_path
+      return @s3_path
+    end
+
+    def upload_template(template_result, opts)
+      r = { :url => nil, :error => nil, :validation_status => nil }
+
+      already_updated = ploy_update_guard(template_result.name, opts)
+      if already_updated
+        r[:err] = 'already_updated'
+        r[:prev_opts] = already_updated
+        return r
+      end
+
+      obj = @bucket.objects.create("#{@s3_path}/#{template_result.name}/template.json", template_result.data.to_json)
+      r[:url] = obj.public_url
+      r[:validation_status] = @cfn.validate_template(r[:url])
+      if r[:validation_status].has_key? :code #error condition
+        r[:error] = 'validation_error'
+      else
+        tp = r[:validation_status].fetch(:parameters, [])
+        r[:opts_parameters] = opts_parameters(tp, opts)
+        upload_parameters(template_result.name, r[:opts_parameters])
+      end
+      return r
+    end
+
+    def opts_parameters(params, opts)
+      opts_parameters = {}
+      params.each do |param|
+        key = param[:parameter_key]
+        opts_parameters[key] = opts[key] if opts.has_key? key
+      end
+      return opts_parameters
+    end
+
+    def upload_parameters(name, opts_parameters)
+      return @bucket.objects.create("#{@s3_path}/#{name}/parameters.yml", opts_parameters.to_yaml)
+    end
+
+    def ploy_update_guard(name, opts)
+      if opts['TemplateSource'] == 'ploy'
+        return find_previous_ploy_update(cfn_bucket, opts)
+      end
+      return nil
+    end
+
+    def find_previous_ploy_update(name, opts)
+      stack_name = opts['StackName']
+      launch_list = @bucket.objects.with_prefix("stacks/#{stack_name}").sort do |a,b|
+        a.key <=> b.key
+      end
+      launch_list.each do |o|
+        if o.key =~ /\/parameters.yml$/
+          prev_opts = YAML::load(o.read)
+          if prev_opts['TemplateSource'] == opts['TemplateSource']
+            if prev_opts['TemplateGitRevision'] == opts['TemplateGitRevision']
+              return prev_opts
+            end
+          end
+        end
+      end
+      return nil
+    end
+
+  end
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: cfnpp
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Michael Bruce
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &21160100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21160100
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &21157980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21157980
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: &21157160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21157160
+- !ruby/object:Gem::Dependency
+  name: travis
+  requirement: &21156220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21156220
+- !ruby/object:Gem::Dependency
+  name: dogapi
+  requirement: &21155240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21155240
+- !ruby/object:Gem::Dependency
+  name: ploy
+  requirement: &21154400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *21154400
+description: ! '["cfnpp", ["Michael Bruce"]]'
+email: mbruce@manta.com
+executables:
+- cfnpp
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cfnpp/uploader.rb
+- lib/cfnpp/transform.rb
+- lib/cfnpp/templateresult.rb
+- lib/cfnpp/replacer.rb
+- bin/cfnpp
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: cfnpp
+test_files: []