cloudformation-tool 0.1.1

data/bin/cftool

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'cloud_formation_tool'
+
+begin
+  CloudFormationTool::CLI::Main.run
+rescue CloudFormationTool::Errors::AppError => e
+  warn e.message
+end

data/lib/cloud_formation_tool.rb

@@ -0,0 +1,94 @@
+require 'logger'
+require 'autoloaded'
+require 'aws-sdk'
+
+def log(message = nil, &block)
+  ($__logger ||= Logger.new(STDERR)).info(if message.nil?
+    yield
+  else
+    message
+  end)
+end
+
+module CloudFormationTool
+  
+  Autoloaded.module do |autoloaded|
+    autoloaded.with :CLI 
+  end
+  
+  def find_profile(dir = nil)
+    dir ||= Dir.pwd
+    return profile if (dir == "/")
+    begin
+      return File.read("#{dir}/.awsprofile").chomp
+    rescue Errno::ENOENT
+      return find_profile(File.dirname(dir))
+    end
+  end
+  
+  def region
+    $__region ||= (ENV['AWS_DEFAULT_REGION'] || 'us-west-1')
+  end
+  
+  def profile
+    $__profile ||= (ENV['AWS_DEFAULT_PROFILE'] || 'default')
+  end
+  
+  def awscreds
+    $__aws_creds ||= Aws::SharedCredentials.new(profile_name: find_profile)
+  end
+  
+  def aws_config
+    {
+      credentials: awscreds,
+      region: region,
+      http_read_timeout: 5
+    }
+  end
+  
+  def awsec2
+    $__aws_ec2 ||= Aws::EC2::Client.new aws_config
+  end
+  
+  def awss3(s3reg = nil)
+    s3reg ||= region
+    ($__aws_s3 ||= {})[region] ||= Aws::S3::Client.new aws_config.merge(region: s3reg)
+  end
+  
+  def awscf
+    $__aws_cf ||= Aws::CloudFormation::Client.new aws_config
+  end
+  
+  def awsas
+    $__aws_as ||= Aws::AutoScaling::Client.new aws_config
+  end
+  
+  def s3_bucket_name(region)
+    name = nil
+    # see if we already have a cf-templates bucket for this region
+    bucket = awss3.list_buckets.buckets.select do |b|
+        b.name =~ /cf-templates-(\w+)-#{region}/
+    end.first
+    
+    # otherwise try to create one
+    if bucket.nil?
+      name = cf_bucket_name(region)
+      log("Creating CF template bucket #{name}")
+      awss3.create_bucket({
+        acl: "private",
+        bucket: name,
+        create_bucket_configuration: { location_constraint: region }
+      })
+      name
+    else
+      bucket[:name]
+    end
+  end
+  
+  def cf_bucket_name(region, key = nil)
+    # generate random key if one wasn't given
+    key ||= ((0...12).map { [*'a'..'z',*'0'..'9'][rand(36)] }.join)
+    "cf-templates-#{key}-#{region}"
+  end
+
+end

data/lib/cloud_formation_tool/cli.rb

@@ -0,0 +1,5 @@
+module CloudFormationTool
+  module CLI
+    Autoloaded.module {  }
+  end
+end

data/lib/cloud_formation_tool/cli/compile.rb

@@ -0,0 +1,19 @@
+module CloudFormationTool
+  module CLI
+    
+    class Compile < Clamp::Command
+      
+      parameter 'FILE', 'Template main file'
+      
+      def execute
+        if file.end_with? '.init'
+          puts CloudInit.new(file).compile
+        else
+          puts CloudFormation.parse(file).to_yaml
+  #        raise AppError.new("not a valid template file. Only .init and .yaml are supported")
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/create.rb

@@ -0,0 +1,50 @@
+module CloudFormationTool
+  module CLI
+    class Create < Clamp::Command
+      
+      parameter 'FILE', 'Template main file'
+      parameter '[STACK_NAME]', 'Name of the stack to create. Defaults to directory name'
+      
+      option [ "-p", "--param" ], "PARAM", [
+        "Parameter to use with the cloudformation, of the format Mame=Tag",
+        "Use multiple times to set multiple parameters.",
+        "See 'parameters' command to list the paramaters supported by the tempalte."
+        ].join("\n"), multivalued: true
+      option [ "-i", "--import" ], "FILE", "Import parameters from YAML file.", :attribute_name => :param_file
+      option [ "-k", "--import-key" ], "KEY", [
+        "When loading parameters from a YAML file, use the specified key to load a named",
+        "map from the file, instead of using just the file itself as the parameter map"
+        ].join("\n"), :attribute_name => :param_key
+      
+      def get_params
+        params = if param_file
+          yaml = YAML.load(File.read(param_file)).to_h
+          if param_key
+            raise "Missing parameter section '#{param_key}' in '#{param_file}'!" unless yaml[param_key].is_a? Hash
+            yaml[param_key]
+          else
+            yaml
+          end
+        else
+          Hash.new
+        end
+        # allow param_list to override parameters from the param file
+        param_list.inject(params) do |h, param|
+          k,v = param.split /\s*[=:]\s*/
+          h[k] = v
+          h
+        end
+      end
+      
+      def execute
+        name = stack_name || File.basename(File.dirname(File.expand_path(file)))
+        st = CloudFormation::Stack.new(name)
+        log "Creating stack #{name}"
+        start = Time.now
+        log "Created " + st.create(file, get_params).to_s
+        st.monitor(start)
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/delete.rb

@@ -0,0 +1,18 @@
+module CloudFormationTool
+  module CLI
+    
+    class Delete < Clamp::Command
+      
+      parameter "STACK_NAME", "Name of the stack to delete"
+      
+      def execute
+        st = CloudFormation::Stack.new(stack_name)
+        start = Time.now
+        st.delete
+        log "Deleted stack #{stack_name}"
+        st.monitor(start)
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/list_stacks.rb

@@ -0,0 +1,15 @@
+module CloudFormationTool
+  module CLI
+    
+    class ListStacks < Clamp::Command
+      include CloudFormationTool
+      
+      def execute
+        awscf.describe_stacks.stacks.each do |stack|
+          puts stack.stack_name.ljust(30,' ') + stack.stack_status
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/main.rb

@@ -0,0 +1,31 @@
+require 'clamp'
+require 'io/console'
+
+
+module CloudFormationTool
+  module CLI
+    class Main < Clamp::Command
+      
+    cftool = Class.new.include(CloudFormationTool).new
+      
+      banner "Compile and deploy CloudFormation templates"
+      
+      option [ "-r", "--region" ], "REGION", "AWS Region to use", default: cftool.region do |s|
+        $__region = s
+      end
+      
+      option [ "-p", "--profile" ], "PROFILE", "AWS credentials profile to use", default: cftool.profile do |s|
+        $__profile = s
+      end
+      
+      subcommand 'list', "List CloudFormation stacks", ListStacks
+      subcommand 'parameters', "List template parameters and their default values", Parameters
+      subcommand 'compile', "Compile the specified template", Compile
+      subcommand 'monitor', "Monitor recent and upcoming events on the stack", Monitor
+      subcommand 'create', "Create a stack from the template or update an existing stack", Create
+      subcommand 'status', "Check the current status of a stack", Status
+      subcommand 'delete', "Delete an existing stack", Delete
+      subcommand 'servers', 'List stack resources', Servers
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/monitor.rb

@@ -0,0 +1,28 @@
+module CloudFormationTool
+  module CLI
+    
+    class Monitor < Clamp::Command
+      
+      parameter 'STACK_NAME', "Name of the stack to monitor"
+      
+      option [ "-a", "--all" ], :flag, "Don't skip old events"
+      
+      def execute
+        begin
+          st = CloudFormation::Stack.new(stack_name)
+          st.see_events unless all?
+          while true
+            st.monitor
+            sleep 1
+          end
+        rescue CloudFormationTool::Errors::StackDoesNotExistError => e
+          log "Stack #{stack_name} does not exist"
+        rescue SystemExit, Interrupt => e
+          # CTRL-C out of the loop
+          puts "\n"
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/parameters.rb

@@ -0,0 +1,20 @@
+module CloudFormationTool
+  module CLI
+    
+    class Parameters < Clamp::Command
+      
+      parameter 'FILE', 'Template main file'
+      def execute
+        donefirst = false
+        CloudFormation.parse(file).each do |name, value|
+          unless donefirst
+            donefirst = true
+            puts "---\n"
+          end
+          puts "#{name}: #{value}"
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/servers.rb

@@ -0,0 +1,30 @@
+require 'set'
+
+module CloudFormationTool
+  module CLI
+    
+    class Servers < Clamp::Command
+      include CloudFormationTool
+      
+      parameter "STACK_NAME", "Name of the stack to delete"
+      
+      def execute
+        st = CloudFormation::Stack.new(stack_name)
+        ts = st.asgroups.collect do |res|
+          Thread.new do
+            awsas.describe_auto_scaling_groups({
+              auto_scaling_group_names: [ res.physical_resource_id ]
+            }).auto_scaling_groups.first.instances.collect do |i|
+              Aws::EC2::Instance.new i.instance_id, client: awsec2
+            end.collect do |i|
+              "#{res.logical_resource_id.ljust(30, ' ')} #{i.public_dns_name}"
+            end
+          end
+        end
+        ts.each(&:join)
+        puts ts.collect { |t| t.value.join "\n" }
+      end
+      
+    end
+  end
+end

data/lib/cloud_formation_tool/cli/status.rb

@@ -0,0 +1,18 @@
+module CloudFormationTool
+  module CLI
+    
+    class Status < Clamp::Command
+      
+      parameter "STACK_NAME", "Name of the stack to delete"
+        
+      def execute
+        if CloudFormation::Stack.new(stack_name).exist?
+          log "OK"
+        else
+          log "Stack #{stack_name} does not exist"
+        end
+      end
+
+    end
+  end
+end

data/lib/cloud_formation_tool/cloud_formation.rb

@@ -0,0 +1,153 @@
+require 'yaml'
+require 'base64'
+
+module CloudFormationTool
+  class CloudFormation
+    
+    Autoloaded.class {  }
+    
+    def self.parse(path)
+      CloudFormation.new(path)
+    end
+    
+    def initialize(path)
+      log "Loading #{path}"
+      @path = path
+      @path = "#{@path}/cloud-formation.yaml" if File.directory? @path
+      @path = "#{@path}.yaml" if !File.exist? @path and File.exist? "#{@path}.yaml"
+      @basedir = File.dirname(@path)
+      @compiled = false
+      text = File.read(@path)
+      # remove comments because white space seen between comments can seriously psych Psych
+      text.gsub!(/^#.*\n/s,'')
+      text = fixShorthand(text)
+      begin
+        @data = YAML.load(text).to_h
+      rescue Psych::SyntaxError => e
+        e.message =~ /line (\d+) column (\d+)/
+        lines = text.split "\n"
+        raise AppError, "Error parsing #{path} at line #{e.line} column #{e.column}:\n" +
+          "#{lines[e.line-1]}\n" +
+          "#{(' ' * (e.column - 1 ))}^- #{e.problem} #{e.context}"
+      rescue Errno::ENOENT => e
+        raise AppError, "Error reading #{path}: #{e.message}"
+      end
+    end
+    
+    def compile
+      return @data if @compiled
+      @compiled = true
+      embed_includes
+      @data = load_files(@data)
+    end
+    
+    def to_yaml
+      compile.to_yaml
+    end
+    
+    def fixShorthand(text)
+      text.gsub(/(?:(\s*)([^![:space:]]+))?(\s+)!(\w+)/) do |match|
+        case $4
+        when *%w(Base64 FindInMap GetAtt GetAZs ImportValue Join Select Sub
+          And Equals If Not Or)
+          ($2.nil? ? "" : "#{$1}#{$2}\n#{$1} ") + "#{$3}\"Fn::#{$4}\":"
+        when 'Ref'
+          "#{$1}#{$2}\n#{$1} #{$3}#{$4}:"
+        else
+          match
+        end
+      end
+    end
+    
+    def fixrefs(data, rmap)
+      case data
+      when Hash
+        data.inject({}) do |h,(k,v)|
+          h[k] = if k == "Ref"
+            rmap[v] || v
+          else
+            fixrefs(v,rmap)
+          end
+          h
+        end
+      when Array
+        data.collect do |item|
+          fixrefs(item, rmap)
+        end
+      else
+        return data
+      end
+    end
+    
+    def embed_includes
+      (@data.delete(@data.keys.find{|k| k.start_with? 'Include'}) || []).each do |path|
+        realpath = "#{@basedir}/#{path}"
+        cfile_key = File.dirname(realpath).gsub(%r{/(.)}){|m| $1.upcase }.gsub(/\W+/,'')
+        rewrites = Hash.new
+        CloudFormation.new(realpath).compile.each do |category, catdata|
+          # some categories are meta-data that we can ignore from includes
+          next if %w(AWSTemplateFormatVersion Description).include? category
+          
+          case category
+          when "Parameters"
+            @data[category].each do |name, param|
+              if catdata.has_key? name
+                next if param['Default'] == catdata[name]['Default']
+                 
+                if catdata[name].has_key?('Override') and catdata[name]['Override'] == false
+                  catdata.delete(name)
+                else
+                  newname = "#{cfile_key}z#{name}"
+                  log "Rewriting conflicting parameter #{name} (='#{catdata[name]['Default']}') to #{newname}"
+                  catdata[newname] = catdata.delete name
+                  rewrites[name] = newname
+                end
+              else
+                @data[category][name] = param
+              end
+            end
+          else
+            # warn against duplicate entities, resources or outputs
+            (@data[category] ||= {}).keys.each do |key|
+              if catdata.has_key? key
+                raise AppError.new("Error compiling #{path} - duplicate '#{category}' item: #{key}")
+              end 
+            end
+            catdata = fixrefs(catdata, rewrites)
+          end
+          
+          # add included properties
+          @data[category].merge! catdata
+        end
+      end
+    end
+    
+    def load_files(data)
+      case data
+      when Array
+        data.collect { |data| load_files(data) }
+      when Hash
+        data.inject({}) do |dict, (key, val)|
+          dict[key] = if (key == "UserData") and (val["File"]) 
+            # Support LaunchConfiguration UserData from file
+            CloudInit.new("#{@basedir}/#{val["File"]}").to_base64
+          elsif (key == "Code") and (val["URL"])
+            # Support Lambda Code from arbitrary URLs
+            LambdaCode.new(val["URL"]).to_cloudformation 
+          else 
+            load_files(val)
+          end
+          dict
+        end
+      else
+        data
+      end
+    end
+    
+    def each
+      compile['Parameters'].each do |name, param|
+        yield name, param['Default']
+      end
+    end
+  end
+end