aws-must 0.0.2

data/lib/aws-must/aws-must.rb

@@ -0,0 +1,96 @@
+require 'yaml'
+require 'json'
+
+module AwsMust
+
+  class AwsMust
+
+    include Utils::MyLogger          # mix logger
+    PROGNAME = "aws-must"            # logger progname
+    include Utils::Hasher            # mix logger
+
+    # ------------------------------------------------------------------
+    # attributes
+
+    attr_accessor  :template         # mustache (or something else)
+     
+    def initialize( options={} )
+
+      # init logger
+      @logger = getLogger( PROGNAME, options )
+      @logger.info( "#{__method__}" )
+
+      # create object, which knows how to generate CloudFormation template json
+      @template = Template.new( options ) 
+      @docu     = Docu.new( @template, options )
+    end
+
+    # ------------------------------------------------------------------
+    # dump 'yaml_file' as json
+
+    def json( yaml_file ) 
+
+      @logger.debug( "#{__method__}, template_name '#{yaml_file}'" )
+
+      # 
+      data = read_yaml_file( yaml_file )
+
+      puts data.to_json
+
+
+    end # 
+
+    # ------------------------------------------------------------------
+    # extract documentation  from 'template_name' 
+
+    def doc( template_name ) 
+
+      @logger.debug( "#{__method__}, template_name '#{template_name}'" )
+      
+      @docu.document( template_name )
+
+
+    end
+
+    # ------------------------------------------------------------------
+    # generate JSON by rendering 'template_name' using configurations
+    # from 'yaml_file'
+
+    def generate( template_name, yaml_file, options ) 
+
+      @logger.debug( "#{__method__}, template_name '#{template_name}'" )
+      @logger.debug( "#{__method__}, yaml_file= '#{yaml_file}'" )
+
+      # 
+      data = read_yaml_file( yaml_file )
+
+      # some 'quirks' on data 
+      data = adjust( data )
+      @logger.debug( "#{__method__}, data= '#{data}'" )
+
+      puts template.to_str( template_name, data )
+
+    end
+
+    private
+
+    # deeply iterate 'data' -hash and add comma (,) to all expect the
+    # last hash in arrays
+    def adjust( data )
+      return addComma( data )
+    end
+
+
+    def  read_yaml_file( yaml_file ) 
+
+      raise "YAML file #{yaml_file} does not exist" unless File.exist?( yaml_file )
+      data = YAML.load_file( yaml_file )
+
+      return data
+
+    end
+
+
+  end
+
+end

data/lib/aws-must/docu.rb

@@ -0,0 +1,164 @@
+
+module AwsMust
+
+  class Docu 
+
+    include ::Utils::MyLogger        # mix logger
+    PROGNAME = "docu"                # progname for logger
+
+    # ------------------------------------------------------------------
+    # constants
+
+    DEFAULT_OPEN_TAG      ="++start++"
+    DEFAULT_CLOSE_TAG     ="++close++"
+    DEFAULT_INCLUDE_TAG   =">"
+
+
+    # ------------------------------------------------------------------
+    # Attributes
+
+    # static
+
+
+    # instance 
+
+    # ------------------------------------------------------------------
+    # Constructor
+    # - template: which knows how to return template files
+    
+    def initialize( template, options={} )
+
+      @logger = getLogger( PROGNAME, options )
+      @logger.info( "#{__method__}, options=#{options}" )
+
+      @template = template
+
+      @directives = {
+        :open => DEFAULT_OPEN_TAG,
+        :close => DEFAULT_CLOSE_TAG,
+        :include => DEFAULT_INCLUDE_TAG,
+      }
+
+    end
+
+    # ------------------------------------------------------------------
+    # docu - output documentation to stdout
+
+    def document( template_name )
+
+      @logger.debug( "#{__method__}, template_name '#{template_name}'" )
+      
+      # 
+      template_string = @template.get_template( template_name )
+      @logger.debug( "#{__method__}, template_string= '#{template_string}'" )
+
+      # 
+      state_opened=false
+
+      # iterate lines
+      template_string && template_string.split( "\n" ).each do | line |
+
+        if !state_opened then
+
+          # waiting for +++open+++
+
+          open_found, line = directive_open( line )
+          if open_found then
+
+            state_opened = true
+            
+            # something follows the start -tag
+            redo if line && ! line.empty?
+
+          end
+
+        else 
+
+          # seen +++open+++, waiting for +++close+++
+
+          close_found, line_pre, line = directive_close( line )
+          # puts "close_found=#{close_found}, #{close_found.class}, line=#{line}"
+          if  close_found then
+            output( line_pre ) if line_pre && !line_pre.empty?
+            state_opened = false
+            redo if line && ! line.empty?
+          elsif template_name = directive_include( line ) then
+            document( template_name )
+          else
+            output( line )
+          end
+
+        end # state_opened
+      end # each line
+
+    end # end - document
+
+    # ------------------------------------------------------------------
+    # output - document output
+
+    def output( line )
+
+      puts( line ) 
+    end
+
+
+    # ------------------------------------------------------------------
+    # Check if line satisfies tag
+
+    def directive_open( line )
+      # return line && line.include?( get_tag(:open) )
+      if line && line.include?( get_tag(:open) ) then
+        # find index pointing _after_ :open tag
+        index = line.index( get_tag(:open) ) + get_tag(:open).length
+        # # return text after directtive
+        # return  true, line[ index..-1]
+
+        # do not output anything for the line
+        return  true, nil
+
+      else
+        return false, nil
+      end
+    end
+    
+    # return bool, line_pre, line
+    def directive_close( line )
+      #return line && line.include?( get_tag( :close ) )
+      if line && line.include?( get_tag( :close ) ) then
+        index = line.index( get_tag(:close) )
+        # return true, index == 0 ? nil : line[0..index-1],  line[ index+ get_tag(:close).length..-1]
+        # do not output anythin on line for directive
+        return true, nil, nil
+      else
+        return false, nil, line
+      end
+    end
+    
+
+    # return include 'template' to include if it is include directive
+    def directive_include( line )
+
+      # not valid input
+      return nil unless line
+      
+      # extract template_name using regexp
+      parse_regexp=/^#{get_tag(:include)}\s+(?<template>[\w]*)\s*/
+      parsed = line.match( parse_regexp )
+
+      # not found
+      return nil unless parsed
+      return parsed['template']
+
+    end
+
+    # ------------------------------------------------------------------
+    # get tag value
+    def get_tag( tag )
+      raise "Unknown tag" unless @directives[tag]
+      return @directives[tag]
+    end
+    
+
+  end # class
+
+end # module

data/lib/aws-must/template.rb

@@ -0,0 +1,81 @@
+
+require 'mustache'                   # extendending implementation of
+
+
+module AwsMust
+
+  class Template < Mustache
+
+    include ::Utils::MyLogger        # mix logger
+    PROGNAME = "template"            # progname for logger
+
+    # ------------------------------------------------------------------
+    # Attributes
+
+    # static
+    @@template_path = nil            # directory where templates stored
+    @@template_extension = "mustache"# type part in filename
+
+    # instance 
+    attr_writer :partials            # f: partial-name --> template string
+    attr_writer :templates           # f: template-name --> template string
+
+    # ------------------------------------------------------------------
+    # Constructor
+    
+    def initialize( options={} )
+      @logger = getLogger( PROGNAME, options )
+      @logger.info( "#{__FILE__}.#{__method__} created" )
+      @logger.debug( "#{__FILE__}.#{__method__}, options='#{options}" )
+      # for mustache templates
+      @@template_path = options[:template_path] if options[:template_path]
+    end
+
+    # ------------------------------------------------------------------
+    # Services
+    
+    def to_str( template_name, data )
+      @logger.debug( "#{__method__}: template_name=#{template_name}, data=#{data}" )
+      if template_name 
+        template = get_template( template_name )
+        render( template, data )
+      else
+        render( data )
+      end
+    end
+
+    # ------------------------------------------------------------------
+    # Integrate with mustache
+
+    # method used by mustache framework - delegate to 'get_partial'
+    def partial(name)
+      @logger.debug( "#{__FILE__}.#{__method__} name=#{name}" )
+      # File.read("#{template_path}/#{name}.#{template_extension}")
+      get_partial( name )
+    end
+
+    # cache @partials - for easier extension
+    def get_partial( name ) 
+      @logger.debug( "#{__FILE__}.#{__method__} name=#{name}" )
+      return @partials[name] if @partials && @partials[name]
+
+      partial_file = "#{@@template_path}/#{name}.#{template_extension}"
+      @logger.info( "#{__FILE__}.#{__method__} read partial_file=#{partial_file}" )
+      File.read( partial_file )
+    end
+
+    # hide @templates - for easier extension
+    def get_template( name ) 
+
+      @logger.debug( "#{__FILE__}.#{__method__} name=#{name}" )
+      return @templates[name] if @templates && @templates[name]
+
+      template_path = "#{@@template_path}/#{name}.#{@@template_extension}"
+      @logger.info( "#{__FILE__}.#{__method__} read template_path=#{template_path}" )
+
+      File.read( template_path )
+    end
+
+  end # class
+
+end # module

data/lib/aws-must.rb

@@ -0,0 +1,11 @@
+module AwsMust
+
+  require_relative 'utils/logger'
+  require_relative 'utils/hasher'
+  require_relative 'aws-must/aws-must'
+  require_relative 'aws-must/template'
+  require_relative 'aws-must/docu'
+
+end 
+
+

data/lib/tasks/demo.rake

@@ -0,0 +1,139 @@
+# -*- mode: ruby -*-
+
+# demo.rake
+
+# Thank You!
+# http://andyatkinson.com/blog/2014/06/23/sharing-rake-tasks-in-gems
+
+require 'json'
+
+namespace "demo" do |ns|
+
+  # --------------------
+  # Demo configs
+
+  cmd = File.join File.dirname(__FILE__), "../../bin/aws-must.rb"   # generator being demonstrated
+  demo_dir=File.join File.dirname(__FILE__), "../../demo"           # directory holding demo configs
+  stack="demo"                                                      # stack name of the on Amazon
+  default_private_key_file="~/.ssh/demo-key/demo-key"                       # private keyfile corresponding 
+
+  all_regions= ["ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "cn-north-1", "eu-central-1", "eu-west-1", "sa-east-1", "us-east-1", "us-gov-west-1", "us-west-1", "us-west-2"] 
+
+  [ 
+
+   { :id => "1", :desc=>"Initial copy", :region=>['eu-central-1'], :ssh => false },
+   { :id => "2", :desc=>"Added 'description' -property to YAML file", :region=>['eu-central-1'], :ssh => false  },
+   { :id => "3", :desc=>"Use resources.mustache -partial to create EC2 intance", :region=>['eu-central-1'], :ssh => false    },
+   { :id => "4", :desc=>"EC2 instance configuration using YAML-data", :region=>['eu-central-1'], :ssh => false    },
+   { :id => "5", :desc=>"Add 'Outputs' -section with reference to EC2 instance", :region=>['eu-central-1'], :ssh => false    },
+   { :id => "6", :desc=>"Add 'Inputs' and 'Mappings' -sections to parametirize", :region=> all_regions, :ssh => false  },
+   { :id => "7", :desc=>"Added support for input parameters, EC2 tags, Instance security group", :region=>all_regions, :ssh => false  },
+
+
+  ].each do  |c|
+
+    desc "Output CF template using configs in '#{demo_dir}/#{c[:id]}' to demonstrate '#{c[:desc]}'"
+    task "template-#{c[:id]}" do
+      sh "#{cmd} gen -t #{demo_dir}/#{c[:id]} #{demo_dir}/#{c[:id]}/conf.yaml"
+    end
+
+    desc "Output configs in '#{demo_dir}/#{c[:id]}' in JSON to demonstrate '#{c[:desc]}'"
+    task "json-#{c[:id]}" do
+      sh "#{cmd} json #{demo_dir}/#{c[:id]}/conf.yaml"
+    end
+
+    desc "Check the difference between version '#{c[:id]}' and '#{c[:id].to_i() -1}' "
+    task "diff-#{c[:id]}" do
+      prev = c[:id].to_i() -1
+      # use bash temporary names pipes to diff two stdouts
+      # Use `jq` to create canonical pretty print json
+      sh "bash -c \"diff <(#{cmd} gen -t #{demo_dir}/#{c[:id]} #{demo_dir}/#{c[:id]}/conf.yaml | jq . ) <(#{cmd} gen -t #{demo_dir}/#{prev} #{demo_dir}/#{prev}/conf.yaml | jq . ) || true \""
+    end
+
+    desc "Create stack #{stack} for demo case #{c[:id]}, supported regions=#{c[:region]}"
+    task "stack-create-#{c[:id]}" do
+      sh "aws cloudformation create-stack --stack-name #{stack}  --template-body \"$(#{cmd} gen -t #{demo_dir}/#{c[:id]} #{demo_dir}/#{c[:id]}/conf.yaml)\""
+    end
+
+    desc "Open html documentation in 'browser' (default 'firefox')"
+    task "html-#{c[:id]}", :browser  do |t,args|
+
+      args.with_defaults( browser: "firefox" )
+
+      sh "#{cmd} doc -t #{demo_dir}/#{c[:id]} | markdown | #{args.browser} \"data:text/html;base64,$(base64 -w 0 <&0)\""
+    end
+
+
+    desc "Copy demo case #{c[:id]} to :templateDir and :configDir"
+    task "bootstrap-#{c[:id]}", :templateDir,:configDir do  |t, args|
+
+      usage =  "Task '#{t}' usage: #{t}[:templateDir,:configDir]" 
+      raise usage if args.templateDir.nil?
+      raise usage if args.configDir.nil?
+
+      raise "Directory #{args.templateDir} does not exist" unless File.exists?( args.templateDir )
+      raise "Directory #{args.configDir} does not exist" unless File.exists?( args.configDir )
+
+
+      puts (<<-EOS) if args.templateDir == args.configDir
+           
+              Warning :templateDir == :configDir == #{args.templateDir}
+              Suggestion to keep templates and configurations in a separate directory
+
+      EOS
+
+      demoTemplates = File.join demo_dir, c[:id], "*.mustache"
+      configFiles = File.join demo_dir, c[:id], "*.yaml"
+
+      Dir[demoTemplates].each { |f| 	
+        FileUtils.cp(f, args.templateDir, :verbose=>true )
+      }
+
+      Dir[configFiles].each { |f| 	
+        FileUtils.cp(f, args.configDir, :verbose=>true )
+      }
+
+    end
+
+
+  end
+
+  desc "Show status of CloudFormation stack"
+  task "stack-status" do
+    sh "aws cloudformation describe-stacks"
+  end
+
+  desc "Ssh connection to ':ipName' using ':private_key' default (#{default_private_key_file})"
+  task 'stack-ssh', :ipName, :private_key  do |t, args|
+
+    raise "Task '#{t}' usage: #{t}[ipName,private_key] - exiting"  if args.ipName.nil?
+
+    stack_string = %x{ aws cloudformation describe-stacks --stack-name #{stack} }
+    raise "Error #{$?.exitstatus}" if $?.exitstatus != 0
+
+    args.with_defaults( private_key: default_private_key_file )
+
+    stack_json = JSON.parse( stack_string )
+    ip_json = stack_json["Stacks"][0]['Outputs'].select { |o| o['OutputKey'] == args.ipName }.first
+    raise "Could not find ip for 'ipName' '#{args.ipName}' in output section of #{stack} stack" unless ip_json
+
+    puts ip_json['OutputValue']
+    identity="-i #{args.private_key}"
+
+    sh "ssh ubuntu@#{ip_json['OutputValue']} #{identity}"
+  end
+
+
+  desc "Show events of CloudFormation stack '#{stack}'"
+  task "stack-events" do
+    sh "aws cloudformation describe-stack-events --stack-name #{stack}"
+  end
+
+  desc "Delete stack CloudFormation stack '#{stack}'"
+  task "stack-delete" do
+    sh "aws cloudformation delete-stack --stack-name #{stack}"
+  end
+
+
+end
+

data/lib/utils/hasher.rb

@@ -0,0 +1,83 @@
+# see http://hawkins.io/2013/08/using-the-ruby-logger/
+
+# ------------------------------------------------------------------
+# Open up class Hash to add `deep_traverse`
+
+
+class Hash 
+  def deep_traverse(&block)
+    self.each do |k,v|
+      puts k,v
+      yield( k,v)
+    end
+  end
+  #   stack = self.map{ |k,v| [ [k], v ] }
+  #   while not stack.empty?
+  #     key, value = stack.pop
+  #     yield(key, value)
+  #     if value.is_a? Hash
+  #       value.each{ |k,v| stack.push [ key.dup << k, v ] }
+  #     end
+  #   end
+  # end
+
+end
+
+
+
+# ------------------------------------------------------------------
+# implement my own 
+
+module Utils
+
+  module Hasher
+
+    # see http://stackoverflow.com/questions/3748744/traversing-a-hash-recursively-in-ruby
+
+    def dfs(hsh, &blk)
+      return enum_for(:dfs, hsh) unless blk
+
+      # no change unless a hash
+      return hsh unless hsh.is_a? Hash
+
+      yield hsh
+      hsh.each do |k,v|
+        if v.is_a? Array
+          v.each do |elm|
+            dfs(elm, &blk)
+          end
+        elsif v.is_a? Hash
+          dfs(v, &blk)
+        end
+      end
+      return hsh
+      # nil
+    end
+
+    # adds comma hash in arrays deeply in obj-hash
+    def addComma( obj )
+
+      if ( obj.is_a?(Hash) ) 
+
+        obj2 = dfs(obj) do |o| 
+          o.each do |k,v| 
+            if v.is_a? Array
+              # add :comma -property expect to the last element
+              v.slice(0,v.length-1).each do |elem|
+                elem["_comma"] = "," if elem.is_a? Hash
+              end
+            end
+          end
+        end # block
+
+        obj = obj2
+
+      end
+
+      
+      return obj
+    end
+
+  end
+
+end # module Utils

data/lib/utils/logger.rb

@@ -0,0 +1,71 @@
+require 'logger'
+
+# see http://hawkins.io/2013/08/using-the-ruby-logger/
+
+
+module Utils
+
+  module MyLogger
+
+    # no logging done
+
+    class NullLoger < Logger
+      def initialize(*args)
+      end
+
+      def add(*args, &block)
+      end
+    end
+
+    LOGFILE="apu.tmp"
+
+    def getLogger( progname, options={} ) 
+
+      level = get_level( options )
+
+      if level.nil? 
+
+        return NullLoger.new 
+
+      else
+        
+        logger = Logger.new( LOGFILE )
+        logger.level=level
+        logger.progname = progname
+        return logger
+
+      end 
+
+    end # getLogger
+
+    # ------------------------------------------------------------------
+    private
+
+    def get_level( options ) 
+
+      # puts  "#{__method__}: options=#{options}" 
+
+      level_name = options && options[:log] ? options[:log] : ENV['LOG_LEVEL']
+
+      level = case  level_name
+              when 'warn', 'WARN'
+                Logger::WARN
+              when 'info', 'INFO'
+                Logger::INFO
+              when 'debug', 'DEBUG'
+                Logger::DEBUG
+              when 'error', 'ERROR'
+                Logger::ERROR
+              else
+                nil
+              end
+
+      return level
+    end
+
+
+
+  end
+  
+
+end