aws-must-templates 0.1.2

data/lib/tasks/suite.rake

@@ -0,0 +1,292 @@
+# -*- mode: ruby -*-
+
+# ------------------------------------------------------------------
+# requires
+
+require 'yaml'
+require 'json'
+require 'rake'
+require 'rspec/core/rake_task'
+
+# ------------------------------------------------------------------
+# test-suites.yaml
+
+require_relative  "../test-suites/test_suites.rb"
+test_suites = AwsMustTemplates::TestSuites::TestSuites.new
+
+# ------------------------------------------------------------------
+# configs
+
+aws_must          = "aws-must.rb"         # command to conver yaml-configs to cf-templates
+
+# read json using aws cli
+describe_stacks_command  =  "aws cloudformation describe-stacks"    
+
+# stack states
+SUCESS_STATES  = ["CREATE_COMPLETE", "UPDATE_COMPLETE"]
+FAILURE_STATES = ["CREATE_FAILED", "DELETE_FAILED", "UPDATE_ROLLBACK_FAILED", "ROLLBACK_FAILED", "ROLLBACK_COMPLETE","ROLLBACK_FAILED","UPDATE_ROLLBACK_COMPLETE","UPDATE_ROLLBACK_FAILED"]
+END_STATES     = SUCESS_STATES + FAILURE_STATES
+
+
+# ------------------------------------------------------------------
+# namespace :suite
+
+namespace :suite do
+
+  # suite_properties.each{  |a| a.keys.first }
+
+  # **********
+  desc "Run all suites"
+  task :all, :gen_opts   do |t,args|
+    Rake::Task['suite:clean'].invoke()
+    Rake::Task['suite:suites'].invoke(args.gen_opts)
+  end
+
+  task  :clean do 
+
+    files =   FileList[ "#{suite_test_report_dirpath()}/**/*"]
+    files.exclude { |f|  File.directory?(f) }
+
+    rm_rf files unless files.empty?
+
+  end
+
+  # **********
+  all_suites = test_suites.suite_ids.map{ |id| "suite:" + id }
+  task :suites, :gen_opts  do |t,args|
+
+    failed_suites = []
+
+    all_suites.each do |t|
+      begin
+        Rake::Task[t].invoke( args.gen_opts ) 
+        failed_suites << t unless $?.success?
+        # # Run in isolation && continue no matter what
+        # sh "rake #{t}; true"
+      rescue => e
+        puts "#{e.class}: #{e.message}"
+        failed_suites << t
+        # puts e.backtrace
+        puts "continue with next suite"
+      end
+
+    end # all_suites.each
+
+    if failed_suites.any?  then
+      puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      puts "Failed suites"
+      puts failed_suites
+      puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      raise "failed_suites=#{failed_suites}"
+    end
+
+  end # task :all
+
+  # **********
+  test_suites.suite_ids.each do |suite_id|
+
+
+    # suite = test_suites.get_suite( suite_id )
+    
+    # find the stack name for suite
+    stack = test_suites.get_suite_stack_id( suite_id )
+
+    # **********
+    # Create stack for a suite
+    desc "Create stack #{stack} for suite #{suite_id}"
+    task "#{suite_id}-stack-create", :gen_opts  do |t,args|
+      args.with_defaults( :gen_opts => "-m aws-must-templates" )
+      json_template="#{aws_must} gen #{stack}.yaml #{args.gen_opts}"  
+      sh "aws cloudformation create-stack --stack-name #{stack} --capabilities CAPABILITY_IAM  --template-body \"$(#{json_template})\"  --disable-rollback"
+    end
+
+    desc "Create stack #{stack} for suite #{suite_id}"
+    task "#{suite_id}-stack-wait" do
+
+      while true
+
+        stack_json = JSON.parse( %x{ #{describe_stacks_command} } )
+        stack_json = stack_json["Stacks"].select{ |a| a["StackName"] == stack }.first
+        raise "Could not find stack '#{stack}'" unless stack_json
+
+        break if END_STATES.include?( stack_json["StackStatus"] ) 
+
+        print "."
+        $stdout.flush
+
+        sleep 10
+
+      end
+
+    end
+
+
+    # **********
+    # delete stack for a suite
+    desc "Delete stack #{stack} for suite #{suite_id}"
+    task "#{suite_id}-stack-delete" do
+      sh "aws cloudformation delete-stack --stack-name #{stack}"
+    end
+
+    task :report_dir  do 
+      t = suite_test_report_dirpath()
+      sh "mkdir -p #{t}" unless File.exist?(t) 
+    end
+
+    # **********
+    desc "Show status for stack #{stack}"
+    task "#{suite_id}-stack-status" do
+      sh "aws cloudformation describe-stacks --stack-name #{stack}"
+    end
+
+    # **********
+    # Test common roles for suite
+    desc "Suite #{suite_id} - common roles"
+    RSpec::Core::RakeTask.new( "#{suite_id}-common" ) do |t|
+      puts "------------------------------------------------------------------"
+      puts "suite=#{suite_id }"
+
+      # see spec/spec_helper.rb
+      ENV['TARGET_SUITE_ID'] = suite_id
+
+      # test all roles for the instance
+      t.rspec_opts = rspec_opts( suite_id )
+      t.fail_on_error = false       
+      t.ruby_opts= rspec_ruby_opts
+      # t.pattern = suite["roles"].map {  |r|  spec_pattern( r ) }.join(",")
+      t.pattern = test_suites.suite_role_ids( suite_id ).map{ |r| spec_pattern( r ) }.join(",")
+
+
+    end if test_suites.suite_roles( suite_id )
+
+
+    # **********
+    # Run tasks for suite suite_id
+
+    desc "Run all takss for suite '#{suite_id}' - {suite['desc']"
+    suite_tasks = 
+      [ 
+       "suite:report_dir", 
+       [ "suite:#{suite_id}-stack-create", "gen_opts" ],
+        "suite:#{suite_id}-stack-wait", 
+      ] + 
+      ( Rake::Task.task_defined?(  "suite:#{suite_id}-common"  ) ? [  "suite:#{suite_id}-common"  ] : [] )  + 
+      # ( suite["instances"] ? suite["instances"].each.map{ |a| "suite:#{suite_id}:" + a.keys.first } : []  ) + 
+      ( test_suites.suite_instance_ids( suite_id ).each.map{ |instance_id| "suite:#{suite_id}:" + instance_id }  ) + 
+      [ "suite:#{suite_id}-stack-delete" ] 
+
+    task suite_id, :gen_opts do |ta,args|
+
+      failed_tasks = []
+
+      suite_tasks.each do |task|
+        begin
+          if task.kind_of?( Array )
+            taskname = task.shift
+            Rake::Task[taskname].invoke( *(task.select{ |arg_name| args[arg_name]}.map{ |arg_name| args[arg_name] }) )
+          else
+            Rake::Task[task].invoke( args )
+          end
+          failed_tasks << task unless $?.success?
+          # # Run in isolation && continue no matter what
+          # sh "rake #{t}; true"
+        rescue => e
+          puts "#{e.class}: #{e.message}"
+          failed_tasks << task
+          # puts e.backtrace
+          puts "continue with next task"
+        end
+      end
+
+      if ! failed_tasks.empty? then
+        puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+        puts "Failed tasks"
+        puts failed_tasks 
+        puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+        raise "failed_tasks = #{failed_tasks}"
+      end 
+        
+    end
+
+
+    # instance tasks (within suite)
+    namespace suite_id do
+
+      # suite["instances"].each do |instance_map|
+      test_suites.suite_instance_ids( suite_id ).each do |instance_id|
+
+        # instance_id = instance_map.keys.first
+        # instance = instance_map[instance_id]
+
+        # **********
+        desc "Test roles for instance '#{instance_id}' in suite '#{suite_id}'"
+        RSpec::Core::RakeTask.new( instance_id ) do |t|
+
+          puts "------------------------------------------------------------------"
+          puts "suite=#{suite_id }, instance=#{instance_id}"
+
+          # see spec/spec_helper.rb
+          ENV['TARGET_SUITE_ID'] = suite_id
+          ENV['TARGET_INSTANCE_ID'] = instance_id
+
+          t.rspec_opts = rspec_opts( suite_id, instance_id )
+          t.fail_on_error = false
+          t.ruby_opts= rspec_ruby_opts
+
+          # test all roles for the instance
+          t.pattern = test_suites.suite_instance_role_ids( suite_id, instance_id ).map{ |r| spec_pattern( r ) }.join(",")
+
+        end
+      end # instance_ids
+    end # ns suite_id
+
+  end # suite_properties.each
+
+  # ------------------------------------------------------------------
+  # DRY methods
+
+  # spec found in 'user_test' or 'gem_test' directory
+  def spec_pattern( role ) 
+    spec_root="spec/aws-must-templates"
+    user_test = File.expand_path( "#{spec_root}/#{role}" )
+    gem_test  = File.expand_path(  File.join( File.dirname( __FILE__), "../..", "#{spec_root}/#{role}"  ))
+      
+    return "#{user_test}/*_spec.rb"  if File.exist?( user_test )
+    return "#{gem_test}/*_spec.rb"   if File.exist?( gem_test )
+
+    message =  <<-EOS
+
+       Could not locate test spec 
+       in test direcotory '#{user_test}'
+    EOS
+    message += <<-EOS if user_test != gem_test
+       nor in gem test directory '#{gem_test}'
+    EOS
+    raise message
+
+  end
+
+  # use -I option to allow Gem and client specs to include spec_helper
+  def rspec_ruby_opts
+    "-I #{File.join( File.dirname(__FILE__), '../../spec/support' )}"
+  end
+
+  # to pass to rpsec
+  def rspec_opts( suite_id, instance_id=nil )
+    # "--format documentation"
+    # "--format progress --format documentation --out generated-docs/suites/#{suite_id}#{ instance_id ? '-'+instance_id : ""}.txt"
+    "--format progress --format documentation --out #{suite_test_report_filepath( suite_id, instance_id )}"
+  end
+
+  def suite_test_report_filepath( suite_id, instance_id=nil )
+    "#{suite_test_report_dirpath()}/#{suite_id}#{ instance_id ? '-' + instance_id : ""}.txt"
+  end
+
+  def suite_test_report_dirpath
+    "generated-docs/suites"
+  end
+
+
+  
+end # ns suite
+

data/lib/test-suites/test_suites.rb

@@ -0,0 +1,115 @@
+require 'yaml'
+
+module AwsMustTemplates
+
+  module TestSuites
+
+    # ------------------------------------------------------------------
+    # Constants
+    SUITE_CONFIGS = 'test-suites.yaml'    # YAML configuration file for suites/instances
+
+    # ------------------------------------------------------------------
+    # class
+
+    class TestSuites
+
+      # ------------------------------------------------------------------
+      # constructor loads YAML configuration, raise expcetion if not found
+      def initialize( )
+        yaml_file = SUITE_CONFIGS
+
+        raise <<-EOS unless File.exist?( yaml_file )
+
+          No such file '#{yaml_file}'
+
+          Could not load test-suites configuration file.
+
+        EOS
+
+        @suites = YAML.load_file( yaml_file ) || []
+      end
+
+      # ------------------------------------------------------------------
+      # return list of suite-ids
+      def suite_ids
+        return @suites.map{ |suite| suite.keys.first }
+      end
+
+      # ------------------------------------------------------------------
+      # return list of suite-roles (i.e. roles not in instances)
+      def suite_roles( suite_id ) 
+        suite = get_suite( suite_id )
+        return (suite ? suite["roles"] : nil)
+      end
+
+      # ------------------------------------------------------------------
+      # return list of suite-roles-ids (i.e. roles not in instances)
+      def suite_role_ids( suite_id ) 
+        roles = suite_roles( suite_id )
+        return nil unless roles
+        return roles.map{ |r| r.is_a?( Hash ) ? r.keys.first : r }
+      end
+
+
+      # ------------------------------------------------------------------
+      # return list of instanceid for a suite
+      def suite_instance_ids( suite_id ) 
+        suite = get_suite( suite_id )
+        return nil unless suite
+        return suite["instances"] ? suite["instances"].map{ |instance_hash| instance_hash.keys.first } : []
+      end
+
+      # ------------------------------------------------------------------
+      # currenty one suite means one to one stack
+      alias_method :stack_ids, :suite_ids
+
+      # return suite configuration for the suite id
+      def get_suite( suite_id )
+        suite = @suites.select { |suite| suite.keys.first == suite_id }.first
+        return suite[suite_id] if suite 
+        return nil
+      end
+
+      # return suite configuration for the suite id
+      def get_suite_stack_id( suite_id )
+        return suite_id
+      end
+
+      # return array of role_ids for 'instance_id' in 'suite_id'
+      def suite_instance_role_ids( suite_id, instance_id )
+        roles = suite_instance_roles( suite_id, instance_id )
+        return roles unless roles
+        return roles.map{ |r| r.is_a?( Hash ) ? r.keys.first : r }
+      end
+
+      # return roles  for 'instance_id' in 'suite_id'
+      def suite_instance_roles( suite_id, instance_id )
+        instance = suite_instance( suite_id, instance_id )
+        return nil unless instance
+        return [] unless instance[instance_id]["roles"]
+        # roles may be a hash or a string
+        return instance[instance_id]["roles"]
+      end
+
+
+      # ------------------------------------------------------------------
+      # return suite instance
+
+      private
+
+      # instance sub document for `instance_id` in `suite_id`
+      def suite_instance( suite_id, instance_id )
+        suite = get_suite( suite_id )
+        return nil unless suite && suite["instances"]
+        return suite["instances"].select { |i| i.keys.first == instance_id }.first
+      end
+
+
+      
+    end # class TestSuites
+
+  end # module TestSuites
+
+
+end # module AwsMustTemplates
+

data/mustache/commonCfnSignal.mustache

@@ -0,0 +1,34 @@
+{{!
+
+
++++start+++
+
+## <a id="commonCfnSignal.mustache"></a>commonCfnSignal.mustache<a class='navigator' href='#top'>[top]</a>
+
+**NOT USED**
+
+<strike>
+Common template to send a signlan
+
+**Attributes**: 
+
+* `WaitHandle` : to signal
+* `Message`    : text message for the signal
+
+</strike>
+
++++close+++
+
+}}
+
+
+{{! +++fold-on+++ }}
+
+{{# WaitHandle}}  
+   ,    "HANDLE='", { "Ref" : "{{WaitHandle}}" }, "'\n"
+   ,   "echo \"Signal success '{{WaitHandle}}' -handle with message '{{Message}}'\" \n"
+   ,   "type cfn-signal && sudo cfn-signal --success true --reason \"{{Message}}\" $HANDLE \n"
+{{/ WaitHandle}}  
+
+
+{{! +++fold-off+++ }}

data/mustache/commonCreationPolicy.mustache

@@ -0,0 +1,33 @@
+{{!
+
+
++++start+++
+
+## <a id="commonCreationPolicy.mustache"></a>commonCreationPolicy.mustache<a class='navigator' href='#top'>[top]</a>
+
+You associate the CreationPolicy attribute with a resource to prevent its status from reaching create complete until AWS CloudFormation receives a specified number of success signals or the timeout period is exceeded. 
+
+
+**Attributes**: 
+
+* `CreationPolicy` : sub document defining creation policy
+  * `Timeout`: The value must be in ISO8601 duration format, in the
+    form: "PT#H#M#S", where each # is the number of hours, minutes,
+    and seconds, respectively.
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+{{# CreationPolicy }}
+   , "CreationPolicy" : {
+             "ResourceSignal": {
+                   "Timeout": "{{Timeout}}"
+             }
+   }
+{{/ CreationPolicy }} 
+
+
+{{! +++fold-off+++ }}

data/mustache/commonDependsOn.mustache

@@ -0,0 +1,25 @@
+{{!
+
+
++++start+++
+
+## <a id="commonDependsOn.mustache"></a>commonDependsOn.mustache<a class='navigator' href='#top'>[top]</a>
+
+
+Common template to output key and value for `DependsOn`
+
+**Attributes**: 
+
+* `DependsOn` : resource name on which dependency defined
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+
+{{# DependsOn }}, "DependsOn" : "{{DependsOn}}"  {{/ DependsOn }} 
+
+
+{{! +++fold-off+++ }}

data/mustache/commonInstanceType.mustache

@@ -0,0 +1,27 @@
+{{!
+
+
++++start+++
+
+## <a id="commonInstanceType.mustache"></a>commonInstanceType.mustache <a class='navigator' href='#top'>[top]</a>
+
+Output value for `InstanceType` -attribute
+
+**Attributes**:
+
+* `InstanceType`: value of instance type
+* `InstanceTypeRef`: reference to instance type
+
+
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+
+{{# InstanceType }}"{{ InstanceType }}"{{/ InstanceType }}
+{{# InstanceTypeRef }}{ "Ref": "{{ InstanceTypeRef }}" }{{/ InstanceTypeRef }}
+
+{{! +++fold-off+++ }}

data/mustache/commonKeyValue.mustache

@@ -0,0 +1,28 @@
+{{!
+
+
++++start+++
+
+## <a id="commonKeyValue.mustache"></a>commonKeyValue.mustache<a class='navigator' href='#top'>[top]</a>
+
+Common template to output key-value pair
+
+**Attributes**: 
+
+* `Key`: 
+*  value given as as commonValue, [commonValue.mustache](#commonValue.mustache)
+
++++close+++
+
+}}
+
+
+{{! +++fold-on+++ }}
+
+{
+   "Key" : "{{Key}}"
+   , "Value" : {{> commonValue }}
+}
+
+
+{{! +++fold-off+++ }}

data/mustache/commonStackRef.mustache

@@ -0,0 +1,23 @@
+{{!
+
+
++++start+++
+
+## <a id="commonStackRef.mustache"></a>commonStackRef.mustache<a class='navigator' href='#top'>[top]</a>
+
+Common template to output nested stack reference
+
+**Attributes**: 
+
+* `Stack` : name of the nested stack resource
+* `Output`: name of output variable in nested stack
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+{ "Fn::GetAtt": [ "{{Stack}}", "Outputs.{{Output}}"] }
+
+{{! +++fold-off+++ }}

data/mustache/commonValue.mustache

@@ -0,0 +1,34 @@
+{{!
+
+
++++start+++
+
+## <a id="commonValue.mustache"></a>commonValue.mustache<a class='navigator' href='#top'>[top]</a>
+
+Common template to output value (Value/Ref/Attr/StackRef)
+
+**Attributes**: 
+
+* `Value` : output value
+* `Ref`: output Ref
+* `StackRef`: 
+   * `Stack`
+   * `Output`
+* `Attr`: 
+   * `Ref`
+   * `Name`
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+
+{{#Value}}"{{Value}}"{{/Value}}
+{{#Ref}}{ "Ref": "{{Ref}}" }{{/Ref}}
+{{#Attr}}{ "Fn::GetAtt" : [ "{{Ref}}", "{{Name}}" ] }{{/Attr}}
+{{#StackRef}}{ "Fn::GetAtt": [ "{{Stack}}", "Outputs.{{Output}}"] }{{/StackRef}}
+
+
+{{! +++fold-off+++ }}

data/mustache/initializeCFinit.mustache

@@ -0,0 +1,27 @@
+{{!
+
+
++++start+++
+
+
+## <a id="initializeCFinit.mustache"></a>initializeCFinit.mustache<a class='navigator' href='#top'> [top]</a>
+
+Launch cfn-init to install the files and packages from the metadata 
+
+**Attributes**: 
+
+* `Resource`: The logical resource ID of the resource that contains the metadata.
+
++++close+++
+
+}}
+
+{{! +++fold-on+++ }}
+
+  ,  "# Install the files and packages from the metadata\n"
+  ,  "STACK='", { "Ref": "AWS::StackName"  }, "'\n"
+  ,  "REGION='", { "Ref": "AWS::Region" }, "'\n"
+  ,  "cfn-init -v --stack $STACK --region $REGION --resource {{Resource}}   \n "
+
+
+{{! +++fold-off+++ }}

data/mustache/initializeCFtools.mustache

@@ -0,0 +1,27 @@
+{{!
+
+
++++start+++
+
+
+## <a id="initializeCFtools.mustache"></a>initializeCFtools.mustache<a class='navigator' href='#top'> [top]</a>
+
+An [adapted](https://gist.github.com/kixorz/10194688) Ubuntu
+CloudFormation Tools installation snippet.
+
+**Attributes**: 
+
+* none
+
++++close+++
+
+}}
+
+
+{{! +++fold-on+++ }}
+  ,    "sudo apt-get -y install python-setuptools  \n"
+  ,    "[ -d aws-cfn-bootstrap-latest  ] || mkdir aws-cfn-bootstrap-latest  \n"
+  ,    "curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1  \n"
+  ,    "sudo easy_install aws-cfn-bootstrap-latest  \n"
+
+{{! +++fold-off+++ }}