rightscale_selfservice 0.0.3 → 0.0.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWQ2YmE1OTk2MjA0ODAyNTRiMjU3ZWYyMDczMDQ0OWJkNzY4YzNhYQ==
+    MmMxMmMxYzA0NDNmNzk2NWVjODZmNjRkYjBlZTc4ODJmZDZmMmM3YQ==
   data.tar.gz: !binary |-
-    MWZmYjA5OTgwZmUxNmM3ZDMzMDc2ZmZkZWQ1MmVmOWI3MjM1OWQyZQ==
+    MGUyNmQ0ZGZiZTAzNjllZmNjM2NiMzBlNjU2ZGE1ZjJlMjViMGRjZQ==
 SHA512:
   metadata.gz: !binary |-
-    MjY4N2QyNjM3YWRkY2NhNjdkYzkxN2E2ZWEwODhmYTliYTBjMTk1ZmVhOWRl
-    YmVkZjc1N2QxZjJiZGY3YjRmMDcwNDU3MmFmODRhYWIyMzkzMTkzYWE5YjYy
-    MjI2NTM3NzQ0Yzc5ZmE0YTZkN2ZjNThkZjY5ODZmYzAwMzZjY2Q=
+    ODg5NTRhOTE2MmNkYzMzZjExMWY0ZDg0YzZmYTg1ZTE4ODE5ZTVjYzQ3MDYx
+    YTc5OWJhNDljMDA5ZjYxN2EwMmMxNWY2ZjQyYmZhZTgyMThjZjA2ZGY2YTU3
+    M2M1ZjliODVjZDU3YTBkMjdlZTM5NTU1NmM5NzFlNGNmMTNlYzQ=
   data.tar.gz: !binary |-
-    MTY5MDEyMmU1ZmM4NTlmNGU0OGM1NWIzZjQ4NDgzNGRlMjMwMWI5YzRkMjg1
-    ZGYzYTlmMmMxMDQ3YWI4NWFmNzYzNTcxNTg3MDY5ZjRmYzhkYTUxMjJmODA5
-    OGE4N2FjMDYxZjA2NTFiZThiNDc1NTg0ZGEwY2Y4YmMzMGZjZTE=
+    OTcxMjc0N2FlZDJjNzNiMWYyYzllYzliY2NlNmJkMGM4MGMyZDRkOWQyM2Vi
+    ODIzMGY0NWFlOTZhYjYzYzI3N2I0OWJkNzJjMzAzZWU3NmZmMjZjNTFmNTQ0
+    NzIwMjVjMGQyOWI4MjI3ZjMzZTY5ODM1NzQ3ZDQxMjZhNTM1ODM=

data/README.md

@@ -1,3 +1,36 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [rightscale_selfservice](#rightscale_selfservice)
+  - [Quick Start](#quick-start)
+  - [Authentication](#authentication)
+  - [CLI](#cli)
+    - [Template Includes](#template-includes)
+  - [Testing](#testing)
+    - [Suite](#suite)
+    - [Template](#template)
+    - [Case](#case)
+      - [Compile](#compile)
+      - [Execute](#execute)
+      - [Operations](#operations)
+  - [API Client](#api-client)
+    - [Supported Services, Resources and Actions](#supported-services-resources-and-actions)
+    - [How it works](#how-it-works)
+      - [Href Tokens](#href-tokens)
+      - [Automatic Multipart](#automatic-multipart)
+    - [Some examples...](#some-examples)
+      - [Authentication](#authentication-1)
+      - [Service Versions](#service-versions)
+      - [Executing Actions](#executing-actions)
+        - [List operations](#list-operations)
+        - [Show a template](#show-a-template)
+        - [Get RestClient::Request Instead of Executing Action](#get-restclientrequest-instead-of-executing-action)
+        - [Format Errors](#format-errors)
+        - [Multipart Detection](#multipart-detection)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 rightscale_selfservice
 ======================
 
@@ -28,13 +61,13 @@ properties as [right_api_client](https://github.com/rightscale/right_api_client)
 
 The only additional required property is "selfservice_url".
 
-The CLI can use a \*.yml file containing those properties or have them passed in
+The CLI can use a .yml file containing those properties or have them passed in
 on the commandline.  The API Client accepts an input hash with these same
 properties.
 
-An example \*.yml file can be found [here](https://github.com/rightscale/right_api_client/blob/v1.5.24/config/login.yml.example)
+An example .yml file can be found [here](https://github.com/rightscale/right_api_client/blob/v1.5.24/config/login.yml.example)
 
-A working \*.yml file might look like;
+A working .yml file might look like;
 ```
 :account_id: 12345
 :email: user@domain.com
@@ -105,6 +138,121 @@ If that works, maybe start up a Cloud App from it
 rightscale_selfservice template execute ~/Code/cat/somecat.cat.rb --auth-file=~/.right_api_client/login.yml
 ```
 
+## Testing
+
+Included as a framework for performing functional tests of your Cloud
+Application Templates (CATs). Tests are performed as part of a [Suite](#suite)
+which contains one or more [Template](#template), which have one or more test
+[Case](#case).
+
+### Suite
+A suite is currently defined simply by the [glob](http://ruby-doc.org/core-1.9.3/Dir.html#method-c-glob)
+you pass either to the [CLI](#cli) or the test suite class.
+
+I might consider creating a config file or other options, but for now the suite
+is basically just a container.
+
+The suite will interrogate the files found by the input glob, and create
+individual [Template](#template) classes.
+
+### Template
+A template is a 1-to-1 match with a CAT file found by the [Suite](#suite).
+
+The template class will interrogate the contents of the CAT and create individual
+[Case](#case) classes.
+
+### Case
+There are three types of cases, which are defined by comments in the CAT.
+
+#### Compile
+Merely checks if the CAT will compile successfully. When a template is tagged
+to include a compile test case every other test will be ignored.
+
+To create a test template which will only be tested for successful compile add
+the following to the top of the template
+```
+#test:compile_only=true
+```
+
+#### Execute
+This checks if the CAT will execute and reach a desired state.
+
+If no tags are added to a template, an execute test case will be automatically
+run, with the assumption that it's expected execution_state is "running".
+
+To create a test template with an execute test case which will succeed if the
+CAT launches successfully.
+
+```
+#test:execution_state=running
+```
+
+For the negative test case, where a CAT which launches and fails is actually a
+successful test.
+
+```
+#test:execution_state=failed
+```
+
+There are also scenarios where a successful test should end in a specified state
+but because of external issues (system bugs, reusable definitions, etc) the
+opposite state is reached. In this case you can specify both the desired state
+as the "execution_state" as well as an "execution_alternate_state".
+
+If the CAT launches and reaches the alternate state the test case will be marked
+as "FAILED (EXPECTED)", and if it reaches the desired state it will be marked
+"FIXED"
+
+```
+#test:execution_state=running
+#test:execution_alternate_state=failed
+```
+
+#### Operations
+Once a template has been successfully launched, any operation test cases will
+be run.
+
+An example of a CAT with two operations which should be run as test cases;
+```
+#test:execution_state=running
+#test:execution_alternate_state=failed
+
+name "Foo"
+rs_ca_ver 20131202
+short_description "Foo"
+
+#test_operation:execution_state=completed
+#test_operation:execution_alternate_state=failed
+#test_operation_param:key=val
+#test_operation_param:key1=val1
+#test_operation_param:key2=val2
+operation "one" do
+  description "one"
+  definition "foo"
+end
+
+#test_operation:execution_state=completed
+#test_operation:execution_alternate_state=failed
+#test_operation_param:key=val
+#test_operation_param:key1=val1
+#test_operation_param:key2=val2
+operation "two" do
+  description "two"
+  definition "foo"
+end
+
+define foo() do
+
+end
+```
+
+This will run the operations named "one" and "two" with the specified
+test_operation_param(s) as inputs to the operation. It will evaluate success
+or failure based on the "execution_state" specified.
+
+This has the same functionality as [Execute](#execute) in that you can specify
+a desired and alternative state.
+
 ## API Client
 
 The [CLI](#cli) bits consume the API client which is deployed with this gem. The

data/lib/rightscale_selfservice/api/client.rb

@@ -238,6 +238,23 @@ module RightScaleSelfService
         url.gsub!(@selfservice_url,"")
       end
 
+      # Accepts a full URL with protocol and hostname, or a relative href and
+      # returns the ID which is the last token in the path.
+      #
+      # I.E. https://hostname/api/foo/bar/baz/12345 == 12345
+      # and /api/foo/bar/baz/12345 == 12345
+      #
+      # @param url_href_or_id [String] The full url with protocol and hostname,
+      #   a relative resource href, or the id.
+      #
+      # @return [String] ID of the resource referred to by the url or href, or
+      #   nil if the id can not be determined.
+      def self.get_resource_id_from_href(url_href_or_id)
+        return url_href_or_id if url_href_or_id =~ /^[a-zA-Z0-9]*$/
+        matchdata = url_href_or_id.match(/[a-zA-Z0-9\/]*\/(?<id>[a-zA-Z0-9]*)$/)
+        matchdata['id'] if matchdata
+      end
+
       # Accepts various possible responses and formats it into useful error text
       #
       # @param [RestClient::ExceptionWithResponse] error The response or error

data/lib/rightscale_selfservice/api/resource.rb

@@ -64,8 +64,18 @@ module RightScaleSelfService
               params[:payload] = args[0]
             else
               params[:payload] = URI.encode_www_form(args[0])
+              args[0].each do |k,v|
+                if v.is_a?(Array)
+                  params[:payload].gsub!("#{k}=","#{k}[]=")
+                end
+              end
             end
           end
+
+          if method == :get && params.has_key?(:payload)
+            params[:url] += "?#{params[:payload]}"
+            params.delete(:payload)
+          end
         end
 
         request = @service.client.get_authorized_rest_client_request(params)

data/lib/rightscale_selfservice/cli/execution.rb

@@ -43,6 +43,28 @@ module RightScaleSelfService
           logger.error(shell.set_color message, :red)
         end
       end
+
+      desc "show <id_or_href>", "Gets details about an execution (CloudApp) specified by <id_or_href>"
+      option :view, :type => :string, :default => 'default', :desc => "Which view to use, one of [default,expanded,source] default is 'default'."
+      option :property, :type => :array, :desc => "When supplied, only the specified properties will be displayed. By default the entire response is supplied. Ignored if view is 'source'"
+      def show(id_or_href)
+        client = get_api_client()
+        id = RightScaleSelfService::Api::Client.get_resource_id_from_href(id_or_href)
+
+        begin
+          response = client.manager.execution.show(:id => id, :view => @options["view"])
+          execution = JSON.parse(response.body)
+          if @options["property"] && @options["view"] != "source"
+            execution.delete_if{|k,v| !(@options["property"].include?(k))}
+          end
+          puts JSON.pretty_generate(execution)
+        rescue RestClient::ExceptionWithResponse => e
+          shell = Thor::Shell::Color.new
+          message = "Failed to show execution id #{id}\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+          logger.error(shell.set_color message, :red)
+        end
+      end
+
     end
   end
 end

data/lib/rightscale_selfservice/cli/main.rb

@@ -33,6 +33,28 @@ module RightScaleSelfService
 
       desc "operation", "Self Service Operation Commands"
       subcommand "operation", Operation
+
+      desc "test <glob>", "Run a CAT Test suite consisting of files found in <glob>"
+      def test(glob)
+        client = get_api_client()
+        suite = RightScaleSelfService::Test::Suite.new(client, glob)
+        report = RightScaleSelfService::Test::ShellReport.new(suite)
+        begin
+          if suite.pump
+            report.progress
+            sleep(10)
+          else
+            break
+          end
+        end while true
+        report.progress
+        puts "\n\n"
+        error_text = report.errors
+        if error_text != ""
+          puts "\n\n"
+        end
+        report.failures
+      end
     end
   end
 end

data/lib/rightscale_selfservice/cli/operation.rb

@@ -44,6 +44,32 @@ module RightScaleSelfService
           logger.error(shell.set_color message, :red)
         end
       end
+
+      desc "list", "Lists all operations, optionally filtered by --filter and/or --property"
+      option :filter, :type => :array, :desc => "Filters to apply see (https://s3.amazonaws.com/rs_api_docs/selfservice/manager/index.html#/1.0/controller/V1::Controller::Operation/index)"
+      option :property, :type => :array, :desc => "When supplied, only the specified properties will be displayed. By default the entire response is supplied."
+      def list()
+        params = {}
+        if @options["filter"]
+          params[:filter] = @options["filter"]
+        end
+
+        begin
+          client = get_api_client()
+          response = client.manager.operation.index(params)
+          operations = JSON.parse(response.body)
+          if @options["property"]
+            operations.each do |op|
+              op.delete_if{|k,v| !(@options["property"].include?(k))}
+            end
+          end
+          puts JSON.pretty_generate(operations)
+        rescue RestClient::ExceptionWithResponse => e
+          shell = Thor::Shell::Color.new
+          message = "Failed to list operations\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+          logger.error(shell.set_color message, :red)
+        end
+      end
     end
   end
 end

data/lib/rightscale_selfservice/test/case.rb

@@ -0,0 +1,155 @@
+# Copyright (c) 2014 Ryan Geyer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScaleSelfService
+  module Test
+    class Case
+      attr_accessor :type
+      attr_accessor :options
+      attr_accessor :result
+      attr_accessor :errors
+      attr_accessor :finished
+      attr_accessor :failures
+      # initialized -> running -> completed | ?? -> finished
+      attr_accessor :state
+      attr_accessor :api_responses
+
+      def initialize(type, options = {})
+        self.type = type
+        self.options = options
+        self.result = ""
+        self.errors = []
+        self.failures = []
+        self.state = 'initialized'
+        self.api_responses = {}
+      end
+
+      # Performs the test for this case. Returns a boolean indicating if the
+      # test is "done" or not.
+      #
+      # @param suite [RightScaleSelfService::Test::Suite] The suite this test
+      #   case belongs to
+      # @param template [RightScaleSelfService::Test::Template] The template
+      #   this case belongs to
+      #
+      # @return [bool] True if this case needs to be pumped again, false if
+      #   the case has been executed and does not need any further time slices
+      def pump(suite, template)
+        if result != ""
+          false
+        else
+          case self.type
+          when :compile_only
+            begin
+              suite.api_client.designer.template.compile(:source => template.template_string)
+              self.result = 'SUCCESS'
+            rescue RestClient::ExceptionWithResponse => e
+              if e.http_code == 422
+                self.result = 'FAILED'
+                self.failures << "Failed to compile template\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+              else
+                self.result = 'ERROR'
+                self.errors << RightScaleSelfService::Api::Client.format_error(e)
+              end
+            end
+            false
+          when :execution
+            if self.options[:state] == template.state
+              self.result = 'SUCCESS'
+              self.result = 'FIXED' if self.options.has_key?(:alternate_state)
+            else
+              if self.options.has_key?(:alternate_state) && self.options[:alternate_state] == template.state
+                self.result = 'FAILED (EXPECTED)'
+              else
+                self.result = 'FAILED'
+                self.failures << "Expected execution end state to be (#{self.options[:state]}) but got execution end state (#{template.state})"
+              end
+            end
+            false
+          when :operation
+            if template.state == 'running'
+              case self.state
+              when 'initialized'
+                execution_id = template.execution_id
+                create_params = {
+                  :execution_id => execution_id,
+                  :name => self.options[:operation_name]
+                }
+
+                if self.options.has_key?(:params)
+                  create_params[:options] = self.options[:params]
+                end
+
+                begin
+                  self.api_responses[:operation_create] = suite.api_client.manager.operation.create(create_params)
+                  self.state = 'running'
+                rescue RestClient::ExceptionWithResponse => e
+                  self.errors << "Failed to create operation #{self.options[:operation_name]} for execution\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+                  self.result = 'ERROR'
+                  return false
+                end
+                true
+              when 'running'
+                begin
+                  operation_id = RightScaleSelfService::Api::Client.get_resource_id_from_href(self.api_responses[:operation_create].headers[:location])
+                  if operation_id
+                    self.api_responses[:operation_show] = suite.api_client.manager.operation.show(:id => operation_id)
+                    json_str = self.api_responses[:operation_show].body
+                    show = JSON.parse(json_str)
+                    self.state = show['status']['summary']
+                  end
+                rescue RestClient::ExceptionWithResponse => e
+                  # TODO: Do I want to catch errors here, or let it fall through and
+                  # let the next pump retry?
+                  self.errors << "Failed to check operation status\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+                end
+              when 'completed','failed'
+                if self.options[:state] == self.state
+                  self.result = 'SUCCESS'
+                  self.result = 'FIXED' if self.options.has_key?(:alternate_state)
+                else
+                  if self.options.has_key?(:alternate_state) && self.options[:alternate_state] == self.state
+                    self.result = 'FAILED (EXPECTED)'
+                  else
+                    self.result = 'FAILED'
+                    self.failures << "Expected operation end state to be (#{self.options[:state]}) but got execution end state (#{self.state})"
+                  end
+                end
+                false
+              else
+              end
+            elsif template.state == 'failed'
+              self.result = 'FAILED'
+              self.failures << "Execution failed to start, could not start a new operation."
+              false
+            else
+              self.result = 'ERROR'
+              self.errors << "Unexpected execution state #{template.state} while processing operation test case."
+              false
+            end
+          else
+            self.result = 'ERROR'
+            self.errors << "Unknown test case type (:#{self.type})"
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rightscale_selfservice/test/report.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2014 Ryan Geyer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScaleSelfService
+  module Test
+    class Report
+      attr_accessor :suite
+      attr_accessor :options
+
+      def initialize(suite, options={})
+        self.suite = suite
+        self.options = options
+      end
+
+      def progress
+        raise NotImplementedError.new()
+      end
+
+      def errors
+        raise NotImplementedError.new()
+      end
+
+      def failures
+        raise NotImplementedError.new()
+      end
+    end
+  end
+end

data/lib/rightscale_selfservice/test/shell_report.rb

@@ -0,0 +1,118 @@
+# Copyright (c) 2014 Ryan Geyer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# Require report.rb first, to satisfy Travis CI
+require File.expand_path(File.join(File.dirname(__FILE__), 'report'))
+
+module RightScaleSelfService
+  module Test
+    class ShellReport < Report
+
+      def initialize(suite, options={})
+        super(suite,options)
+        @reported_templates = []
+        @keyword_substitutions = {
+          "SUCCESS" => "\e[42m\e[30mSUCCESS\e[0m",
+          "FAILED" => "\e[41m\e[30mFAILED\e[0m",
+          "FAILED (EXPECTED)" => "\e[43m\e[30mFAILED (EXPECTED)\e[0m",
+          "ERROR" => "\e[41m\e[30mERROR\e[0m",
+          "FIXED" => "\e[44m\e[30mFIXED\e[0m"
+        }
+      end
+
+      def progress
+        self.suite.templates.each do |template|
+          if template.state == "finished" && !@reported_templates.include?(template)
+            puts "#{template.name}: #{template.state}"
+            template.cases.each do |testcase|
+              puts "  #{get_case_type_and_name(testcase)}: #{@keyword_substitutions[testcase.result]}"
+            end
+            @reported_templates << template
+          end
+        end
+      end
+
+      def errors
+        finished_templates = self.suite.templates.select{|t| t.state == "finished"}
+        if finished_templates.size == self.suite.templates.size
+          to_puts = ""
+          self.suite.templates.each do |template|
+            cases_with_errors = template.cases.select{|c| c.errors.size > 0}
+            if template.errors.size > 0 ||  cases_with_errors.size > 0
+              to_puts += "#{template.name}:\n"
+            end
+            template.errors.each do |error|
+              to_puts += "  #{error}\n"
+            end
+            cases_with_errors.each do |case_with_errors|
+              to_puts += "  #{get_case_type_and_name(case_with_errors)}:\n"
+              case_with_errors.errors.each do |error|
+                to_puts += "    #{error}\n"
+              end
+            end
+          end
+          if to_puts.size > 0
+            puts "\e[41m\e[30mERRORS:\e[0m"
+            puts "\e[31m#{to_puts}\e[0m"
+          end
+        end
+      end
+
+      def failures
+        finished_templates = self.suite.templates.select{|t| t.state == "finished"}
+        to_puts = ""
+        if finished_templates.size == self.suite.templates.size
+          to_puts = ""
+          self.suite.templates.each do |template|
+            cases_with_failures = template.cases.select{|c| c.failures.size > 0}
+            if cases_with_failures.size > 0
+              to_puts += "#{template.name}:\n"
+            end
+            cases_with_failures.each do |case_with_fail|
+              to_puts += "  #{get_case_type_and_name(case_with_fail)}:\n"
+              # TODO: Add details for an operation case type to differentiate
+              # them. This is probably an argument for having some of the
+              # progress, error, and failure reporting live in the case
+              # class, but we don't want the case class to have to know details
+              # about how to output. Hrrmnn.
+              case_with_fail.failures.each do |failure|
+                to_puts += "    #{failure}\n"
+              end
+            end
+          end
+          if to_puts.size > 0
+            to_puts = "Failures:\n#{to_puts}"
+            puts to_puts
+          end
+        end
+      end
+
+      private
+
+      def get_case_type_and_name(test_case)
+        retval = test_case.type.to_s
+        if test_case.type == :operation
+          retval += " (#{test_case.options[:operation_name]})"
+        end
+        retval
+      end
+
+    end
+  end
+end

data/lib/rightscale_selfservice/test/suite.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2014 Ryan Geyer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScaleSelfService
+  module Test
+    class Suite
+      attr_accessor :api_client
+      attr_accessor :templates
+
+      def initialize(api_client, glob)
+        self.templates = []
+        self.api_client = api_client
+        template_files = Dir.glob(glob)
+        template_files.each do |template|
+          templates << RightScaleSelfService::Test::Template.new(template)
+        end
+      end
+
+      def pump
+        finished_templates = templates.select do |t|
+          t.pump(self)
+          t.state == 'finished'
+        end
+        templates.length != finished_templates.length
+      end
+    end
+  end
+end

data/lib/rightscale_selfservice/test/template.rb

@@ -0,0 +1,161 @@
+# Copyright (c) 2014 Ryan Geyer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScaleSelfService
+  module Test
+    class Template
+      attr_accessor :cases
+      attr_accessor :template_string
+      # intialized -> launching -> running | failed -> terminating -> finished
+      attr_accessor :state
+
+      attr_accessor :errors
+      attr_accessor :api_responses
+
+      attr_reader :name
+
+      def initialize(filepath)
+        @name = File.basename(filepath)
+        self.api_responses = {}
+        self.state = "initialized"
+        self.errors = []
+        self.cases = []
+        self.template_string = template_str = RightScaleSelfService::Utilities::Template.preprocess(filepath)
+        test_config = {}
+        if template_str.include?('#test:compile_only=true')
+          self.cases = [RightScaleSelfService::Test::Case.new(:compile_only)]
+          self.state = "running"
+        else
+          execution_state_matches = template_str.match(/^#test:execution_state=(?<state>[0-9a-zA-Z ]*)/)
+          execution_state = 'running'
+          if execution_state_matches && execution_state_matches.names.include?('state')
+            execution_state = execution_state_matches['state']
+          end
+          options = {:state => execution_state}
+          alt_state_matches = template_str.match(/^#test:execution_alternate_state=(?<state>[0-9a-zA-Z ]*)/)
+          if alt_state_matches && alt_state_matches.size > 0
+            options[:alternate_state] = alt_state_matches['state']
+          end
+          self.cases << RightScaleSelfService::Test::Case.new(:execution, options)
+
+          template_str.scan(/(#test_operation:.*?)\noperation ["'](.*?)["'] do/m).each do |operation|
+            tags = operation[0]
+            operation_name = operation[1]
+
+            options = {:operation_name => operation_name}
+
+            execution_state_matches = tags.match(/^#test_operation:execution_state=(?<state>[0-9a-zA-Z ]*)/)
+            if execution_state_matches && execution_state_matches.names.include?('state')
+              options[:state] = execution_state_matches['state']
+            end
+
+            alt_state_matches = tags.match(/^#test_operation:execution_alternate_state=(?<state>[0-9a-zA-Z ]*)/)
+            if alt_state_matches && alt_state_matches.names.include?('state')
+              options[:alternate_state] = alt_state_matches['state']
+            end
+
+            tags.scan(/#test_operation_param:(?<key>.*?)=(?<val>.*?)$/).each do |param_pair|
+              options[:params] = {} unless options.has_key?(:params)
+              options[:params][param_pair[0]] = param_pair[1]
+            end
+
+            self.cases << RightScaleSelfService::Test::Case.new(:operation, options)
+          end
+        end
+      end
+
+      def pump(suite)
+        case self.state
+        when 'initialized'
+          begin
+            self.api_responses[:execution_create] = suite.api_client.manager.execution.create(:source => self.template_string)
+            self.state = 'launching'
+          rescue RestClient::ExceptionWithResponse => e
+            self.errors << "Failed to create execution from template\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+            self.state = 'failed'
+          end
+        when 'launching'
+          get_execution_status_and_set_as_state(suite)
+        when 'running','failed'
+          # TODO: Any error handling here?
+          unfinished_cases = cases.select {|c| c.pump(suite, self)}
+          if unfinished_cases.size == 0
+            if self.api_responses.has_key?(:execution_create)
+              exec_id = execution_id()
+              self.api_responses[:terminate_operation_create] =
+                suite.api_client.manager.operation.create(
+                  :name => 'terminate', :execution_id => exec_id
+                )
+              self.state = 'terminating'
+            else
+              self.state = 'terminated'
+            end
+          end
+        when 'terminating'
+          get_execution_status_and_set_as_state(suite)
+        when 'terminated'
+          begin
+            exec_id = execution_id()
+            if exec_id
+              suite.api_client.manager.execution.delete(:id => exec_id)
+            end
+          rescue RestClient::ExceptionWithResponse => e
+            self.errors << "Failed to delete execution #{self.api_response[:execution_create][:headers][:location]}\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+          end
+          self.state = 'finished'
+        when 'finished'
+          # Do nothing
+        else
+          self.errors << "unknown template state #{self.state}"
+          self.state = 'finished'
+        end
+      end
+
+      def execution_id
+        unless self.api_responses.has_key?(:execution_create)
+          nil
+        else
+          execution_id = RightScaleSelfService::Api::Client
+            .get_resource_id_from_href(
+              self.api_responses[:execution_create].headers[:location]
+            )
+        end
+      end
+
+      private
+
+      def get_execution_status_and_set_as_state(suite)
+        begin
+          exec_id = execution_id()
+          if exec_id
+            self.api_responses[:execution_show] = suite.api_client.manager.execution.show(:id => exec_id)
+            json_str = self.api_responses[:execution_show].body
+            show = JSON.parse(json_str)
+            self.state = show['status']
+          end
+        rescue RestClient::ExceptionWithResponse => e
+          # TODO: Do I want to catch errors here, or let it fall through and
+          # let the next pump retry?
+          self.errors << "Failed to check execution status\n\n#{RightScaleSelfService::Api::Client.format_error(e)}"
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rightscale_selfservice
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Ryan J. Geyer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-15 00:00:00.000000000 Z
+date: 2015-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -73,6 +73,11 @@ files:
 - lib/rightscale_selfservice/cli/main.rb
 - lib/rightscale_selfservice/cli/operation.rb
 - lib/rightscale_selfservice/cli/template.rb
+- lib/rightscale_selfservice/test/case.rb
+- lib/rightscale_selfservice/test/report.rb
+- lib/rightscale_selfservice/test/shell_report.rb
+- lib/rightscale_selfservice/test/suite.rb
+- lib/rightscale_selfservice/test/template.rb
 - lib/rightscale_selfservice/utilities/template.rb
 homepage: https://github.com/rgeyer/rightscale_selfservice
 licenses: