resat 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.rdoc +321 -0
- data/Rakefile +33 -0
- data/bin/resat +223 -0
- data/examples/rightscale/README.rdoc +39 -0
- data/examples/rightscale/additional/run_server.yml +75 -0
- data/examples/rightscale/config/resat.yaml +34 -0
- data/examples/rightscale/scenarios/create_server.yml +48 -0
- data/examples/rightscale/scenarios/delete_server.yml +13 -0
- data/examples/rightscale/scenarios/list_servers.yml +9 -0
- data/examples/twitter/README.rdoc +50 -0
- data/examples/twitter/additional/follow.yml +15 -0
- data/examples/twitter/additional/send_message.yml +19 -0
- data/examples/twitter/config/resat.yaml +40 -0
- data/examples/twitter/output.yml +5 -0
- data/examples/twitter/scenarios/timelines.yml +31 -0
- data/examples/twitter/scenarios/tweet.yml +14 -0
- data/lib/api_request.rb +145 -0
- data/lib/config.rb +94 -0
- data/lib/engine.rb +98 -0
- data/lib/file_set.rb +33 -0
- data/lib/filter.rb +113 -0
- data/lib/guard.rb +36 -0
- data/lib/handler.rb +50 -0
- data/lib/kwalify_helper.rb +31 -0
- data/lib/log.rb +114 -0
- data/lib/net_patch.rb +15 -0
- data/lib/rdoc_patch.rb +37 -0
- data/lib/resat.rb +5 -0
- data/lib/scenario_runner.rb +203 -0
- data/lib/variables.rb +116 -0
- data/schemas/config.yaml +48 -0
- data/schemas/scenarios.yaml +169 -0
- data/schemas/variables.yaml +18 -0
- metadata +98 -0
data/lib/config.rb
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
# Configuration information read from given configuration file
|
2
|
+
# ('config/resat.yaml' by default).
|
3
|
+
#
|
4
|
+
# Configuration example:
|
5
|
+
#
|
6
|
+
# # Hostname used for API calls
|
7
|
+
# host: my.rightscale.com
|
8
|
+
#
|
9
|
+
# # Common base URL to all API calls
|
10
|
+
# base_url: '/api/acct/71/'
|
11
|
+
#
|
12
|
+
# # Use HTTPS?
|
13
|
+
# use_ssl: yes
|
14
|
+
#
|
15
|
+
# # Basic auth username if any
|
16
|
+
# username: raphael@rightscale.com
|
17
|
+
#
|
18
|
+
# # Basic auth password if any
|
19
|
+
# password: Secret
|
20
|
+
#
|
21
|
+
# # Common request headers for all API calls
|
22
|
+
# headers:
|
23
|
+
# - name: X-API-VERSION
|
24
|
+
# value: '1.0'
|
25
|
+
#
|
26
|
+
# # Common parameters for all API calls
|
27
|
+
# params:
|
28
|
+
#
|
29
|
+
# See resat.rb for usage information.
|
30
|
+
#
|
31
|
+
|
32
|
+
module Resat
|
33
|
+
|
34
|
+
class Config
|
35
|
+
|
36
|
+
DEFAULT_FILE = 'config/resat.yaml'
|
37
|
+
|
38
|
+
DEFAULT_SCHEMA_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'schemas'))
|
39
|
+
|
40
|
+
DEFAULTS = {
|
41
|
+
'base_url' => '',
|
42
|
+
'use_ssl' => false,
|
43
|
+
'variables' => {}
|
44
|
+
}
|
45
|
+
|
46
|
+
def Config.init(filename, schemasdir)
|
47
|
+
(Config.methods - (Object.methods + [ 'init', 'valid?', 'method_missing' ])).each { |m| class << Config;self;end.send :remove_method, m.to_sym }
|
48
|
+
schemafile = File.join(schemasdir || DEFAULT_SCHEMA_DIR, 'config.yaml')
|
49
|
+
unless File.exists?(schemafile)
|
50
|
+
Log.error("Missing configuration file schema '#{schemafile}'")
|
51
|
+
@valid = false
|
52
|
+
return
|
53
|
+
end
|
54
|
+
schema = Kwalify::Yaml.load_file(schemafile)
|
55
|
+
validator = Kwalify::Validator.new(schema)
|
56
|
+
parser = Kwalify::Yaml::Parser.new(validator)
|
57
|
+
@valid = true
|
58
|
+
@config = { 'use_ssl' => false, 'username' => nil, 'password' => nil, 'port' => nil }
|
59
|
+
cfg_file = filename || DEFAULT_FILE
|
60
|
+
config = parser.parse_file(cfg_file)
|
61
|
+
if parser.errors.empty?
|
62
|
+
if config.nil?
|
63
|
+
Log.error("Configuration file '#{cfg_file}' is empty.")
|
64
|
+
@valid = false
|
65
|
+
else
|
66
|
+
@config.merge!(config)
|
67
|
+
# Dynamically define the methods to forward to the config hash
|
68
|
+
@config.each_key do |meth|
|
69
|
+
(class << self; self; end).class_eval do
|
70
|
+
define_method meth do |*args|
|
71
|
+
@config[meth] || DEFAULTS[meth]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
else
|
77
|
+
errors = parser.errors.inject("") do |msg, e|
|
78
|
+
msg << "#{e.linenum}:#{e.column} [#{e.path}] #{e.message}\n\n"
|
79
|
+
end
|
80
|
+
Log.error("Configuration file '#{cfg_file}' is invalid:\n#{errors}")
|
81
|
+
@valid = false
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def Config.valid?
|
86
|
+
@valid
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.method_missing(*args)
|
90
|
+
nil
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
end
|
data/lib/engine.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
# Resat test engine, reads test files and run them.
|
2
|
+
# See resat.rb for usage information.
|
3
|
+
#
|
4
|
+
|
5
|
+
ENG_DIR = File.dirname(__FILE__)
|
6
|
+
require File.join(ENG_DIR, 'file_set')
|
7
|
+
require File.join(ENG_DIR, 'log')
|
8
|
+
require File.join(ENG_DIR, 'scenario_runner')
|
9
|
+
|
10
|
+
module Resat
|
11
|
+
|
12
|
+
class Engine
|
13
|
+
|
14
|
+
attr_accessor :run_count # Total number of run scenarios
|
15
|
+
attr_accessor :requests_count # Total number of HTTP requests
|
16
|
+
attr_accessor :ignored_count # Total number of ignored scenarios
|
17
|
+
attr_accessor :skipped_count # Total number of skipped YAML files
|
18
|
+
attr_accessor :failures # Hash of error messages (string arrays)
|
19
|
+
# indexed by scenario filename
|
20
|
+
|
21
|
+
def initialize(options)
|
22
|
+
@options = options
|
23
|
+
@failures = Hash.new
|
24
|
+
@run_count = 0
|
25
|
+
@ignored_count = 0
|
26
|
+
@requests_count = 0
|
27
|
+
@skipped_count = 0
|
28
|
+
end
|
29
|
+
|
30
|
+
# Was test run successful?
|
31
|
+
def succeeded?
|
32
|
+
@failures.size == 0
|
33
|
+
end
|
34
|
+
|
35
|
+
# Run all scenarios and set attributes accordingly
|
36
|
+
def run(target=nil)
|
37
|
+
target ||= @options.target
|
38
|
+
begin
|
39
|
+
if File.directory?(target)
|
40
|
+
files = FileSet.new(target, %w{.yml .yaml})
|
41
|
+
elsif File.file?(target)
|
42
|
+
files = [target]
|
43
|
+
else
|
44
|
+
@failures[target] << "Invalid taget #{target}: Not a directory, nor a file"
|
45
|
+
return
|
46
|
+
end
|
47
|
+
@failures[target] ||= []
|
48
|
+
schemasdir = @options.schemasdir || Config::DEFAULT_SCHEMA_DIR
|
49
|
+
files.each do |file|
|
50
|
+
runner = ScenarioRunner.new(file, schemasdir, @options.config,
|
51
|
+
@options.variables, @options.failonerror, @options.dry_run)
|
52
|
+
@ignored_count += 1 if runner.ignored?
|
53
|
+
@skipped_count += 1 unless runner.valid?
|
54
|
+
if runner.valid? && !runner.ignored?
|
55
|
+
runner.run
|
56
|
+
@run_count += 1
|
57
|
+
@requests_count += runner.requests_count
|
58
|
+
@failures[file] = runner.failures unless runner.failures.empty?
|
59
|
+
else
|
60
|
+
unless runner.valid?
|
61
|
+
Log.info "Skipping '#{file}' (#{runner.parser_errors})"
|
62
|
+
end
|
63
|
+
Log.info "Ignoring '#{file}'" if runner.ignored?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
rescue Exception => e
|
67
|
+
Log.error(e.message)
|
68
|
+
backtrace = " " + e.backtrace.inject("") { |msg, s| msg << "#{s}\n" }
|
69
|
+
Log.debug(backtrace)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def summary
|
74
|
+
if succeeded?
|
75
|
+
case run_count
|
76
|
+
when 0 then res = "\nNo scenario to run."
|
77
|
+
when 1 then res = "\nOne scenario SUCCEEDED"
|
78
|
+
else res = "\n#{run_count} scenarios SUCCEEDED"
|
79
|
+
end
|
80
|
+
else
|
81
|
+
i = 1
|
82
|
+
res = "\nErrors summary:\n"
|
83
|
+
failures.each do |file, errors|
|
84
|
+
res << "\n#{i.to_s}) Scenario '#{file}' failed with: "
|
85
|
+
errors.each do |error|
|
86
|
+
res << "\n "
|
87
|
+
res << error
|
88
|
+
end
|
89
|
+
i = i + 1
|
90
|
+
end
|
91
|
+
res << "\n\n#{i - 1} of #{run_count} scenario#{'s' if run_count > 1} FAILED"
|
92
|
+
end
|
93
|
+
res
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
data/lib/file_set.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# Set of files with given extension in given directory
|
2
|
+
# See resat.rb for usage information.
|
3
|
+
#
|
4
|
+
|
5
|
+
module Resat
|
6
|
+
class FileSet < Array
|
7
|
+
|
8
|
+
# Folders that won't be scanned for files
|
9
|
+
IGNORED_FOLDERS = %w{ . .. .svn .git }
|
10
|
+
|
11
|
+
# Initialize with all file names found in 'dir' and its sub-directories
|
12
|
+
# with given file extensions
|
13
|
+
def initialize(dir, extensions)
|
14
|
+
super(0)
|
15
|
+
concat(FileSet.gather_files(dir, extensions))
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.gather_files(dir, extensions)
|
19
|
+
files = Array.new
|
20
|
+
Dir.foreach(dir) do |filename|
|
21
|
+
if File.directory?(filename)
|
22
|
+
unless IGNORED_FOLDERS.include?(filename)
|
23
|
+
files.concat(FileSet.gather_files(filename, extensions))
|
24
|
+
end
|
25
|
+
elsif extensions.include?(File.extname(filename))
|
26
|
+
files << File.join(dir, filename)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
files
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
data/lib/filter.rb
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
# Resat response filter
|
2
|
+
# Use response filters to validate responses and/or store response elements in
|
3
|
+
# variables.
|
4
|
+
# Automatically hydrated with Kwalify from YAML definition.
|
5
|
+
# See resat.rb for usage information.
|
6
|
+
#
|
7
|
+
|
8
|
+
module Resat
|
9
|
+
|
10
|
+
class Filter
|
11
|
+
include Kwalify::Util::HashLike
|
12
|
+
attr_accessor :failures
|
13
|
+
|
14
|
+
def prepare
|
15
|
+
@is_empty ||= false
|
16
|
+
@failures = []
|
17
|
+
Log.info("Running filter '#{@name}'")
|
18
|
+
end
|
19
|
+
|
20
|
+
# Run filter on given response
|
21
|
+
def run(request)
|
22
|
+
@request = request
|
23
|
+
@response = request.response
|
24
|
+
validate
|
25
|
+
extract
|
26
|
+
end
|
27
|
+
|
28
|
+
# Validate response
|
29
|
+
def validate
|
30
|
+
unless @response
|
31
|
+
@failures << "No response to validate."
|
32
|
+
return
|
33
|
+
end
|
34
|
+
|
35
|
+
# 1. Check emptyness
|
36
|
+
if @target == 'header'
|
37
|
+
if @is_empty != (@response.size == 0)
|
38
|
+
@failures << "Response header #{'not ' if @is_empty}empty."
|
39
|
+
end
|
40
|
+
else
|
41
|
+
if @is_empty != (@response.body.nil? || @response.body.size <= 1)
|
42
|
+
@failures << "Response body #{'not ' if @is_empty}empty."
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# 2. Check required fields
|
47
|
+
@required_fields.each do |field|
|
48
|
+
unless @request.has_response_field?(field, target)
|
49
|
+
@failures << "Missing #{target} field '#{field}'."
|
50
|
+
end
|
51
|
+
end if @required_fields
|
52
|
+
|
53
|
+
# 3. Evaluate validators
|
54
|
+
@validators.each do |v|
|
55
|
+
if @request.has_response_field?(v.field, @target)
|
56
|
+
field = @request.get_response_field(v.field, @target)
|
57
|
+
if v.pattern
|
58
|
+
Variables.substitute!(v.pattern)
|
59
|
+
unless Regexp.new(v.pattern).match(field)
|
60
|
+
@failures << "Validator /#{v.pattern} failed on '#{field}' from #{@target} field '#{v.field}'."
|
61
|
+
end
|
62
|
+
end
|
63
|
+
unless !!v.is_empty == field.empty?
|
64
|
+
@failures << "#{@target.capitalize} field '#{v.field}' #{'not ' if v.is_empty}empty."
|
65
|
+
end
|
66
|
+
else
|
67
|
+
@failures << "Missing #{@target} field '#{v.field}'."
|
68
|
+
end
|
69
|
+
end if @validators
|
70
|
+
end
|
71
|
+
|
72
|
+
# Extract elements from response
|
73
|
+
def extract
|
74
|
+
@extractors.each do |ex|
|
75
|
+
Variables.substitute!(ex.field)
|
76
|
+
if @request.has_response_field?(ex.field, @target)
|
77
|
+
field = @request.get_response_field(ex.field, @target)
|
78
|
+
if ex.pattern
|
79
|
+
Variables.substitute!(ex.pattern)
|
80
|
+
Regexp.new(ex.pattern).match(field)
|
81
|
+
if Regexp.last_match
|
82
|
+
Variables[ex.variable] = Regexp.last_match(1)
|
83
|
+
else
|
84
|
+
Log.warn("Extraction from response #{@target} field '#{ex.field}' ('#{field}') with pattern '#{ex.pattern}' failed.")
|
85
|
+
end
|
86
|
+
else
|
87
|
+
Variables[ex.variable] = field
|
88
|
+
end
|
89
|
+
Variables.mark_for_save(ex.variable) if ex.save
|
90
|
+
Variables.export(ex.variable) if ex.export
|
91
|
+
else
|
92
|
+
Log.warn("Extraction from response #{@target} field '#{ex.field}' failed: field not found.")
|
93
|
+
end
|
94
|
+
end if @extractors
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
|
99
|
+
# Classes for instances hydrated by Kwalify
|
100
|
+
|
101
|
+
class Validator
|
102
|
+
include Kwalify::Util::HashLike
|
103
|
+
attr_accessor :field, :is_empty, :pattern
|
104
|
+
end
|
105
|
+
|
106
|
+
class Extractor
|
107
|
+
include Kwalify::Util::HashLike
|
108
|
+
attr_accessor :field, :pattern, :variable
|
109
|
+
def save; @save || false; end
|
110
|
+
def export; @export || false; end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
data/lib/guard.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# Resat response filter
|
2
|
+
# Use response filters to validate responses and/or store response elements in
|
3
|
+
# variables.
|
4
|
+
# Automatically hydrated with Kwalify from YAML definition.
|
5
|
+
# See resat.rb for usage information.
|
6
|
+
#
|
7
|
+
|
8
|
+
module Resat
|
9
|
+
|
10
|
+
class Guard
|
11
|
+
include Kwalify::Util::HashLike
|
12
|
+
attr_accessor :failures
|
13
|
+
|
14
|
+
def prepare
|
15
|
+
@timeout ||= 120
|
16
|
+
@period ||= 5
|
17
|
+
@failures = []
|
18
|
+
Variables.substitute!(@pattern)
|
19
|
+
Log.info("Waiting for guard #{@name} with pattern /#{@pattern.to_s}/")
|
20
|
+
end
|
21
|
+
|
22
|
+
def wait(request)
|
23
|
+
r = Regexp.new(@pattern)
|
24
|
+
r.match(request.get_response_field(@field, @target))
|
25
|
+
expiration = DateTime.now + @timeout
|
26
|
+
while !Regexp.last_match && DateTime.now < expiration && request.failures.empty?
|
27
|
+
sleep @period
|
28
|
+
request.send
|
29
|
+
r.match(request.get_response_field(@field, @target))
|
30
|
+
end
|
31
|
+
@failures << "Guard '#{@name}' timed out waiting for field '#{@field}' with pattern '#{@pattern ? @pattern : '<NONE>'}' from response #{@target}." if !Regexp.last_match
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
data/lib/handler.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# Resat response handler
|
2
|
+
# Allows defining a handler that gets the request and response objects for custom processing
|
3
|
+
# Handler should be a module and expose the following methods:
|
4
|
+
# - :process takes two argumnents: first argument is an instance of Net::HTTPRequest while
|
5
|
+
# second argument is an instance of Net::HTTPResponse.
|
6
|
+
# :process should initialize the value returned by :failures (see below)
|
7
|
+
# - :failures which returns an array of error messages when processing
|
8
|
+
# the response results in errors or an empty array when the processing
|
9
|
+
# is successful.
|
10
|
+
|
11
|
+
module Resat
|
12
|
+
|
13
|
+
class Handler
|
14
|
+
include Kwalify::Util::HashLike
|
15
|
+
attr_accessor :failures
|
16
|
+
|
17
|
+
def prepare
|
18
|
+
@failures = []
|
19
|
+
Log.info("Running handler '#{@name}'")
|
20
|
+
end
|
21
|
+
|
22
|
+
def run(request)
|
23
|
+
klass = module_class(@module)
|
24
|
+
h = klass.new
|
25
|
+
h.process(request.request, request.response)
|
26
|
+
@failures += h.failures.values.to_a if h.failures
|
27
|
+
end
|
28
|
+
|
29
|
+
protected
|
30
|
+
|
31
|
+
# Create and cache instance of Class which includes
|
32
|
+
# given module.
|
33
|
+
def module_class(mod)
|
34
|
+
@@modules ||= {}
|
35
|
+
unless klass = @@modules[mod]
|
36
|
+
klass = Class.new(Object) { include Handler.get_module(mod) }
|
37
|
+
@@modules[mod] = klass
|
38
|
+
end
|
39
|
+
klass
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.get_module(name)
|
43
|
+
parts = name.split('::')
|
44
|
+
index = 0
|
45
|
+
res = Kernel
|
46
|
+
index += 1 while (index < parts.size) && (res = res.const_get(parts[index]))
|
47
|
+
res
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# Helper methods that wrap common Kwalify use case
|
2
|
+
#
|
3
|
+
|
4
|
+
require 'kwalify'
|
5
|
+
|
6
|
+
module Resat
|
7
|
+
|
8
|
+
class KwalifyHelper
|
9
|
+
|
10
|
+
# Create new parser from given schema file
|
11
|
+
def KwalifyHelper.new_parser(schema_file)
|
12
|
+
schema = Kwalify::Yaml.load_file(schema_file)
|
13
|
+
validator = Kwalify::Validator.new(schema)
|
14
|
+
res = Kwalify::Yaml::Parser.new(validator)
|
15
|
+
res.data_binding = true
|
16
|
+
res
|
17
|
+
end
|
18
|
+
|
19
|
+
# Format error message from parser errors
|
20
|
+
def KwalifyHelper.parser_error(parser)
|
21
|
+
first = true
|
22
|
+
parser.errors.inject("") do |msg, e|
|
23
|
+
msg << "\n" unless first
|
24
|
+
first = false if first
|
25
|
+
msg << "#{e.linenum}:#{e.column} [#{e.path}] #{e.message}"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|