attestify 0.1.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rubocop.yml +37 -0
- data/.travis.yml +8 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +75 -0
- data/Rakefile +5 -0
- data/attestify.gemspec +26 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/attestify +3 -0
- data/lib/attestify.rb +20 -0
- data/lib/attestify/assertion_results.rb +93 -0
- data/lib/attestify/assertions.rb +334 -0
- data/lib/attestify/assertions/output_assertion.rb +62 -0
- data/lib/attestify/cli.rb +84 -0
- data/lib/attestify/color_reporter.rb +127 -0
- data/lib/attestify/mock.rb +118 -0
- data/lib/attestify/rake_task.rb +29 -0
- data/lib/attestify/reporter.rb +171 -0
- data/lib/attestify/skipped_error.rb +6 -0
- data/lib/attestify/test.rb +89 -0
- data/lib/attestify/test_list.rb +162 -0
- data/lib/attestify/test_runner.rb +46 -0
- data/lib/attestify/timer.rb +30 -0
- data/lib/attestify/version.rb +3 -0
- metadata +115 -0
@@ -0,0 +1,62 @@
|
|
1
|
+
module Attestify
|
2
|
+
module Assertions
|
3
|
+
# A helper class for Attestify::Assertions#assert_output.
|
4
|
+
class OutputAssertion
|
5
|
+
def initialize(expected_stdout, expected_stderr, stdout, stderr, message)
|
6
|
+
@expected_stdout = expected_stdout
|
7
|
+
@expected_stderr = expected_stderr
|
8
|
+
@stdout = stdout
|
9
|
+
@stderr = stderr
|
10
|
+
@message = message
|
11
|
+
end
|
12
|
+
|
13
|
+
def assert
|
14
|
+
assert_stdout && assert_stderr
|
15
|
+
end
|
16
|
+
|
17
|
+
def message
|
18
|
+
return @message if @message
|
19
|
+
messages = [stdout_message, stderr_message]
|
20
|
+
"Expected #{messages.compact.join(", and ")}"
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def assert_stdout
|
26
|
+
assert_output(@expected_stdout, @stdout)
|
27
|
+
end
|
28
|
+
|
29
|
+
def assert_stderr
|
30
|
+
assert_output(@expected_stderr, @stderr)
|
31
|
+
end
|
32
|
+
|
33
|
+
def assert_output(expected, actual)
|
34
|
+
return true unless expected
|
35
|
+
|
36
|
+
if expected.is_a?(String)
|
37
|
+
expected == actual
|
38
|
+
else
|
39
|
+
actual =~ expected
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def stdout_message
|
44
|
+
output_message("$stdout", @expected_stdout, @stdout)
|
45
|
+
end
|
46
|
+
|
47
|
+
def stderr_message
|
48
|
+
output_message("$stderr", @expected_stderr, @stderr)
|
49
|
+
end
|
50
|
+
|
51
|
+
def output_message(label, expected, actual)
|
52
|
+
return nil unless expected
|
53
|
+
|
54
|
+
if expected.is_a?(String)
|
55
|
+
"#{label}: #{expected.inspect} == #{actual.inspect}"
|
56
|
+
else
|
57
|
+
"#{label}: #{actual.inspect} =~ #{expected.inspect}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require "attestify"
|
2
|
+
require "optparse"
|
3
|
+
|
4
|
+
module Attestify
|
5
|
+
# Command Line Interface for running Attestify tests.
|
6
|
+
class CLI
|
7
|
+
def initialize(args = ARGV)
|
8
|
+
@args = args
|
9
|
+
@exit_code = true
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.start
|
13
|
+
new.start
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_list
|
17
|
+
@test_list ||= Attestify::TestList.new(@args, dir: options[:directory])
|
18
|
+
end
|
19
|
+
|
20
|
+
def reporter
|
21
|
+
@reporter ||=
|
22
|
+
if options[:color]
|
23
|
+
Attestify::ColorReporter.new
|
24
|
+
else
|
25
|
+
Attestify::Reporter.new
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def options
|
30
|
+
@options ||= {
|
31
|
+
color: true
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
35
|
+
def option_parser # rubocop:disable Metrics/MethodLength
|
36
|
+
@option_parser ||= OptionParser.new do |opts|
|
37
|
+
opts.banner = "Usage: attestify [options] [test_files ...]"
|
38
|
+
|
39
|
+
opts.on("-c", "--color", "Run with color") do
|
40
|
+
options[:color] = true
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on("-C", "--no-color", "Run without color") do
|
44
|
+
options[:color] = false
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on("-d", "--directory [DIR]", "Run the tests in the provided DIR") do |dir|
|
48
|
+
options[:directory] = dir
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.on("-h", "--help", "Output this help") do
|
52
|
+
puts opts
|
53
|
+
ignore_reporting
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def ignore_reporting
|
60
|
+
@ignore_reporting = true
|
61
|
+
end
|
62
|
+
|
63
|
+
def parse_arguments
|
64
|
+
option_parser.parse!(@args)
|
65
|
+
end
|
66
|
+
|
67
|
+
def start
|
68
|
+
timer = Attestify::Timer.time { run }
|
69
|
+
rescue => e
|
70
|
+
@exit_code = 2
|
71
|
+
STDERR.puts("Error running tests: #{e}\n #{e.backtrace.join("\n ")}")
|
72
|
+
ensure
|
73
|
+
reporter.timer = timer
|
74
|
+
reporter.report unless @ignore_reporting
|
75
|
+
exit(@exit_code)
|
76
|
+
end
|
77
|
+
|
78
|
+
def run
|
79
|
+
parse_arguments
|
80
|
+
Attestify::TestRunner.new(test_list, reporter).run
|
81
|
+
@exit_code = 1 unless reporter.passed?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require "attestify"
|
2
|
+
|
3
|
+
module Attestify
|
4
|
+
# Reports results to the console, with color!
|
5
|
+
class ColorReporter < Attestify::Reporter # rubocop:disable Metrics/ClassLength
|
6
|
+
private
|
7
|
+
|
8
|
+
def puts_failure_header(result, number)
|
9
|
+
print color_code(color_for(result))
|
10
|
+
super
|
11
|
+
print color_code(:reset)
|
12
|
+
end
|
13
|
+
|
14
|
+
def puts_failure_detail(failure_detail, number, sub_number)
|
15
|
+
print color_code(color_for_detail(failure_detail))
|
16
|
+
super
|
17
|
+
print color_code(:reset)
|
18
|
+
end
|
19
|
+
|
20
|
+
def print_result_code(result)
|
21
|
+
print color_code(color_for(result))
|
22
|
+
super
|
23
|
+
print color_code(:reset)
|
24
|
+
end
|
25
|
+
|
26
|
+
def total_tests
|
27
|
+
colorize_from_totals(super)
|
28
|
+
end
|
29
|
+
|
30
|
+
def total_failures
|
31
|
+
colorize_if_positive(super, @total_failures, :red)
|
32
|
+
end
|
33
|
+
|
34
|
+
def total_errors
|
35
|
+
colorize_if_positive(super, @total_errors, :bold_red)
|
36
|
+
end
|
37
|
+
|
38
|
+
def total_skips
|
39
|
+
colorize_if_positive(super, @total_skips, :yellow)
|
40
|
+
end
|
41
|
+
|
42
|
+
def total_assertions
|
43
|
+
colorize_from_totals(super)
|
44
|
+
end
|
45
|
+
|
46
|
+
def total_failed_assertions
|
47
|
+
colorize_if_positive(super, @total_failed_assertions, :red)
|
48
|
+
end
|
49
|
+
|
50
|
+
def rerun_test_command(result)
|
51
|
+
colorize(super, color_for(result))
|
52
|
+
end
|
53
|
+
|
54
|
+
def comment(message)
|
55
|
+
colorize(super, :cyan)
|
56
|
+
end
|
57
|
+
|
58
|
+
def colorize_from_totals(text) # rubocop:disable Metrics/MethodLength
|
59
|
+
color =
|
60
|
+
if @total_errors > 0
|
61
|
+
:bold_red
|
62
|
+
elsif @total_failures > 0
|
63
|
+
:red
|
64
|
+
elsif @total_skips > 0
|
65
|
+
:yellow
|
66
|
+
else
|
67
|
+
:green
|
68
|
+
end
|
69
|
+
|
70
|
+
colorize(text, color)
|
71
|
+
end
|
72
|
+
|
73
|
+
def colorize_if_positive(text, amount, color)
|
74
|
+
if amount > 0
|
75
|
+
colorize(text, color)
|
76
|
+
else
|
77
|
+
text
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def colorize(text, color)
|
82
|
+
"#{color_code(color)}#{text}#{color_code(:reset)}"
|
83
|
+
end
|
84
|
+
|
85
|
+
def color_code(color) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
|
86
|
+
case color
|
87
|
+
when :reset
|
88
|
+
"\e[0m"
|
89
|
+
when :bold_red
|
90
|
+
"\e[1;31m"
|
91
|
+
when :red
|
92
|
+
"\e[31m"
|
93
|
+
when :yellow
|
94
|
+
"\e[33m"
|
95
|
+
when :green
|
96
|
+
"\e[32m"
|
97
|
+
when :cyan
|
98
|
+
"\e[36m"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def color_for(result) # rubocop:disable Metrics/MethodLength
|
103
|
+
if result.skipped?
|
104
|
+
:yellow
|
105
|
+
elsif result.passed?
|
106
|
+
:green
|
107
|
+
elsif result.errored?
|
108
|
+
:bold_red
|
109
|
+
elsif result.failed?
|
110
|
+
:red
|
111
|
+
else
|
112
|
+
:none
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def color_for_detail(failure_detail)
|
117
|
+
case failure_detail.type
|
118
|
+
when :error
|
119
|
+
:bold_red
|
120
|
+
when :failure
|
121
|
+
:red
|
122
|
+
else
|
123
|
+
:none
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
module Attestify
|
2
|
+
# Simple class for mocking objects.
|
3
|
+
class Mock
|
4
|
+
def initialize(test_or_assertions)
|
5
|
+
@assertions = test_or_assertions
|
6
|
+
@assertions = test_or_assertions.assertions if test_or_assertions.respond_to?(:assertions)
|
7
|
+
@expectations_hash = Hash.new { |hash, key| hash[key] = [] }
|
8
|
+
@expectations = []
|
9
|
+
@called_expectations = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def expect(name, return_value, args = [], &block)
|
13
|
+
name = name.to_sym
|
14
|
+
expectation = Attestify::Mock::ExpectedCall.new(name, return_value, args, block)
|
15
|
+
@expectations << expectation
|
16
|
+
@expectations_hash[name.to_sym] << expectation
|
17
|
+
self
|
18
|
+
end
|
19
|
+
|
20
|
+
def verify
|
21
|
+
@called_expectations.each { |x| x.verify(@assertions) }
|
22
|
+
@expectations.reject(&:called?).each { |x| x.verify(@assertions) }
|
23
|
+
end
|
24
|
+
|
25
|
+
def method_missing(method, *args, &block)
|
26
|
+
expectation =
|
27
|
+
if @expectations_hash[method].empty?
|
28
|
+
UnexpectedCall.new(method, args, block)
|
29
|
+
else
|
30
|
+
@expectations_hash[method].shift
|
31
|
+
end
|
32
|
+
|
33
|
+
@called_expectations << expectation
|
34
|
+
expectation.call(args, block)
|
35
|
+
end
|
36
|
+
|
37
|
+
# A base class for both ExpectedCall and UnexpectedCall.
|
38
|
+
class CallExpectation
|
39
|
+
attr_reader :name, :return_value, :args, :block, :actual_args, :actual_block
|
40
|
+
|
41
|
+
def initialize(name, return_value, args, block)
|
42
|
+
@called = false
|
43
|
+
@name = name
|
44
|
+
@return_value = return_value
|
45
|
+
@args = args
|
46
|
+
@block = block
|
47
|
+
end
|
48
|
+
|
49
|
+
def called?
|
50
|
+
@called
|
51
|
+
end
|
52
|
+
|
53
|
+
def call(actual_args, actual_block)
|
54
|
+
@called = true
|
55
|
+
@actual_args = actual_args
|
56
|
+
@actual_block = actual_block
|
57
|
+
block.call(*actual_args, &actual_block) if block
|
58
|
+
return_value
|
59
|
+
end
|
60
|
+
|
61
|
+
def to_s(style = :expected)
|
62
|
+
if style == :expected
|
63
|
+
"#{name}(#{args.map(&:inspect).join(", ")})"
|
64
|
+
else
|
65
|
+
with_block = " { ... }" if actual_block
|
66
|
+
"#{name}(#{actual_args.map(&:inspect).join(", ")})#{with_block}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Contains a mock's method call expectation.
|
72
|
+
class ExpectedCall < CallExpectation
|
73
|
+
def call(args, block)
|
74
|
+
result = super
|
75
|
+
@caller_locations = caller_locations(2) unless arguments_valid?
|
76
|
+
result
|
77
|
+
end
|
78
|
+
|
79
|
+
def verify(assertions)
|
80
|
+
if !called?
|
81
|
+
assertions.record(false, "Missing expected call to mock: #{self}", @caller_locations)
|
82
|
+
elsif !arguments_valid?
|
83
|
+
assertions.record(false, "Expected call to mock: #{self}, but got: #{to_s(:actual)}", @caller_locations)
|
84
|
+
else
|
85
|
+
assertions.record(true)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
private
|
90
|
+
|
91
|
+
def arguments_valid?
|
92
|
+
return false unless args.size == actual_args.size
|
93
|
+
|
94
|
+
args.each_with_index do |arg, i|
|
95
|
+
return false unless arg === actual_args[i] # rubocop:disable Style/CaseEquality
|
96
|
+
end
|
97
|
+
|
98
|
+
true
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# A method call that wasn't expected.
|
103
|
+
class UnexpectedCall < CallExpectation
|
104
|
+
def initialize(name, args, block)
|
105
|
+
super(name, nil, args, block)
|
106
|
+
end
|
107
|
+
|
108
|
+
def call(args, block)
|
109
|
+
@caller_locations = caller_locations(2)
|
110
|
+
super
|
111
|
+
end
|
112
|
+
|
113
|
+
def verify(assertions)
|
114
|
+
assertions.record(false, "Unexpected call to mock: #{to_s(:actual)}", @caller_locations)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "rake"
|
2
|
+
require "rake/tasklib"
|
3
|
+
|
4
|
+
module Attestify
|
5
|
+
# Rake task to run Attestify tests.
|
6
|
+
class RakeTask < Rake::TaskLib
|
7
|
+
attr_reader :name
|
8
|
+
|
9
|
+
def initialize(*args, &block)
|
10
|
+
@name = args.shift || :test
|
11
|
+
define(args, &block)
|
12
|
+
end
|
13
|
+
|
14
|
+
def run_task
|
15
|
+
Attestify::CLI.new([]).start
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def define(args)
|
21
|
+
desc "Run Attestify tests" unless Rake.application.last_description
|
22
|
+
|
23
|
+
task name, *args do |_, task_args|
|
24
|
+
yield(self, task_args) if block_given?
|
25
|
+
run_task
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
require "pathname"
|
2
|
+
|
3
|
+
module Attestify
|
4
|
+
# Reports results to the console.
|
5
|
+
class Reporter # rubocop:disable Metrics/ClassLength
|
6
|
+
attr_accessor :timer
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@failures = []
|
10
|
+
@total_tests = 0
|
11
|
+
@total_assertions = 0
|
12
|
+
@total_failed_assertions = 0
|
13
|
+
@total_failures = 0
|
14
|
+
@total_errors = 0
|
15
|
+
@total_skips = 0
|
16
|
+
end
|
17
|
+
|
18
|
+
def passed?
|
19
|
+
@total_failures + @total_errors == 0
|
20
|
+
end
|
21
|
+
|
22
|
+
def record(result)
|
23
|
+
add_to_totals(result)
|
24
|
+
@failures << result if !result.skipped? && !result.passed?
|
25
|
+
print_result_code(result)
|
26
|
+
end
|
27
|
+
|
28
|
+
def report
|
29
|
+
puts_failures
|
30
|
+
puts_footer
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def add_to_totals(result)
|
36
|
+
@total_tests += 1
|
37
|
+
@total_assertions += result.assertions.total
|
38
|
+
@total_failed_assertions += result.assertions.failed
|
39
|
+
|
40
|
+
if result.skipped?
|
41
|
+
@total_skips += 1
|
42
|
+
elsif result.errored?
|
43
|
+
@total_errors += 1
|
44
|
+
elsif result.failed?
|
45
|
+
@total_failures += 1
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def print_result_code(result)
|
50
|
+
print result.result_code
|
51
|
+
end
|
52
|
+
|
53
|
+
def record_failure(result)
|
54
|
+
@failures << result
|
55
|
+
|
56
|
+
if result.assertions.errored?
|
57
|
+
@total_errors += 1
|
58
|
+
else
|
59
|
+
@total_failures += 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def puts_failures
|
64
|
+
puts
|
65
|
+
|
66
|
+
@failures.each_with_index do |failure, i|
|
67
|
+
puts_failure(failure, i + 1)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def puts_failure(failure, number)
|
72
|
+
puts
|
73
|
+
puts_failure_header(failure, number)
|
74
|
+
puts_failure_details(failure, number)
|
75
|
+
end
|
76
|
+
|
77
|
+
def puts_failure_header(failure, number)
|
78
|
+
puts "#{number}) #{failure.name}"
|
79
|
+
end
|
80
|
+
|
81
|
+
def puts_failure_details(failure, number)
|
82
|
+
failure.assertions.failure_details.each_with_index do |failure_detail, i|
|
83
|
+
puts_failure_detail(failure_detail, number, i + 1)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def puts_failure_detail(failure_detail, number, sub_number)
|
88
|
+
puts
|
89
|
+
puts " #{number}.#{sub_number}) #{failure_detail.message}"
|
90
|
+
puts " #{failure_detail.backtrace_locations.join("\n ")}"
|
91
|
+
end
|
92
|
+
|
93
|
+
def puts_footer
|
94
|
+
puts
|
95
|
+
puts "Finished in #{elapsed_time}, #{tests_per_second}, #{assertions_per_second}"
|
96
|
+
puts "#{total_tests}, #{total_failures}, #{total_errors}, #{total_skips}, " \
|
97
|
+
"#{total_assertions}, #{total_failed_assertions}"
|
98
|
+
puts_failure_reruns
|
99
|
+
end
|
100
|
+
|
101
|
+
def elapsed_time
|
102
|
+
timer || "?"
|
103
|
+
end
|
104
|
+
|
105
|
+
def tests_per_second
|
106
|
+
if timer
|
107
|
+
format("%.1f tests/second", @total_tests.to_f / timer.duration)
|
108
|
+
else
|
109
|
+
"? tests/second"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def assertions_per_second
|
114
|
+
if timer
|
115
|
+
format("%.1f assertions/second", @total_assertions.to_f / timer.duration)
|
116
|
+
else
|
117
|
+
"? assertions/second"
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def total_tests
|
122
|
+
"#{@total_tests} tests"
|
123
|
+
end
|
124
|
+
|
125
|
+
def total_failures
|
126
|
+
"#{@total_failures} failures"
|
127
|
+
end
|
128
|
+
|
129
|
+
def total_errors
|
130
|
+
"#{@total_errors} errors"
|
131
|
+
end
|
132
|
+
|
133
|
+
def total_skips
|
134
|
+
"#{@total_skips} skips"
|
135
|
+
end
|
136
|
+
|
137
|
+
def total_assertions
|
138
|
+
"#{@total_assertions} assertions"
|
139
|
+
end
|
140
|
+
|
141
|
+
def total_failed_assertions
|
142
|
+
"#{@total_failed_assertions} failed assertions"
|
143
|
+
end
|
144
|
+
|
145
|
+
def puts_failure_reruns
|
146
|
+
puts
|
147
|
+
puts "Failed tests:"
|
148
|
+
puts
|
149
|
+
|
150
|
+
@failures.each do |failure|
|
151
|
+
puts_failure_rerun(failure)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def puts_failure_rerun(failure)
|
156
|
+
puts "#{rerun_test_command(failure)} #{comment(failure.name)}"
|
157
|
+
end
|
158
|
+
|
159
|
+
def rerun_test_command(failure)
|
160
|
+
# TODO: Should I create a new method to get the test method...?
|
161
|
+
test_method = failure.instance_variable_get(:@_test_method)
|
162
|
+
source = failure.method(test_method).source_location
|
163
|
+
source[0] = Pathname.new(File.realpath(source[0])).relative_path_from(Pathname.new(File.realpath(".")))
|
164
|
+
"attestify #{source.join(":")}"
|
165
|
+
end
|
166
|
+
|
167
|
+
def comment(message)
|
168
|
+
"# #{message}"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|