webspicy 0.16.1 → 0.19.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -5
- data/bin/webspicy +6 -1
- data/doc/1-black-box-scene.md +109 -0
- data/doc/2-black-box-testing.md +27 -0
- data/doc/3-specification-importance.md +41 -0
- data/doc/4-sequence-diagram.md +82 -0
- data/lib/webspicy.rb +15 -4
- data/lib/webspicy/configuration.rb +57 -8
- data/lib/webspicy/configuration/scope.rb +22 -16
- data/lib/webspicy/formaldoc.fio +3 -1
- data/lib/webspicy/specification.rb +12 -10
- data/lib/webspicy/specification/condition.rb +23 -0
- data/lib/webspicy/specification/errcondition.rb +17 -0
- data/lib/webspicy/specification/postcondition.rb +1 -0
- data/lib/webspicy/specification/precondition.rb +1 -0
- data/lib/webspicy/specification/precondition/robust_to_invalid_input.rb +1 -1
- data/lib/webspicy/specification/service.rb +38 -25
- data/lib/webspicy/specification/test_case.rb +10 -17
- data/lib/webspicy/support.rb +22 -0
- data/lib/webspicy/support/data_object.rb +25 -0
- data/lib/webspicy/support/hooks.rb +65 -0
- data/lib/webspicy/support/world.rb +47 -0
- data/lib/webspicy/tester.rb +120 -61
- data/lib/webspicy/tester/asserter.rb +9 -4
- data/lib/webspicy/tester/assertions.rb +8 -9
- data/lib/webspicy/tester/client.rb +22 -42
- data/lib/webspicy/tester/failure.rb +6 -0
- data/lib/webspicy/tester/file_checker.rb +22 -0
- data/lib/webspicy/tester/invocation.rb +15 -196
- data/lib/webspicy/tester/reporter.rb +85 -0
- data/lib/webspicy/tester/reporter/composite.rb +38 -0
- data/lib/webspicy/tester/reporter/documentation.rb +67 -0
- data/lib/webspicy/tester/reporter/error_count.rb +25 -0
- data/lib/webspicy/tester/reporter/exceptions.rb +60 -0
- data/lib/webspicy/tester/reporter/file_progress.rb +22 -0
- data/lib/webspicy/tester/reporter/file_summary.rb +42 -0
- data/lib/webspicy/tester/reporter/progress.rb +28 -0
- data/lib/webspicy/tester/reporter/summary.rb +62 -0
- data/lib/webspicy/tester/result.rb +139 -0
- data/lib/webspicy/tester/result/assert_met.rb +29 -0
- data/lib/webspicy/tester/result/check.rb +33 -0
- data/lib/webspicy/tester/result/errcondition_met.rb +29 -0
- data/lib/webspicy/tester/result/error_schema_met.rb +24 -0
- data/lib/webspicy/tester/result/output_schema_met.rb +24 -0
- data/lib/webspicy/tester/result/postcondition_met.rb +29 -0
- data/lib/webspicy/tester/result/response_header_met.rb +43 -0
- data/lib/webspicy/tester/result/response_status_met.rb +25 -0
- data/lib/webspicy/version.rb +2 -2
- data/lib/webspicy/web.rb +4 -0
- data/lib/webspicy/web/client.rb +15 -0
- data/lib/webspicy/{tester → web}/client/http_client.rb +34 -14
- data/lib/webspicy/{tester → web}/client/rack_test_client.rb +3 -3
- data/lib/webspicy/{tester → web}/client/support.rb +2 -2
- data/lib/webspicy/web/invocation.rb +65 -0
- data/lib/webspicy/web/mocker.rb +90 -0
- data/lib/webspicy/web/mocker/config.ru +6 -0
- data/lib/webspicy/{openapi.rb → web/openapi.rb} +0 -0
- data/lib/webspicy/web/openapi/generator.rb +129 -0
- data/spec/unit/configuration/scope/test_each_service.rb +2 -2
- data/spec/unit/configuration/scope/test_each_specification.rb +7 -7
- data/spec/unit/specification/test_condition.rb +26 -0
- data/spec/unit/support/hooks/test_fire_after_each.rb +53 -0
- data/spec/unit/{tester/client/test_around.rb → support/hooks/test_fire_around.rb} +15 -10
- data/spec/unit/support/hooks/test_fire_before_each.rb +53 -0
- data/spec/unit/support/world/fixtures/array.json +8 -0
- data/spec/unit/support/world/fixtures/queue.rb +1 -0
- data/spec/unit/support/world/fixtures/single.json +11 -0
- data/spec/unit/support/world/fixtures/yaml.yml +3 -0
- data/spec/unit/support/world/test_world.rb +56 -0
- data/spec/unit/test_configuration.rb +50 -1
- data/spec/unit/tester/test_asserter.rb +198 -3
- data/spec/unit/tester/test_assertions.rb +8 -6
- data/spec/unit/web/mocker/test_mocker.rb +35 -0
- data/spec/unit/web/openapi/test_generator.rb +31 -0
- metadata +72 -61
- data/examples/restful/Gemfile +0 -5
- data/examples/restful/Gemfile.lock +0 -105
- data/examples/restful/Rakefile +0 -25
- data/examples/restful/app.rb +0 -180
- data/examples/restful/webspicy/config.rb +0 -23
- data/examples/restful/webspicy/rack.rb +0 -7
- data/examples/restful/webspicy/real.rb +0 -8
- data/examples/restful/webspicy/schema.fio +0 -20
- data/examples/restful/webspicy/support/must_be_an_admin.rb +0 -20
- data/examples/restful/webspicy/support/must_be_authenticated.rb +0 -48
- data/examples/restful/webspicy/support/todo_removed.rb +0 -18
- data/examples/restful/webspicy/todo/deleteTodo.yml +0 -52
- data/examples/restful/webspicy/todo/getTodo.yml +0 -50
- data/examples/restful/webspicy/todo/getTodoSingleServiceFormat.yml +0 -46
- data/examples/restful/webspicy/todo/getTodos.yml +0 -36
- data/examples/restful/webspicy/todo/options.yml +0 -32
- data/examples/restful/webspicy/todo/patchTodo.yml +0 -66
- data/examples/restful/webspicy/todo/postCsv.yml +0 -43
- data/examples/restful/webspicy/todo/postFile.yml +0 -40
- data/examples/restful/webspicy/todo/postTodos.yml +0 -51
- data/examples/restful/webspicy/todo/putTodo.yml +0 -65
- data/examples/restful/webspicy/todo/todos.csv +0 -4
- data/examples/single_spec/spec.yml +0 -59
- data/examples/website/config.rb +0 -2
- data/examples/website/schema.fio +0 -1
- data/examples/website/specification/get-http.yml +0 -34
- data/examples/website/specification/get-https.yml +0 -34
- data/lib/webspicy/checker.rb +0 -25
- data/lib/webspicy/mocker.rb +0 -88
- data/lib/webspicy/openapi/generator.rb +0 -127
- data/lib/webspicy/tester/rspec_asserter.rb +0 -108
- data/lib/webspicy/tester/rspec_matchers.rb +0 -104
- data/spec/unit/mocker/test_mocker.rb +0 -32
- data/spec/unit/openapi/test_generator.rb +0 -28
@@ -7,38 +7,33 @@ module Webspicy
|
|
7
7
|
end
|
8
8
|
attr_reader :config
|
9
9
|
|
10
|
-
def preconditions
|
11
|
-
config.preconditions
|
12
|
-
end
|
13
|
-
|
14
|
-
def postconditions
|
15
|
-
config.postconditions
|
16
|
-
end
|
17
|
-
|
18
10
|
###
|
19
11
|
### Eachers -- Allow navigating the web service definitions
|
20
12
|
###
|
21
13
|
|
22
14
|
# Yields each specification file in the current scope
|
23
|
-
def each_specification_file(&bl)
|
24
|
-
return enum_for(:each_specification_file) unless block_given?
|
25
|
-
_each_specification_file(config, &bl)
|
15
|
+
def each_specification_file(apply_filter = true, &bl)
|
16
|
+
return enum_for(:each_specification_file, apply_filter) unless block_given?
|
17
|
+
_each_specification_file(config, apply_filter, &bl)
|
26
18
|
end
|
27
19
|
|
28
20
|
# Recursive implementation of `each_specification_file` for each
|
29
21
|
# folder in the configuration.
|
30
|
-
def _each_specification_file(config)
|
22
|
+
def _each_specification_file(config, apply_filter = true)
|
31
23
|
folder = config.folder
|
32
|
-
folder
|
24
|
+
world = config.folder/"world"
|
25
|
+
fs = folder.glob("**/*.yml").reject{|f| f.to_s.start_with?(world.to_s) }
|
26
|
+
fs = fs.select(&to_filter_proc(config.file_filter)) if apply_filter
|
27
|
+
fs.each do |file|
|
33
28
|
yield file, folder
|
34
29
|
end
|
35
30
|
end
|
36
31
|
private :_each_specification_file
|
37
32
|
|
38
33
|
# Yields each specification in the current scope in turn.
|
39
|
-
def each_specification(&bl)
|
40
|
-
return enum_for(:each_specification) unless block_given?
|
41
|
-
each_specification_file do |file, folder|
|
34
|
+
def each_specification(apply_filter = true, &bl)
|
35
|
+
return enum_for(:each_specification, apply_filter) unless block_given?
|
36
|
+
each_specification_file(apply_filter) do |file, folder|
|
42
37
|
yield Webspicy.specification(file.load, file, self)
|
43
38
|
end
|
44
39
|
end
|
@@ -76,6 +71,17 @@ module Webspicy
|
|
76
71
|
each_generated_counterexamples(service, &bl)
|
77
72
|
end
|
78
73
|
|
74
|
+
def find_test_case(method, url)
|
75
|
+
each_specification(false) do |spec|
|
76
|
+
next unless spec.url == url
|
77
|
+
spec.services.each do |service|
|
78
|
+
next unless service.method == method
|
79
|
+
return service.examples.first
|
80
|
+
end
|
81
|
+
end
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
|
79
85
|
###
|
80
86
|
### Schemas -- For parsing input and output data schemas found in
|
81
87
|
### web service definitions
|
data/lib/webspicy/formaldoc.fio
CHANGED
@@ -25,11 +25,12 @@ Specification = .Webspicy::Specification
|
|
25
25
|
}
|
26
26
|
<singleservice> {
|
27
27
|
name :? String
|
28
|
-
url :
|
28
|
+
url : String
|
29
29
|
method : Method
|
30
30
|
description : String
|
31
31
|
preconditions :? [String]|String
|
32
32
|
postconditions :? [String]|String
|
33
|
+
errconditions :? [String]|String
|
33
34
|
input_schema : Schema
|
34
35
|
output_schema : Schema
|
35
36
|
error_schema : Schema
|
@@ -45,6 +46,7 @@ Service =
|
|
45
46
|
description : String
|
46
47
|
preconditions :? [String]|String
|
47
48
|
postconditions :? [String]|String
|
49
|
+
errconditions :? [String]|String
|
48
50
|
input_schema : Schema
|
49
51
|
output_schema : Schema
|
50
52
|
error_schema : Schema
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module Webspicy
|
2
2
|
class Specification
|
3
|
+
include Support::DataObject
|
3
4
|
|
4
5
|
def initialize(raw, location = nil)
|
5
|
-
|
6
|
+
super(raw)
|
6
7
|
@location = location
|
7
8
|
bind_services
|
8
9
|
end
|
10
|
+
attr_accessor :config
|
9
11
|
attr_reader :location
|
10
12
|
|
11
13
|
def self.info(raw)
|
@@ -14,11 +16,13 @@ module Webspicy
|
|
14
16
|
|
15
17
|
def self.singleservice(raw)
|
16
18
|
converted = {
|
17
|
-
name: raw
|
19
|
+
name: raw[:name] || "Unamed specification",
|
18
20
|
url: raw[:url],
|
19
|
-
services: [
|
21
|
+
services: [
|
22
|
+
Webspicy.service(raw.reject{|k| k==:url or k==:name }, Webspicy.current_scope)
|
23
|
+
]
|
20
24
|
}
|
21
|
-
|
25
|
+
info(converted)
|
22
26
|
end
|
23
27
|
|
24
28
|
def located_at!(location)
|
@@ -44,7 +48,7 @@ module Webspicy
|
|
44
48
|
end
|
45
49
|
|
46
50
|
def services
|
47
|
-
@raw[:services]
|
51
|
+
@raw[:services] || []
|
48
52
|
end
|
49
53
|
|
50
54
|
def url_placeholders
|
@@ -60,10 +64,6 @@ module Webspicy
|
|
60
64
|
[ url, rest ]
|
61
65
|
end
|
62
66
|
|
63
|
-
def to_info
|
64
|
-
@raw
|
65
|
-
end
|
66
|
-
|
67
67
|
def to_singleservice
|
68
68
|
raise NotImplementedError
|
69
69
|
end
|
@@ -85,7 +85,7 @@ module Webspicy
|
|
85
85
|
end
|
86
86
|
|
87
87
|
def bind_services
|
88
|
-
|
88
|
+
services.each do |s|
|
89
89
|
s.specification = self
|
90
90
|
end
|
91
91
|
end
|
@@ -93,7 +93,9 @@ module Webspicy
|
|
93
93
|
end # class Specification
|
94
94
|
end # module Webspicy
|
95
95
|
require_relative 'specification/service'
|
96
|
+
require_relative 'specification/condition'
|
96
97
|
require_relative 'specification/precondition'
|
97
98
|
require_relative 'specification/postcondition'
|
99
|
+
require_relative 'specification/errcondition'
|
98
100
|
require_relative 'specification/test_case'
|
99
101
|
require_relative 'specification/file_upload'
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Webspicy
|
2
|
+
class Specification
|
3
|
+
module Condition
|
4
|
+
|
5
|
+
MATCH_ALL = "__all__"
|
6
|
+
|
7
|
+
attr_accessor :matching_description
|
8
|
+
|
9
|
+
def to_s
|
10
|
+
if matching_description == MATCH_ALL
|
11
|
+
self.class.name
|
12
|
+
else
|
13
|
+
matching_description
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def sooner_or_later(*args, &bl)
|
18
|
+
Webspicy::Support.sooner_or_later(*args, &bl)
|
19
|
+
end
|
20
|
+
|
21
|
+
end # module Condition
|
22
|
+
end # class Specification
|
23
|
+
end # module Webspicy
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Webspicy
|
2
|
+
class Specification
|
3
|
+
module Errcondition
|
4
|
+
include Condition
|
5
|
+
|
6
|
+
def self.match(service, descr)
|
7
|
+
end
|
8
|
+
|
9
|
+
def instrument(test_case, client)
|
10
|
+
end
|
11
|
+
|
12
|
+
def check(invocation)
|
13
|
+
end
|
14
|
+
|
15
|
+
end # module Errcondition
|
16
|
+
end # module Specification
|
17
|
+
end # module Webspicy
|
@@ -40,7 +40,7 @@ module Webspicy
|
|
40
40
|
def empty_input_counterexamples(service, first)
|
41
41
|
placeholders = service.specification.url_placeholders
|
42
42
|
empty_input = first.params.reject{|k| !placeholders.include?(k) }
|
43
|
-
if
|
43
|
+
if invalid_input?(service, empty_input)
|
44
44
|
[first.mutate({
|
45
45
|
:description => "it is robust to an invalid empty input (RobustToInvalidInput)",
|
46
46
|
:dress_params => false,
|
@@ -1,13 +1,12 @@
|
|
1
1
|
module Webspicy
|
2
2
|
class Specification
|
3
3
|
class Service
|
4
|
+
include Support::DataObject
|
4
5
|
|
5
6
|
def initialize(raw)
|
6
|
-
|
7
|
+
super(raw)
|
7
8
|
bind_examples
|
8
9
|
bind_counterexamples
|
9
|
-
@preconditions = compile_preconditions
|
10
|
-
@postconditions = compile_postconditions
|
11
10
|
end
|
12
11
|
attr_accessor :specification
|
13
12
|
|
@@ -15,6 +14,10 @@ module Webspicy
|
|
15
14
|
new(raw)
|
16
15
|
end
|
17
16
|
|
17
|
+
def config
|
18
|
+
specification.config
|
19
|
+
end
|
20
|
+
|
18
21
|
def method
|
19
22
|
@raw[:method]
|
20
23
|
end
|
@@ -24,7 +27,7 @@ module Webspicy
|
|
24
27
|
end
|
25
28
|
|
26
29
|
def preconditions
|
27
|
-
@preconditions
|
30
|
+
@preconditions ||= compile_preconditions
|
28
31
|
end
|
29
32
|
|
30
33
|
def has_preconditions?
|
@@ -32,23 +35,31 @@ module Webspicy
|
|
32
35
|
end
|
33
36
|
|
34
37
|
def postconditions
|
35
|
-
@postconditions
|
38
|
+
@postconditions ||= compile_postconditions
|
36
39
|
end
|
37
40
|
|
38
41
|
def has_postconditions?
|
39
42
|
!postconditions.empty?
|
40
43
|
end
|
41
44
|
|
45
|
+
def errconditions
|
46
|
+
@errconditions ||= compile_errconditions
|
47
|
+
end
|
48
|
+
|
49
|
+
def has_errconditions?
|
50
|
+
!errconditions.empty?
|
51
|
+
end
|
52
|
+
|
42
53
|
def default_example
|
43
54
|
@raw[:default_example]
|
44
55
|
end
|
45
56
|
|
46
57
|
def examples
|
47
|
-
@raw[:examples]
|
58
|
+
@raw[:examples] || []
|
48
59
|
end
|
49
60
|
|
50
61
|
def counterexamples
|
51
|
-
@raw[:counterexamples]
|
62
|
+
@raw[:counterexamples] || []
|
52
63
|
end
|
53
64
|
|
54
65
|
def generated_counterexamples
|
@@ -76,49 +87,51 @@ module Webspicy
|
|
76
87
|
input_schema.dress(params)
|
77
88
|
end
|
78
89
|
|
79
|
-
def to_info
|
80
|
-
@raw
|
81
|
-
end
|
82
|
-
|
83
90
|
def to_s
|
84
91
|
"#{method} #{specification.url}"
|
85
92
|
end
|
86
93
|
|
87
94
|
private
|
88
95
|
|
89
|
-
def scope
|
90
|
-
Webspicy.current_scope
|
91
|
-
end
|
92
|
-
|
93
96
|
def compile_preconditions
|
94
97
|
@raw[:preconditions] = [@raw[:preconditions]] if @raw[:preconditions].is_a?(String)
|
95
|
-
compile_conditions(@raw[:preconditions] ||= [],
|
98
|
+
compile_conditions(@raw[:preconditions] ||= [], config.preconditions)
|
96
99
|
end
|
97
100
|
|
98
101
|
def compile_postconditions
|
99
102
|
@raw[:postconditions] = [@raw[:postconditions]] if @raw[:postconditions].is_a?(String)
|
100
|
-
compile_conditions(@raw[:postconditions] ||= [],
|
103
|
+
compile_conditions(@raw[:postconditions] ||= [], config.postconditions)
|
104
|
+
end
|
105
|
+
|
106
|
+
def compile_errconditions
|
107
|
+
@raw[:errconditions] = [@raw[:errconditions]] if @raw[:errconditions].is_a?(String)
|
108
|
+
compile_conditions(@raw[:errconditions] ||= [], config.errconditions)
|
101
109
|
end
|
102
110
|
|
103
111
|
def compile_conditions(descriptions, conditions)
|
104
112
|
# Because we want pre & post to be able to match in all cases
|
105
113
|
# we need at least one condition
|
106
|
-
descriptions = [
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
114
|
+
descriptions = [Condition::MATCH_ALL] if descriptions.empty?
|
115
|
+
conditions.map{|c|
|
116
|
+
instance = nil
|
117
|
+
descr = descriptions.find do |d|
|
118
|
+
instance = c.match(self, d)
|
119
|
+
end
|
120
|
+
if instance && instance.respond_to?(:matching_description=)
|
121
|
+
instance.matching_description = descr
|
122
|
+
end
|
123
|
+
instance
|
124
|
+
}.compact
|
112
125
|
end
|
113
126
|
|
114
127
|
def bind_examples
|
115
|
-
|
128
|
+
examples.each do |ex|
|
116
129
|
ex.bind(self, false)
|
117
130
|
end
|
118
131
|
end
|
119
132
|
|
120
133
|
def bind_counterexamples
|
121
|
-
|
134
|
+
counterexamples.each do |ex|
|
122
135
|
ex.bind(self, true)
|
123
136
|
end
|
124
137
|
end
|
@@ -1,23 +1,25 @@
|
|
1
1
|
module Webspicy
|
2
2
|
class Specification
|
3
3
|
class TestCase
|
4
|
+
include Support::DataObject
|
4
5
|
|
5
6
|
def initialize(raw)
|
6
|
-
|
7
|
+
super(raw)
|
7
8
|
@counterexample = nil
|
8
9
|
end
|
9
10
|
attr_reader :service
|
10
11
|
attr_reader :counterexample
|
11
12
|
|
12
|
-
attr_accessor :raw
|
13
|
-
protected :raw, :raw=
|
14
|
-
|
15
13
|
def bind(service, counterexample)
|
16
14
|
@service = service
|
17
15
|
@counterexample = counterexample
|
18
16
|
self
|
19
17
|
end
|
20
18
|
|
19
|
+
def example?
|
20
|
+
!@counterexample
|
21
|
+
end
|
22
|
+
|
21
23
|
def counterexample?
|
22
24
|
!!@counterexample
|
23
25
|
end
|
@@ -87,6 +89,10 @@ module Webspicy
|
|
87
89
|
expected_status === status
|
88
90
|
end
|
89
91
|
|
92
|
+
def has_expected_status?
|
93
|
+
not expected[:status].nil?
|
94
|
+
end
|
95
|
+
|
90
96
|
def expected_error
|
91
97
|
expected[:error]
|
92
98
|
end
|
@@ -111,19 +117,6 @@ module Webspicy
|
|
111
117
|
!assert.empty?
|
112
118
|
end
|
113
119
|
|
114
|
-
def to_info
|
115
|
-
@raw
|
116
|
-
end
|
117
|
-
|
118
|
-
def instrument(client)
|
119
|
-
service.preconditions.each do |pre|
|
120
|
-
pre.instrument(self, client)
|
121
|
-
end
|
122
|
-
service.postconditions.each do |post|
|
123
|
-
post.instrument(self, client) if post.respond_to?(:instrument)
|
124
|
-
end
|
125
|
-
end
|
126
|
-
|
127
120
|
def mutate(override)
|
128
121
|
m = self.dup
|
129
122
|
m.raw = self.raw.merge(override)
|
data/lib/webspicy/support.rb
CHANGED
@@ -1,2 +1,24 @@
|
|
1
|
+
module Webspicy
|
2
|
+
module Support
|
3
|
+
|
4
|
+
SORL_OPTS = { max: 5, wait: 0.05, raise: false }
|
5
|
+
|
6
|
+
def sooner_or_later(opts = nil)
|
7
|
+
opts = SORL_OPTS.merge(opts || {})
|
8
|
+
left, wait_ms = opts[:max], opts[:wait]
|
9
|
+
until (r = yield) || left == 0
|
10
|
+
sleep(wait_ms)
|
11
|
+
wait_ms, left = wait_ms*2, left - 1
|
12
|
+
end
|
13
|
+
raise TimeoutError, "Timeout on sooner-or-later" if r.nil? && opts[:raise]
|
14
|
+
r
|
15
|
+
end
|
16
|
+
module_function :sooner_or_later
|
17
|
+
|
18
|
+
end # module Support
|
19
|
+
end # module Webspicy
|
20
|
+
require_relative 'support/data_object'
|
1
21
|
require_relative 'support/status_range'
|
2
22
|
require_relative 'support/colorize'
|
23
|
+
require_relative 'support/world'
|
24
|
+
require_relative 'support/hooks'
|