webspicy 0.16.2 → 0.20.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +9 -3
  3. data/bin/webspicy +6 -1
  4. data/doc/1-black-box-scene.md +109 -0
  5. data/doc/2-black-box-testing.md +27 -0
  6. data/doc/3-specification-importance.md +41 -0
  7. data/doc/4-sequence-diagram.md +82 -0
  8. data/lib/webspicy.rb +18 -9
  9. data/lib/webspicy/configuration.rb +57 -8
  10. data/lib/webspicy/configuration/scope.rb +22 -16
  11. data/lib/webspicy/formaldoc.fio +7 -5
  12. data/lib/webspicy/specification.rb +10 -9
  13. data/lib/webspicy/specification/condition.rb +48 -0
  14. data/lib/webspicy/specification/err.rb +18 -0
  15. data/lib/webspicy/specification/oldies.rb +4 -0
  16. data/lib/webspicy/specification/oldies/bridge.rb +29 -0
  17. data/lib/webspicy/specification/oldies/errcondition.rb +22 -0
  18. data/lib/webspicy/specification/{postcondition.rb → oldies/postcondition.rb} +6 -0
  19. data/lib/webspicy/specification/{precondition.rb → oldies/precondition.rb} +6 -2
  20. data/lib/webspicy/specification/post.rb +20 -0
  21. data/lib/webspicy/specification/post/missing_condition_impl.rb +15 -0
  22. data/lib/webspicy/specification/post/unexpected_condition_impl.rb +15 -0
  23. data/lib/webspicy/specification/pre.rb +19 -0
  24. data/lib/webspicy/specification/{precondition → pre}/global_request_headers.rb +4 -4
  25. data/lib/webspicy/specification/{precondition → pre}/robust_to_invalid_input.rb +5 -5
  26. data/lib/webspicy/specification/service.rb +58 -24
  27. data/lib/webspicy/specification/test_case.rb +10 -17
  28. data/lib/webspicy/support.rb +32 -0
  29. data/lib/webspicy/support/data_object.rb +25 -0
  30. data/lib/webspicy/support/hooks.rb +65 -0
  31. data/lib/webspicy/support/world.rb +47 -0
  32. data/lib/webspicy/tester.rb +180 -60
  33. data/lib/webspicy/tester/asserter.rb +9 -4
  34. data/lib/webspicy/tester/assertions.rb +8 -9
  35. data/lib/webspicy/tester/client.rb +4 -50
  36. data/lib/webspicy/tester/failure.rb +6 -0
  37. data/lib/webspicy/tester/fakeses.rb +41 -0
  38. data/lib/webspicy/tester/fakeses/email.rb +38 -0
  39. data/lib/webspicy/tester/fakesmtp.rb +39 -0
  40. data/lib/webspicy/tester/fakesmtp/email.rb +27 -0
  41. data/lib/webspicy/tester/file_checker.rb +22 -0
  42. data/lib/webspicy/tester/invocation.rb +15 -196
  43. data/lib/webspicy/tester/reporter.rb +85 -0
  44. data/lib/webspicy/tester/reporter/composite.rb +38 -0
  45. data/lib/webspicy/tester/reporter/documentation.rb +74 -0
  46. data/lib/webspicy/tester/reporter/error_count.rb +25 -0
  47. data/lib/webspicy/tester/reporter/exceptions.rb +62 -0
  48. data/lib/webspicy/tester/reporter/file_progress.rb +25 -0
  49. data/lib/webspicy/tester/reporter/file_summary.rb +43 -0
  50. data/lib/webspicy/tester/reporter/progress.rb +30 -0
  51. data/lib/webspicy/tester/reporter/summary.rb +63 -0
  52. data/lib/webspicy/tester/result.rb +142 -0
  53. data/lib/webspicy/tester/result/assert_met.rb +29 -0
  54. data/lib/webspicy/tester/result/check.rb +33 -0
  55. data/lib/webspicy/tester/result/errcondition_met.rb +29 -0
  56. data/lib/webspicy/tester/result/error_schema_met.rb +24 -0
  57. data/lib/webspicy/tester/result/invocation_succeeded.rb +13 -0
  58. data/lib/webspicy/tester/result/output_schema_met.rb +24 -0
  59. data/lib/webspicy/tester/result/postcondition_met.rb +29 -0
  60. data/lib/webspicy/tester/result/response_header_met.rb +43 -0
  61. data/lib/webspicy/tester/result/response_status_met.rb +25 -0
  62. data/lib/webspicy/version.rb +2 -2
  63. data/lib/webspicy/web.rb +4 -0
  64. data/lib/webspicy/web/client.rb +15 -0
  65. data/lib/webspicy/{tester → web}/client/http_client.rb +34 -14
  66. data/lib/webspicy/{tester → web}/client/rack_test_client.rb +3 -3
  67. data/lib/webspicy/{tester → web}/client/support.rb +2 -2
  68. data/lib/webspicy/web/invocation.rb +65 -0
  69. data/lib/webspicy/web/mocker.rb +90 -0
  70. data/lib/webspicy/{mocker → web/mocker}/config.ru +3 -2
  71. data/lib/webspicy/{openapi.rb → web/openapi.rb} +0 -0
  72. data/lib/webspicy/web/openapi/generator.rb +129 -0
  73. data/spec/unit/configuration/scope/test_each_service.rb +2 -2
  74. data/spec/unit/configuration/scope/test_each_specification.rb +7 -7
  75. data/spec/unit/specification/{precondition → pre}/test_global_request_headers.rb +9 -4
  76. data/spec/unit/specification/test_condition.rb +44 -0
  77. data/spec/unit/support/hooks/test_fire_after_each.rb +53 -0
  78. data/spec/unit/{tester/client/test_around.rb → support/hooks/test_fire_around.rb} +15 -10
  79. data/spec/unit/support/hooks/test_fire_before_each.rb +53 -0
  80. data/spec/unit/support/world/fixtures/array.json +8 -0
  81. data/spec/unit/support/world/fixtures/queue.rb +1 -0
  82. data/spec/unit/support/world/fixtures/single.json +11 -0
  83. data/spec/unit/support/world/fixtures/yaml.yml +3 -0
  84. data/spec/unit/support/world/test_world.rb +56 -0
  85. data/spec/unit/test_configuration.rb +50 -1
  86. data/spec/unit/tester/fakeses/test_email.rb +40 -0
  87. data/spec/unit/tester/test_asserter.rb +198 -3
  88. data/spec/unit/tester/test_assertions.rb +8 -6
  89. data/spec/unit/web/mocker/test_mocker.rb +35 -0
  90. data/spec/unit/web/openapi/test_generator.rb +31 -0
  91. metadata +90 -67
  92. data/examples/restful/Gemfile +0 -5
  93. data/examples/restful/Gemfile.lock +0 -105
  94. data/examples/restful/Rakefile +0 -25
  95. data/examples/restful/app.rb +0 -180
  96. data/examples/restful/webspicy/config.rb +0 -23
  97. data/examples/restful/webspicy/rack.rb +0 -7
  98. data/examples/restful/webspicy/real.rb +0 -8
  99. data/examples/restful/webspicy/schema.fio +0 -20
  100. data/examples/restful/webspicy/support/must_be_an_admin.rb +0 -20
  101. data/examples/restful/webspicy/support/must_be_authenticated.rb +0 -48
  102. data/examples/restful/webspicy/support/todo_removed.rb +0 -18
  103. data/examples/restful/webspicy/todo/deleteTodo.yml +0 -52
  104. data/examples/restful/webspicy/todo/getTodo.yml +0 -50
  105. data/examples/restful/webspicy/todo/getTodoSingleServiceFormat.yml +0 -46
  106. data/examples/restful/webspicy/todo/getTodos.yml +0 -36
  107. data/examples/restful/webspicy/todo/options.yml +0 -32
  108. data/examples/restful/webspicy/todo/patchTodo.yml +0 -66
  109. data/examples/restful/webspicy/todo/postCsv.yml +0 -43
  110. data/examples/restful/webspicy/todo/postFile.yml +0 -40
  111. data/examples/restful/webspicy/todo/postTodos.yml +0 -51
  112. data/examples/restful/webspicy/todo/putTodo.yml +0 -65
  113. data/examples/restful/webspicy/todo/todos.csv +0 -4
  114. data/examples/single_spec/spec.yml +0 -59
  115. data/examples/website/config.rb +0 -2
  116. data/examples/website/schema.fio +0 -1
  117. data/examples/website/specification/get-http.yml +0 -30
  118. data/examples/website/specification/get-https.yml +0 -30
  119. data/lib/webspicy/checker.rb +0 -25
  120. data/lib/webspicy/mocker.rb +0 -88
  121. data/lib/webspicy/openapi/generator.rb +0 -127
  122. data/lib/webspicy/tester/rspec_asserter.rb +0 -108
  123. data/lib/webspicy/tester/rspec_matchers.rb +0 -104
  124. data/spec/unit/mocker/test_mocker.rb +0 -32
  125. data/spec/unit/openapi/test_generator.rb +0 -28
@@ -78,14 +78,19 @@ module Webspicy
78
78
  expected = id
79
79
  id, path = path, ''
80
80
  end
81
- unless @assertions.idFD(@target, path, id, expected)
82
- _! "Expected #{_s(@target, path)} to meet FD #{expected.inspect}"
81
+ element = @assertions.element_with_id(@target, path, id)
82
+ unless element
83
+ _! "Expected an element with id #{id} to contain the key(s) and value(s) #{expected}, but there is no element with that id"
84
+ end
85
+
86
+ unless @assertions.idFD(element, expected)
87
+ _! "Expected #{_s(@target, path)} to contain the key(s) and value(s) #{expected}"
83
88
  end
84
89
  end
85
90
 
86
91
  def pathFD(path, expected)
87
92
  unless @assertions.pathFD(@target, path, expected)
88
- _! "#{expected.inspect} vs. #{_s(@target, path)}"
93
+ _! "Expected #{_s(@target, path)} to contain the key(s) and value(s) #{expected}"
89
94
  end
90
95
  end
91
96
 
@@ -121,7 +126,7 @@ module Webspicy
121
126
  end
122
127
 
123
128
  def _!(msg)
124
- raise msg
129
+ raise Failure, msg
125
130
  end
126
131
 
127
132
  end # class Asserter
@@ -63,16 +63,15 @@ module Webspicy
63
63
  (ids.to_set & expected.to_set).empty?
64
64
  end
65
65
 
66
- def idFD(target, path, id, expected = NO_ARG)
67
- if expected == NO_ARG
68
- expected = id
69
- id, path = path, ''
70
- end
66
+ def element_with_id(target, path, id)
71
67
  target = extract_path(target, path)
72
- found = an_array(target).find{|t| t[:id] == id }
73
- expected.keys.all?{|k|
74
- value_equal(expected[k], found[k])
75
- }
68
+ an_array(target).find { |t| t[:id] == id }
69
+ end
70
+
71
+ def idFD(element, expected)
72
+ expected.keys.all? do |k|
73
+ value_equal(expected[k], element[k])
74
+ end
76
75
  end
77
76
 
78
77
  def pathFD(target, path, expected)
@@ -1,63 +1,17 @@
1
1
  module Webspicy
2
2
  class Tester
3
3
  class Client
4
+ extend Forwardable
4
5
 
5
6
  def initialize(scope)
6
7
  @scope = scope
7
8
  end
8
9
  attr_reader :scope
9
10
 
10
- def config
11
- scope.config
12
- end
13
-
14
- def around(*args, &bl)
15
- args << self
16
- ls = config.listeners(:around_each)
17
- if ls.size == 0
18
- bl.call
19
- elsif ls.size > 1
20
- _around(ls.first, ls[1..-1], args, &bl)
21
- else
22
- ls.first.call(*args, &bl)
23
- end
24
- end
25
-
26
- def _around(head, tail, args, &bl)
27
- head.call(*args) do
28
- if tail.empty?
29
- bl.call
30
- else
31
- _around(tail.first, tail[1..-1], args, &bl)
32
- end
33
- end
34
- end
35
- private :_around
36
-
37
- def instrument(*args, &bl)
38
- args << self
39
- config.listeners(:instrument).each do |i|
40
- i.call(*args, &bl)
41
- end
42
- end
43
-
44
- def before(*args, &bl)
45
- args << self
46
- config.listeners(:before_each).each do |beach|
47
- beach.call(*args, &bl)
48
- end
49
- end
50
-
51
- def after(*args, &bl)
52
- args << self
53
- config.listeners(:after_each).each do |aeach|
54
- aeach.call(*args, &bl)
55
- end
56
- end
11
+ def_delegators :@scope, *[
12
+ :config
13
+ ]
57
14
 
58
15
  end # class Client
59
16
  end # class Tester
60
17
  end # module Webspicy
61
- require_relative 'client/support'
62
- require_relative 'client/http_client'
63
- require_relative 'client/rack_test_client'
@@ -0,0 +1,6 @@
1
+ module Webspicy
2
+ class Tester
3
+ class Failure < Exception
4
+ end # class Failure
5
+ end # class Tester
6
+ end # module Webspicy
@@ -0,0 +1,41 @@
1
+ require 'base64'
2
+
3
+ module Webspicy
4
+ class Tester
5
+ class Fakeses
6
+ include Webspicy::Support::World::Item
7
+
8
+ DEFAULT_OPTIONS = {
9
+ endpoint: "http://fake-ses/_/api"
10
+ }
11
+
12
+ def initialize(options = {})
13
+ @options = DEFAULT_OPTIONS.merge(options)
14
+ end
15
+ attr_reader :options
16
+
17
+ def endpoint
18
+ options[:endpoint]
19
+ end
20
+
21
+ def clear!
22
+ res = HTTP.post("#{endpoint}/reset")
23
+ end
24
+
25
+ def emails
26
+ res = HTTP.get("#{endpoint}/emails")
27
+ JSON.parse(res.body).map{|data| Email.new(data) }
28
+ end
29
+
30
+ def emails_count
31
+ emails.length
32
+ end
33
+
34
+ def last_email
35
+ emails.last
36
+ end
37
+
38
+ end # class Fakeses
39
+ end # class Tester
40
+ end # module Websipcy
41
+ require_relative 'fakeses/email'
@@ -0,0 +1,38 @@
1
+ module Webspicy
2
+ class Tester
3
+ class Fakeses
4
+ class Email
5
+
6
+ def initialize(data)
7
+ @data = data
8
+ end
9
+ attr_reader :data
10
+
11
+ def from
12
+ data["body"]["Source"]
13
+ end
14
+
15
+ def to
16
+ data["body"]
17
+ .each_pair
18
+ .select{|(k,v)|
19
+ k =~ /Destinations.member/
20
+ }
21
+ .map{|(k,v)| v }
22
+ end
23
+
24
+ def subject
25
+ rx = /^Subject:\s*(.*)$/
26
+ raw_data
27
+ .each_line
28
+ .find{|l| l =~ rx }[rx, 1]
29
+ end
30
+
31
+ def raw_data
32
+ @raw_data ||= Base64.decode64(data["body"]["RawMessage.Data"])
33
+ end
34
+
35
+ end # class Email
36
+ end # class Fakeses
37
+ end # class Tester
38
+ end # module Webspicy
@@ -0,0 +1,39 @@
1
+ module Webspicy
2
+ class Tester
3
+ class Fakesmtp
4
+ include Webspicy::Support::World::Item
5
+
6
+ DEFAULT_OPTIONS = {
7
+ endpoint: "http://fakesmtp"
8
+ }
9
+
10
+ def initialize(options = {})
11
+ @options = DEFAULT_OPTIONS.merge(options)
12
+ end
13
+ attr_reader :options
14
+
15
+ def endpoint
16
+ options[:endpoint]
17
+ end
18
+
19
+ def clear!
20
+ res = HTTP.delete("#{endpoint}/emails")
21
+ end
22
+
23
+ def emails
24
+ res = HTTP.get("#{endpoint}/emails")
25
+ JSON.parse(res.body).map{|data| Email.new(data) }
26
+ end
27
+
28
+ def emails_count
29
+ emails.length
30
+ end
31
+
32
+ def last_email
33
+ emails.last
34
+ end
35
+
36
+ end # class Fakesmtp
37
+ end # class Tester
38
+ end # module Webspicy
39
+ require_relative 'fakesmtp/email'
@@ -0,0 +1,27 @@
1
+ module Webspicy
2
+ class Tester
3
+ class Fakesmtp
4
+ class Email
5
+
6
+ def initialize(data)
7
+ @data = data
8
+ end
9
+ attr_reader :data
10
+
11
+ def from
12
+ @from ||= data["headerLines"]
13
+ .select{|h| h["key"] == "from" }
14
+ .map{|h| h["line"][/From:\s*(.*)$/, 1] }
15
+ .first
16
+ end
17
+
18
+ def to
19
+ @to ||= data["headerLines"]
20
+ .select{|h| h["key"] == "to" }
21
+ .map{|h| h["line"][/To:\s*(.*)$/, 1] }
22
+ end
23
+
24
+ end # class Email
25
+ end # class Fakesmtp
26
+ end # class Tester
27
+ end # module Webspicy
@@ -0,0 +1,22 @@
1
+ module Webspicy
2
+ class Tester
3
+ class FileChecker < Tester
4
+
5
+ def default_reporter
6
+ @reporter = Reporter::Composite.new
7
+ @reporter << Reporter::FileProgress.new
8
+ @reporter << Reporter::Exceptions.new
9
+ @reporter << Reporter::FileSummary.new
10
+ @reporter << Reporter::ErrorCount.new
11
+ end
12
+
13
+ def run_scope
14
+ scope.each_specification_file do |spec_file|
15
+ @specification = load_specification(spec_file)
16
+ reporter.spec_file_done
17
+ end
18
+ end
19
+
20
+ end # class FileChecker
21
+ end # class Tester
22
+ end # module Webspicy
@@ -1,218 +1,37 @@
1
1
  module Webspicy
2
2
  class Tester
3
3
  class Invocation
4
+ extend Forwardable
4
5
 
5
6
  def initialize(test_case, response, client)
6
7
  @test_case = test_case
7
8
  @response = response
8
9
  @client = client
9
10
  end
10
-
11
11
  attr_reader :test_case, :response, :client
12
12
 
13
- def service
14
- test_case.service
15
- end
16
-
17
- def rspec_assert!(rspec)
18
- RSpecAsserter.new(rspec, self).assert!
19
- end
20
-
21
- def errors
22
- @errors ||= begin
23
- errs = [
24
- [:expected_status_unmet, true],
25
- [:expected_content_type_unmet, !test_case.is_expected_status?(204)],
26
- [:expected_headers_unmet, test_case.has_expected_headers?],
27
- [:expected_schema_unmet, !test_case.is_expected_status?(204)],
28
- [:assertions_unmet, test_case.has_assertions?],
29
- [:postconditions_unmet, test_case.service.has_postconditions? && !test_case.counterexample?],
30
- [:expected_error_unmet, test_case.has_expected_error?]
31
- ].map do |(expectation,only_if)|
32
- next unless only_if
33
- begin
34
- self.send(expectation)
35
- rescue => ex
36
- ex.message
37
- end
38
- end
39
- errs.compact
40
- end
41
- end
42
-
43
- def has_error?
44
- !errors.empty?
45
- end
46
-
47
- ### Getters on response
48
-
49
- def response_code
50
- code = response.status
51
- code = code.code unless code.is_a?(Integer)
52
- code
53
- end
54
-
55
- ### Query methods
56
-
57
- def done?
58
- !response.nil?
59
- end
60
-
61
- def is_expected_success?
62
- test_case.expected_status.to_i >= 200 && test_case.expected_status.to_i < 300
63
- end
64
-
65
- def is_success?
66
- response_code >= 200 && response_code < 300
67
- end
68
-
69
- def is_empty_response?
70
- response_code == 204
71
- end
72
-
73
- def is_redirect?
74
- response_code >= 300 && response_code < 400
75
- end
76
-
77
- ### Check of HTTP status
78
-
79
- def expected_status_unmet
80
- expected = test_case.expected_status
81
- got = response.status
82
- expected === got ? nil : "[status] #{expected} !== #{got}"
83
- end
84
-
85
- def meets_expected_status?
86
- expected_status_unmet.nil?
87
- end
88
-
89
- ### Check of the expected output type
90
-
91
- def expected_content_type_unmet
92
- ect = test_case.expected_content_type
93
- return nil unless ect
94
- got = response.content_type
95
- got = got.mime_type if got.respond_to?(:mime_type)
96
- if ect.nil?
97
- got.nil? ? nil : "[content type] #{ect} != #{got}"
98
- else
99
- got.to_s.start_with?(ect.to_s) ? nil : "[content type] #{ect} != #{got}"
100
- end
101
- end
102
-
103
- def meets_expected_content_type?
104
- expected_content_type_unmet.nil?
105
- end
106
-
107
- ### Check of output schema
108
-
109
- def expected_schema_unmet
110
- if is_empty_response?
111
- body = response.body.to_s.strip
112
- body.empty? ? nil : "[body] empty vs. #{body}"
113
- elsif is_redirect?
114
- else
115
- case dressed_body
116
- when Finitio::TypeError
117
- rc = dressed_body.root_cause
118
- "#{rc.message} (#{rc.location ? rc.location : 'unknown location'})"
119
- when StandardError
120
- dressed_body.message
121
- else nil
122
- end
123
- end
124
- end
125
-
126
- def meets_expected_schema?
127
- expected_schema_unmet.nil?
128
- end
129
-
130
- ### Check of assertions
131
-
132
- def assertions_unmet
133
- unmet = []
134
- asserter = Tester::Asserter.new(dressed_body)
135
- test_case.assert.each do |assert|
136
- begin
137
- asserter.instance_eval(assert)
138
- rescue => ex
139
- unmet << ex.message
140
- end
141
- end
142
- unmet.empty? ? nil : unmet.join("\n")
143
- end
144
-
145
- def value_equal(exp, got)
146
- case exp
147
- when Hash
148
- exp.all?{|(k,v)|
149
- got[k] == v
150
- }
151
- else
152
- exp == got
153
- end
154
- end
155
-
156
- ### Check of expected error message
157
-
158
- def expected_error_unmet
159
- expected = test_case.expected_error
160
- case test_case.expected_content_type
161
- when %r{json}
162
- got = meets_expected_schema? ? dressed_body[:description] : response.body
163
- expected == got ? nil : "[error message] `#{expected}` vs. `#{got}`"
164
- else
165
- dressed_body.include?(expected) ? nil : "[error message] `#{expected}` not found" unless expected.nil?
166
- end
167
- end
168
-
169
- ### Check of expected headers
170
-
171
- def expected_headers_unmet
172
- unmet = []
173
- expected = test_case.expected_headers
174
- expected.each_pair do |k,v|
175
- got = response.headers[k]
176
- unmet << "[headers] #{v} expected for #{k}, got #{got}" unless (got == v)
177
- end
178
- unmet.empty? ? nil : unmet.join("\n")
179
- end
13
+ def_delegators :@client, *[
14
+ :config
15
+ ]
180
16
 
181
- ### Check of postconditions
17
+ def_delegators :@test_case, *[
18
+ :scope,
19
+ :specification,
20
+ :service
21
+ ]
182
22
 
183
- def postconditions_unmet
184
- failures = service.postconditions.map{|post|
185
- post.check(self)
186
- }.compact
187
- failures.empty? ? nil : failures.join("\n")
23
+ def raw_output
24
+ raise NotImplementedError
188
25
  end
189
26
 
190
- def loaded_body
191
- case test_case.expected_content_type
192
- when %r{json}
193
- raise "Body empty while expected" if response.body.to_s.empty?
194
- @loaded_body ||= ::JSON.parse(response.body)
195
- else
196
- response.body.to_s
197
- end
27
+ def loaded_output
28
+ raise NotImplementedError
198
29
  end
199
30
 
200
- def dressed_body
201
- @dressed_body ||= case test_case.expected_content_type
202
- when %r{json}
203
- schema = is_expected_success? ? service.output_schema : service.error_schema
204
- begin
205
- schema.dress(loaded_body)
206
- rescue Finitio::TypeError => ex
207
- ex
208
- end
209
- else
210
- loaded_body
211
- end
31
+ def output
32
+ raise NotImplementedError
212
33
  end
213
34
 
214
35
  end # class Invocation
215
36
  end # class Tester
216
37
  end # module Webspicy
217
- require_relative 'rspec_matchers'
218
- require_relative 'rspec_asserter'