webspicy 0.16.1 → 0.19.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -5
  3. data/bin/webspicy +6 -1
  4. data/doc/1-black-box-scene.md +109 -0
  5. data/doc/2-black-box-testing.md +27 -0
  6. data/doc/3-specification-importance.md +41 -0
  7. data/doc/4-sequence-diagram.md +82 -0
  8. data/lib/webspicy.rb +15 -4
  9. data/lib/webspicy/configuration.rb +57 -8
  10. data/lib/webspicy/configuration/scope.rb +22 -16
  11. data/lib/webspicy/formaldoc.fio +3 -1
  12. data/lib/webspicy/specification.rb +12 -10
  13. data/lib/webspicy/specification/condition.rb +23 -0
  14. data/lib/webspicy/specification/errcondition.rb +17 -0
  15. data/lib/webspicy/specification/postcondition.rb +1 -0
  16. data/lib/webspicy/specification/precondition.rb +1 -0
  17. data/lib/webspicy/specification/precondition/robust_to_invalid_input.rb +1 -1
  18. data/lib/webspicy/specification/service.rb +38 -25
  19. data/lib/webspicy/specification/test_case.rb +10 -17
  20. data/lib/webspicy/support.rb +22 -0
  21. data/lib/webspicy/support/data_object.rb +25 -0
  22. data/lib/webspicy/support/hooks.rb +65 -0
  23. data/lib/webspicy/support/world.rb +47 -0
  24. data/lib/webspicy/tester.rb +120 -61
  25. data/lib/webspicy/tester/asserter.rb +9 -4
  26. data/lib/webspicy/tester/assertions.rb +8 -9
  27. data/lib/webspicy/tester/client.rb +22 -42
  28. data/lib/webspicy/tester/failure.rb +6 -0
  29. data/lib/webspicy/tester/file_checker.rb +22 -0
  30. data/lib/webspicy/tester/invocation.rb +15 -196
  31. data/lib/webspicy/tester/reporter.rb +85 -0
  32. data/lib/webspicy/tester/reporter/composite.rb +38 -0
  33. data/lib/webspicy/tester/reporter/documentation.rb +67 -0
  34. data/lib/webspicy/tester/reporter/error_count.rb +25 -0
  35. data/lib/webspicy/tester/reporter/exceptions.rb +60 -0
  36. data/lib/webspicy/tester/reporter/file_progress.rb +22 -0
  37. data/lib/webspicy/tester/reporter/file_summary.rb +42 -0
  38. data/lib/webspicy/tester/reporter/progress.rb +28 -0
  39. data/lib/webspicy/tester/reporter/summary.rb +62 -0
  40. data/lib/webspicy/tester/result.rb +139 -0
  41. data/lib/webspicy/tester/result/assert_met.rb +29 -0
  42. data/lib/webspicy/tester/result/check.rb +33 -0
  43. data/lib/webspicy/tester/result/errcondition_met.rb +29 -0
  44. data/lib/webspicy/tester/result/error_schema_met.rb +24 -0
  45. data/lib/webspicy/tester/result/output_schema_met.rb +24 -0
  46. data/lib/webspicy/tester/result/postcondition_met.rb +29 -0
  47. data/lib/webspicy/tester/result/response_header_met.rb +43 -0
  48. data/lib/webspicy/tester/result/response_status_met.rb +25 -0
  49. data/lib/webspicy/version.rb +2 -2
  50. data/lib/webspicy/web.rb +4 -0
  51. data/lib/webspicy/web/client.rb +15 -0
  52. data/lib/webspicy/{tester → web}/client/http_client.rb +34 -14
  53. data/lib/webspicy/{tester → web}/client/rack_test_client.rb +3 -3
  54. data/lib/webspicy/{tester → web}/client/support.rb +2 -2
  55. data/lib/webspicy/web/invocation.rb +65 -0
  56. data/lib/webspicy/web/mocker.rb +90 -0
  57. data/lib/webspicy/web/mocker/config.ru +6 -0
  58. data/lib/webspicy/{openapi.rb → web/openapi.rb} +0 -0
  59. data/lib/webspicy/web/openapi/generator.rb +129 -0
  60. data/spec/unit/configuration/scope/test_each_service.rb +2 -2
  61. data/spec/unit/configuration/scope/test_each_specification.rb +7 -7
  62. data/spec/unit/specification/test_condition.rb +26 -0
  63. data/spec/unit/support/hooks/test_fire_after_each.rb +53 -0
  64. data/spec/unit/{tester/client/test_around.rb → support/hooks/test_fire_around.rb} +15 -10
  65. data/spec/unit/support/hooks/test_fire_before_each.rb +53 -0
  66. data/spec/unit/support/world/fixtures/array.json +8 -0
  67. data/spec/unit/support/world/fixtures/queue.rb +1 -0
  68. data/spec/unit/support/world/fixtures/single.json +11 -0
  69. data/spec/unit/support/world/fixtures/yaml.yml +3 -0
  70. data/spec/unit/support/world/test_world.rb +56 -0
  71. data/spec/unit/test_configuration.rb +50 -1
  72. data/spec/unit/tester/test_asserter.rb +198 -3
  73. data/spec/unit/tester/test_assertions.rb +8 -6
  74. data/spec/unit/web/mocker/test_mocker.rb +35 -0
  75. data/spec/unit/web/openapi/test_generator.rb +31 -0
  76. metadata +72 -61
  77. data/examples/restful/Gemfile +0 -5
  78. data/examples/restful/Gemfile.lock +0 -105
  79. data/examples/restful/Rakefile +0 -25
  80. data/examples/restful/app.rb +0 -180
  81. data/examples/restful/webspicy/config.rb +0 -23
  82. data/examples/restful/webspicy/rack.rb +0 -7
  83. data/examples/restful/webspicy/real.rb +0 -8
  84. data/examples/restful/webspicy/schema.fio +0 -20
  85. data/examples/restful/webspicy/support/must_be_an_admin.rb +0 -20
  86. data/examples/restful/webspicy/support/must_be_authenticated.rb +0 -48
  87. data/examples/restful/webspicy/support/todo_removed.rb +0 -18
  88. data/examples/restful/webspicy/todo/deleteTodo.yml +0 -52
  89. data/examples/restful/webspicy/todo/getTodo.yml +0 -50
  90. data/examples/restful/webspicy/todo/getTodoSingleServiceFormat.yml +0 -46
  91. data/examples/restful/webspicy/todo/getTodos.yml +0 -36
  92. data/examples/restful/webspicy/todo/options.yml +0 -32
  93. data/examples/restful/webspicy/todo/patchTodo.yml +0 -66
  94. data/examples/restful/webspicy/todo/postCsv.yml +0 -43
  95. data/examples/restful/webspicy/todo/postFile.yml +0 -40
  96. data/examples/restful/webspicy/todo/postTodos.yml +0 -51
  97. data/examples/restful/webspicy/todo/putTodo.yml +0 -65
  98. data/examples/restful/webspicy/todo/todos.csv +0 -4
  99. data/examples/single_spec/spec.yml +0 -59
  100. data/examples/website/config.rb +0 -2
  101. data/examples/website/schema.fio +0 -1
  102. data/examples/website/specification/get-http.yml +0 -34
  103. data/examples/website/specification/get-https.yml +0 -34
  104. data/lib/webspicy/checker.rb +0 -25
  105. data/lib/webspicy/mocker.rb +0 -88
  106. data/lib/webspicy/openapi/generator.rb +0 -127
  107. data/lib/webspicy/tester/rspec_asserter.rb +0 -108
  108. data/lib/webspicy/tester/rspec_matchers.rb +0 -104
  109. data/spec/unit/mocker/test_mocker.rb +0 -32
  110. data/spec/unit/openapi/test_generator.rb +0 -28
@@ -7,38 +7,33 @@ module Webspicy
7
7
  end
8
8
  attr_reader :config
9
9
 
10
- def preconditions
11
- config.preconditions
12
- end
13
-
14
- def postconditions
15
- config.postconditions
16
- end
17
-
18
10
  ###
19
11
  ### Eachers -- Allow navigating the web service definitions
20
12
  ###
21
13
 
22
14
  # Yields each specification file in the current scope
23
- def each_specification_file(&bl)
24
- return enum_for(:each_specification_file) unless block_given?
25
- _each_specification_file(config, &bl)
15
+ def each_specification_file(apply_filter = true, &bl)
16
+ return enum_for(:each_specification_file, apply_filter) unless block_given?
17
+ _each_specification_file(config, apply_filter, &bl)
26
18
  end
27
19
 
28
20
  # Recursive implementation of `each_specification_file` for each
29
21
  # folder in the configuration.
30
- def _each_specification_file(config)
22
+ def _each_specification_file(config, apply_filter = true)
31
23
  folder = config.folder
32
- folder.glob("**/*.yml").select(&to_filter_proc(config.file_filter)).each do |file|
24
+ world = config.folder/"world"
25
+ fs = folder.glob("**/*.yml").reject{|f| f.to_s.start_with?(world.to_s) }
26
+ fs = fs.select(&to_filter_proc(config.file_filter)) if apply_filter
27
+ fs.each do |file|
33
28
  yield file, folder
34
29
  end
35
30
  end
36
31
  private :_each_specification_file
37
32
 
38
33
  # Yields each specification in the current scope in turn.
39
- def each_specification(&bl)
40
- return enum_for(:each_specification) unless block_given?
41
- each_specification_file do |file, folder|
34
+ def each_specification(apply_filter = true, &bl)
35
+ return enum_for(:each_specification, apply_filter) unless block_given?
36
+ each_specification_file(apply_filter) do |file, folder|
42
37
  yield Webspicy.specification(file.load, file, self)
43
38
  end
44
39
  end
@@ -76,6 +71,17 @@ module Webspicy
76
71
  each_generated_counterexamples(service, &bl)
77
72
  end
78
73
 
74
+ def find_test_case(method, url)
75
+ each_specification(false) do |spec|
76
+ next unless spec.url == url
77
+ spec.services.each do |service|
78
+ next unless service.method == method
79
+ return service.examples.first
80
+ end
81
+ end
82
+ nil
83
+ end
84
+
79
85
  ###
80
86
  ### Schemas -- For parsing input and output data schemas found in
81
87
  ### web service definitions
@@ -25,11 +25,12 @@ Specification = .Webspicy::Specification
25
25
  }
26
26
  <singleservice> {
27
27
  name :? String
28
- url : String
28
+ url : String
29
29
  method : Method
30
30
  description : String
31
31
  preconditions :? [String]|String
32
32
  postconditions :? [String]|String
33
+ errconditions :? [String]|String
33
34
  input_schema : Schema
34
35
  output_schema : Schema
35
36
  error_schema : Schema
@@ -45,6 +46,7 @@ Service =
45
46
  description : String
46
47
  preconditions :? [String]|String
47
48
  postconditions :? [String]|String
49
+ errconditions :? [String]|String
48
50
  input_schema : Schema
49
51
  output_schema : Schema
50
52
  error_schema : Schema
@@ -1,11 +1,13 @@
1
1
  module Webspicy
2
2
  class Specification
3
+ include Support::DataObject
3
4
 
4
5
  def initialize(raw, location = nil)
5
- @raw = raw
6
+ super(raw)
6
7
  @location = location
7
8
  bind_services
8
9
  end
10
+ attr_accessor :config
9
11
  attr_reader :location
10
12
 
11
13
  def self.info(raw)
@@ -14,11 +16,13 @@ module Webspicy
14
16
 
15
17
  def self.singleservice(raw)
16
18
  converted = {
17
- name: raw.has_key?(:name) ? raw[:name] : "Unamed specification",
19
+ name: raw[:name] || "Unamed specification",
18
20
  url: raw[:url],
19
- services: [raw.reject{|k, _| k == :name || k == :url}]
21
+ services: [
22
+ Webspicy.service(raw.reject{|k| k==:url or k==:name }, Webspicy.current_scope)
23
+ ]
20
24
  }
21
- self.info(raw)
25
+ info(converted)
22
26
  end
23
27
 
24
28
  def located_at!(location)
@@ -44,7 +48,7 @@ module Webspicy
44
48
  end
45
49
 
46
50
  def services
47
- @raw[:services]
51
+ @raw[:services] || []
48
52
  end
49
53
 
50
54
  def url_placeholders
@@ -60,10 +64,6 @@ module Webspicy
60
64
  [ url, rest ]
61
65
  end
62
66
 
63
- def to_info
64
- @raw
65
- end
66
-
67
67
  def to_singleservice
68
68
  raise NotImplementedError
69
69
  end
@@ -85,7 +85,7 @@ module Webspicy
85
85
  end
86
86
 
87
87
  def bind_services
88
- (@raw[:services] ||= []).each do |s|
88
+ services.each do |s|
89
89
  s.specification = self
90
90
  end
91
91
  end
@@ -93,7 +93,9 @@ module Webspicy
93
93
  end # class Specification
94
94
  end # module Webspicy
95
95
  require_relative 'specification/service'
96
+ require_relative 'specification/condition'
96
97
  require_relative 'specification/precondition'
97
98
  require_relative 'specification/postcondition'
99
+ require_relative 'specification/errcondition'
98
100
  require_relative 'specification/test_case'
99
101
  require_relative 'specification/file_upload'
@@ -0,0 +1,23 @@
1
+ module Webspicy
2
+ class Specification
3
+ module Condition
4
+
5
+ MATCH_ALL = "__all__"
6
+
7
+ attr_accessor :matching_description
8
+
9
+ def to_s
10
+ if matching_description == MATCH_ALL
11
+ self.class.name
12
+ else
13
+ matching_description
14
+ end
15
+ end
16
+
17
+ def sooner_or_later(*args, &bl)
18
+ Webspicy::Support.sooner_or_later(*args, &bl)
19
+ end
20
+
21
+ end # module Condition
22
+ end # class Specification
23
+ end # module Webspicy
@@ -0,0 +1,17 @@
1
+ module Webspicy
2
+ class Specification
3
+ module Errcondition
4
+ include Condition
5
+
6
+ def self.match(service, descr)
7
+ end
8
+
9
+ def instrument(test_case, client)
10
+ end
11
+
12
+ def check(invocation)
13
+ end
14
+
15
+ end # module Errcondition
16
+ end # module Specification
17
+ end # module Webspicy
@@ -1,6 +1,7 @@
1
1
  module Webspicy
2
2
  class Specification
3
3
  module Postcondition
4
+ include Condition
4
5
 
5
6
  def self.match(service, descr)
6
7
  end
@@ -1,6 +1,7 @@
1
1
  module Webspicy
2
2
  class Specification
3
3
  module Precondition
4
+ include Condition
4
5
 
5
6
  def self.match(service, pre)
6
7
  end
@@ -40,7 +40,7 @@ module Webspicy
40
40
  def empty_input_counterexamples(service, first)
41
41
  placeholders = service.specification.url_placeholders
42
42
  empty_input = first.params.reject{|k| !placeholders.include?(k) }
43
- if !empty_input.empty? && invalid_input?(service, empty_input)
43
+ if invalid_input?(service, empty_input)
44
44
  [first.mutate({
45
45
  :description => "it is robust to an invalid empty input (RobustToInvalidInput)",
46
46
  :dress_params => false,
@@ -1,13 +1,12 @@
1
1
  module Webspicy
2
2
  class Specification
3
3
  class Service
4
+ include Support::DataObject
4
5
 
5
6
  def initialize(raw)
6
- @raw = raw
7
+ super(raw)
7
8
  bind_examples
8
9
  bind_counterexamples
9
- @preconditions = compile_preconditions
10
- @postconditions = compile_postconditions
11
10
  end
12
11
  attr_accessor :specification
13
12
 
@@ -15,6 +14,10 @@ module Webspicy
15
14
  new(raw)
16
15
  end
17
16
 
17
+ def config
18
+ specification.config
19
+ end
20
+
18
21
  def method
19
22
  @raw[:method]
20
23
  end
@@ -24,7 +27,7 @@ module Webspicy
24
27
  end
25
28
 
26
29
  def preconditions
27
- @preconditions
30
+ @preconditions ||= compile_preconditions
28
31
  end
29
32
 
30
33
  def has_preconditions?
@@ -32,23 +35,31 @@ module Webspicy
32
35
  end
33
36
 
34
37
  def postconditions
35
- @postconditions
38
+ @postconditions ||= compile_postconditions
36
39
  end
37
40
 
38
41
  def has_postconditions?
39
42
  !postconditions.empty?
40
43
  end
41
44
 
45
+ def errconditions
46
+ @errconditions ||= compile_errconditions
47
+ end
48
+
49
+ def has_errconditions?
50
+ !errconditions.empty?
51
+ end
52
+
42
53
  def default_example
43
54
  @raw[:default_example]
44
55
  end
45
56
 
46
57
  def examples
47
- @raw[:examples]
58
+ @raw[:examples] || []
48
59
  end
49
60
 
50
61
  def counterexamples
51
- @raw[:counterexamples]
62
+ @raw[:counterexamples] || []
52
63
  end
53
64
 
54
65
  def generated_counterexamples
@@ -76,49 +87,51 @@ module Webspicy
76
87
  input_schema.dress(params)
77
88
  end
78
89
 
79
- def to_info
80
- @raw
81
- end
82
-
83
90
  def to_s
84
91
  "#{method} #{specification.url}"
85
92
  end
86
93
 
87
94
  private
88
95
 
89
- def scope
90
- Webspicy.current_scope
91
- end
92
-
93
96
  def compile_preconditions
94
97
  @raw[:preconditions] = [@raw[:preconditions]] if @raw[:preconditions].is_a?(String)
95
- compile_conditions(@raw[:preconditions] ||= [], scope.preconditions)
98
+ compile_conditions(@raw[:preconditions] ||= [], config.preconditions)
96
99
  end
97
100
 
98
101
  def compile_postconditions
99
102
  @raw[:postconditions] = [@raw[:postconditions]] if @raw[:postconditions].is_a?(String)
100
- compile_conditions(@raw[:postconditions] ||= [], scope.postconditions)
103
+ compile_conditions(@raw[:postconditions] ||= [], config.postconditions)
104
+ end
105
+
106
+ def compile_errconditions
107
+ @raw[:errconditions] = [@raw[:errconditions]] if @raw[:errconditions].is_a?(String)
108
+ compile_conditions(@raw[:errconditions] ||= [], config.errconditions)
101
109
  end
102
110
 
103
111
  def compile_conditions(descriptions, conditions)
104
112
  # Because we want pre & post to be able to match in all cases
105
113
  # we need at least one condition
106
- descriptions = ["all"] if descriptions.empty?
107
- descriptions
108
- .map{|descr|
109
- conditions.map{|c| c.match(self, descr) }.compact
110
- }
111
- .flatten
114
+ descriptions = [Condition::MATCH_ALL] if descriptions.empty?
115
+ conditions.map{|c|
116
+ instance = nil
117
+ descr = descriptions.find do |d|
118
+ instance = c.match(self, d)
119
+ end
120
+ if instance && instance.respond_to?(:matching_description=)
121
+ instance.matching_description = descr
122
+ end
123
+ instance
124
+ }.compact
112
125
  end
113
126
 
114
127
  def bind_examples
115
- (@raw[:examples] ||= []).each do |ex|
128
+ examples.each do |ex|
116
129
  ex.bind(self, false)
117
130
  end
118
131
  end
119
132
 
120
133
  def bind_counterexamples
121
- (@raw[:counterexamples] ||= []).each do |ex|
134
+ counterexamples.each do |ex|
122
135
  ex.bind(self, true)
123
136
  end
124
137
  end
@@ -1,23 +1,25 @@
1
1
  module Webspicy
2
2
  class Specification
3
3
  class TestCase
4
+ include Support::DataObject
4
5
 
5
6
  def initialize(raw)
6
- @raw = raw
7
+ super(raw)
7
8
  @counterexample = nil
8
9
  end
9
10
  attr_reader :service
10
11
  attr_reader :counterexample
11
12
 
12
- attr_accessor :raw
13
- protected :raw, :raw=
14
-
15
13
  def bind(service, counterexample)
16
14
  @service = service
17
15
  @counterexample = counterexample
18
16
  self
19
17
  end
20
18
 
19
+ def example?
20
+ !@counterexample
21
+ end
22
+
21
23
  def counterexample?
22
24
  !!@counterexample
23
25
  end
@@ -87,6 +89,10 @@ module Webspicy
87
89
  expected_status === status
88
90
  end
89
91
 
92
+ def has_expected_status?
93
+ not expected[:status].nil?
94
+ end
95
+
90
96
  def expected_error
91
97
  expected[:error]
92
98
  end
@@ -111,19 +117,6 @@ module Webspicy
111
117
  !assert.empty?
112
118
  end
113
119
 
114
- def to_info
115
- @raw
116
- end
117
-
118
- def instrument(client)
119
- service.preconditions.each do |pre|
120
- pre.instrument(self, client)
121
- end
122
- service.postconditions.each do |post|
123
- post.instrument(self, client) if post.respond_to?(:instrument)
124
- end
125
- end
126
-
127
120
  def mutate(override)
128
121
  m = self.dup
129
122
  m.raw = self.raw.merge(override)
@@ -1,2 +1,24 @@
1
+ module Webspicy
2
+ module Support
3
+
4
+ SORL_OPTS = { max: 5, wait: 0.05, raise: false }
5
+
6
+ def sooner_or_later(opts = nil)
7
+ opts = SORL_OPTS.merge(opts || {})
8
+ left, wait_ms = opts[:max], opts[:wait]
9
+ until (r = yield) || left == 0
10
+ sleep(wait_ms)
11
+ wait_ms, left = wait_ms*2, left - 1
12
+ end
13
+ raise TimeoutError, "Timeout on sooner-or-later" if r.nil? && opts[:raise]
14
+ r
15
+ end
16
+ module_function :sooner_or_later
17
+
18
+ end # module Support
19
+ end # module Webspicy
20
+ require_relative 'support/data_object'
1
21
  require_relative 'support/status_range'
2
22
  require_relative 'support/colorize'
23
+ require_relative 'support/world'
24
+ require_relative 'support/hooks'