webspicy 0.16.1 → 0.19.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -5
  3. data/bin/webspicy +6 -1
  4. data/doc/1-black-box-scene.md +109 -0
  5. data/doc/2-black-box-testing.md +27 -0
  6. data/doc/3-specification-importance.md +41 -0
  7. data/doc/4-sequence-diagram.md +82 -0
  8. data/lib/webspicy.rb +15 -4
  9. data/lib/webspicy/configuration.rb +57 -8
  10. data/lib/webspicy/configuration/scope.rb +22 -16
  11. data/lib/webspicy/formaldoc.fio +3 -1
  12. data/lib/webspicy/specification.rb +12 -10
  13. data/lib/webspicy/specification/condition.rb +23 -0
  14. data/lib/webspicy/specification/errcondition.rb +17 -0
  15. data/lib/webspicy/specification/postcondition.rb +1 -0
  16. data/lib/webspicy/specification/precondition.rb +1 -0
  17. data/lib/webspicy/specification/precondition/robust_to_invalid_input.rb +1 -1
  18. data/lib/webspicy/specification/service.rb +38 -25
  19. data/lib/webspicy/specification/test_case.rb +10 -17
  20. data/lib/webspicy/support.rb +22 -0
  21. data/lib/webspicy/support/data_object.rb +25 -0
  22. data/lib/webspicy/support/hooks.rb +65 -0
  23. data/lib/webspicy/support/world.rb +47 -0
  24. data/lib/webspicy/tester.rb +120 -61
  25. data/lib/webspicy/tester/asserter.rb +9 -4
  26. data/lib/webspicy/tester/assertions.rb +8 -9
  27. data/lib/webspicy/tester/client.rb +22 -42
  28. data/lib/webspicy/tester/failure.rb +6 -0
  29. data/lib/webspicy/tester/file_checker.rb +22 -0
  30. data/lib/webspicy/tester/invocation.rb +15 -196
  31. data/lib/webspicy/tester/reporter.rb +85 -0
  32. data/lib/webspicy/tester/reporter/composite.rb +38 -0
  33. data/lib/webspicy/tester/reporter/documentation.rb +67 -0
  34. data/lib/webspicy/tester/reporter/error_count.rb +25 -0
  35. data/lib/webspicy/tester/reporter/exceptions.rb +60 -0
  36. data/lib/webspicy/tester/reporter/file_progress.rb +22 -0
  37. data/lib/webspicy/tester/reporter/file_summary.rb +42 -0
  38. data/lib/webspicy/tester/reporter/progress.rb +28 -0
  39. data/lib/webspicy/tester/reporter/summary.rb +62 -0
  40. data/lib/webspicy/tester/result.rb +139 -0
  41. data/lib/webspicy/tester/result/assert_met.rb +29 -0
  42. data/lib/webspicy/tester/result/check.rb +33 -0
  43. data/lib/webspicy/tester/result/errcondition_met.rb +29 -0
  44. data/lib/webspicy/tester/result/error_schema_met.rb +24 -0
  45. data/lib/webspicy/tester/result/output_schema_met.rb +24 -0
  46. data/lib/webspicy/tester/result/postcondition_met.rb +29 -0
  47. data/lib/webspicy/tester/result/response_header_met.rb +43 -0
  48. data/lib/webspicy/tester/result/response_status_met.rb +25 -0
  49. data/lib/webspicy/version.rb +2 -2
  50. data/lib/webspicy/web.rb +4 -0
  51. data/lib/webspicy/web/client.rb +15 -0
  52. data/lib/webspicy/{tester → web}/client/http_client.rb +34 -14
  53. data/lib/webspicy/{tester → web}/client/rack_test_client.rb +3 -3
  54. data/lib/webspicy/{tester → web}/client/support.rb +2 -2
  55. data/lib/webspicy/web/invocation.rb +65 -0
  56. data/lib/webspicy/web/mocker.rb +90 -0
  57. data/lib/webspicy/web/mocker/config.ru +6 -0
  58. data/lib/webspicy/{openapi.rb → web/openapi.rb} +0 -0
  59. data/lib/webspicy/web/openapi/generator.rb +129 -0
  60. data/spec/unit/configuration/scope/test_each_service.rb +2 -2
  61. data/spec/unit/configuration/scope/test_each_specification.rb +7 -7
  62. data/spec/unit/specification/test_condition.rb +26 -0
  63. data/spec/unit/support/hooks/test_fire_after_each.rb +53 -0
  64. data/spec/unit/{tester/client/test_around.rb → support/hooks/test_fire_around.rb} +15 -10
  65. data/spec/unit/support/hooks/test_fire_before_each.rb +53 -0
  66. data/spec/unit/support/world/fixtures/array.json +8 -0
  67. data/spec/unit/support/world/fixtures/queue.rb +1 -0
  68. data/spec/unit/support/world/fixtures/single.json +11 -0
  69. data/spec/unit/support/world/fixtures/yaml.yml +3 -0
  70. data/spec/unit/support/world/test_world.rb +56 -0
  71. data/spec/unit/test_configuration.rb +50 -1
  72. data/spec/unit/tester/test_asserter.rb +198 -3
  73. data/spec/unit/tester/test_assertions.rb +8 -6
  74. data/spec/unit/web/mocker/test_mocker.rb +35 -0
  75. data/spec/unit/web/openapi/test_generator.rb +31 -0
  76. metadata +72 -61
  77. data/examples/restful/Gemfile +0 -5
  78. data/examples/restful/Gemfile.lock +0 -105
  79. data/examples/restful/Rakefile +0 -25
  80. data/examples/restful/app.rb +0 -180
  81. data/examples/restful/webspicy/config.rb +0 -23
  82. data/examples/restful/webspicy/rack.rb +0 -7
  83. data/examples/restful/webspicy/real.rb +0 -8
  84. data/examples/restful/webspicy/schema.fio +0 -20
  85. data/examples/restful/webspicy/support/must_be_an_admin.rb +0 -20
  86. data/examples/restful/webspicy/support/must_be_authenticated.rb +0 -48
  87. data/examples/restful/webspicy/support/todo_removed.rb +0 -18
  88. data/examples/restful/webspicy/todo/deleteTodo.yml +0 -52
  89. data/examples/restful/webspicy/todo/getTodo.yml +0 -50
  90. data/examples/restful/webspicy/todo/getTodoSingleServiceFormat.yml +0 -46
  91. data/examples/restful/webspicy/todo/getTodos.yml +0 -36
  92. data/examples/restful/webspicy/todo/options.yml +0 -32
  93. data/examples/restful/webspicy/todo/patchTodo.yml +0 -66
  94. data/examples/restful/webspicy/todo/postCsv.yml +0 -43
  95. data/examples/restful/webspicy/todo/postFile.yml +0 -40
  96. data/examples/restful/webspicy/todo/postTodos.yml +0 -51
  97. data/examples/restful/webspicy/todo/putTodo.yml +0 -65
  98. data/examples/restful/webspicy/todo/todos.csv +0 -4
  99. data/examples/single_spec/spec.yml +0 -59
  100. data/examples/website/config.rb +0 -2
  101. data/examples/website/schema.fio +0 -1
  102. data/examples/website/specification/get-http.yml +0 -34
  103. data/examples/website/specification/get-https.yml +0 -34
  104. data/lib/webspicy/checker.rb +0 -25
  105. data/lib/webspicy/mocker.rb +0 -88
  106. data/lib/webspicy/openapi/generator.rb +0 -127
  107. data/lib/webspicy/tester/rspec_asserter.rb +0 -108
  108. data/lib/webspicy/tester/rspec_matchers.rb +0 -104
  109. data/spec/unit/mocker/test_mocker.rb +0 -32
  110. data/spec/unit/openapi/test_generator.rb +0 -28
@@ -0,0 +1,25 @@
1
+ module Webspicy
2
+ module Support
3
+ module DataObject
4
+
5
+ def initialize(raw)
6
+ @raw = raw
7
+ end
8
+ attr_accessor :raw
9
+ protected :raw, :raw=
10
+
11
+ def method_missing(name, *args, &bl)
12
+ if @raw.has_key?(name) && args.empty? && bl.nil?
13
+ @raw[name]
14
+ else
15
+ super
16
+ end
17
+ end
18
+
19
+ def to_info
20
+ @raw
21
+ end
22
+
23
+ end # module DataObject
24
+ end # module Support
25
+ end # module Webspicy
@@ -0,0 +1,65 @@
1
+ module Webspicy
2
+ module Support
3
+ module Hooks
4
+
5
+ class Object
6
+ def initialize(config)
7
+ @config = config
8
+ end
9
+ attr_accessor :config
10
+ include Hooks
11
+ end
12
+
13
+ def self.for(config)
14
+ Object.new(config)
15
+ end
16
+
17
+ def fire_around(*args, &bl)
18
+ ls = config.listeners(:around_each)
19
+ if ls.size == 0
20
+ bl.call
21
+ elsif ls.size > 1
22
+ _fire_around(ls.first, ls[1..-1], args, &bl)
23
+ else
24
+ ls.first.call(*args, &bl)
25
+ end
26
+ end
27
+
28
+ def _fire_around(head, tail, args, &bl)
29
+ head.call(*args) do
30
+ if tail.empty?
31
+ bl.call
32
+ else
33
+ _fire_around(tail.first, tail[1..-1], args, &bl)
34
+ end
35
+ end
36
+ end
37
+ private :_fire_around
38
+
39
+ def fire_before_each(*args, &bl)
40
+ config.listeners(:before_each).each do |beach|
41
+ beach.call(*args, &bl)
42
+ end
43
+ end
44
+
45
+ def fire_after_each(*args, &bl)
46
+ config.listeners(:after_each).each do |aeach|
47
+ aeach.call(*args, &bl)
48
+ end
49
+ end
50
+
51
+ def fire_before_all(*args, &bl)
52
+ config.listeners(:before_all).each do |beach|
53
+ beach.call(*args, &bl)
54
+ end
55
+ end
56
+
57
+ def fire_after_all(*args, &bl)
58
+ config.listeners(:after_all).each do |aeach|
59
+ aeach.call(*args, &bl)
60
+ end
61
+ end
62
+
63
+ end # module Hooks
64
+ end # module Support
65
+ end # module Webspicy
@@ -0,0 +1,47 @@
1
+ module Webspicy
2
+ module Support
3
+ class World < OpenStruct
4
+
5
+ def initialize(folder, config = nil)
6
+ super({})
7
+ @folder = folder
8
+ @config = config
9
+ end
10
+ attr_reader :folder, :config
11
+
12
+ def method_missing(name, *args, &bl)
13
+ return super if name =~ /=$/
14
+ file = ['json', 'yml', 'yaml', 'rb']
15
+ .map{|e| folder/"#{name}.#{e}" }
16
+ .find{|f| f.file? }
17
+ data = case file && file.ext
18
+ when /json/
19
+ JSON.parse(file.read, object_class: OpenStruct)
20
+ when /ya?ml/
21
+ JSON.parse(file.load.to_json, object_class: OpenStruct)
22
+ when /rb/
23
+ ::Kernel.eval(file.read).tap{|x|
24
+ x.config = self.config if x.is_a?(Item)
25
+ }
26
+ end
27
+ self.send(:"#{name}=", data)
28
+ end
29
+
30
+ def to_data(which)
31
+ case which
32
+ when Hash
33
+ OpenStruct.new(which)
34
+ when Array
35
+ which.map{|x| to_data(x) }
36
+ else
37
+ which
38
+ end
39
+ end
40
+
41
+ module Item
42
+ attr_accessor :config
43
+ end # module Item
44
+
45
+ end # class World
46
+ end # module Support
47
+ end # module Webspicy
@@ -1,89 +1,148 @@
1
1
  module Webspicy
2
2
  class Tester
3
- include Webspicy::Support::Colorize
3
+
4
+ class FailFast < Exception; end
4
5
 
5
6
  def initialize(config)
6
7
  @config = Configuration.dress(config)
7
- @test_suite = load
8
+ @scope = nil
9
+ @hooks = nil
10
+ @client = nil
11
+ @spec_file = nil
12
+ @specification = nil
13
+ @service = nil
14
+ @test_case = nil
15
+ @invocation = nil
16
+ @reporter = default_reporter
8
17
  end
9
- attr_reader :config
18
+ attr_reader :config, :scope, :hooks, :client
19
+ attr_reader :specification, :spec_file
20
+ attr_reader :service, :test_case
21
+ attr_reader :invocation, :result
22
+ attr_reader :reporter
10
23
 
11
- def call(err=$stderr, out=$stdout)
12
- $_rspec_core_load_started_at = nil
13
- options = RSpec::Core::ConfigurationOptions.new(config.rspec_options)
14
- conf = RSpec::Core::Configuration.new
15
- RSpec::Core::Runner.new(options, conf).run(err, out)
24
+ def failfast?
25
+ config.failfast
16
26
  end
17
27
 
18
- # protected
19
-
20
- def load
21
- tester = self
22
- RSpec.reset
23
- rspec_config!
24
- RSpec.describe "" do
25
- before(:all) do
26
- tester.config.listeners(:before_all).each do |l|
27
- l.call(tester.config)
28
- end
29
- end
30
- after(:all) do
31
- tester.config.listeners(:after_all).each do |l|
32
- l.call(tester.config)
33
- end
34
- end
35
- tester.config.each_scope do |scope|
36
- client = scope.get_client
37
- scope.each_specification do |specification|
38
- scope.each_service(specification) do |service|
39
- tester.rspec_service!(self, service, client, scope)
40
- end
41
- end
28
+ def default_reporter
29
+ @reporter = Reporter::Composite.new
30
+ #@reporter << Reporter::Progress.new
31
+ @reporter << Reporter::Documentation.new
32
+ @reporter << Reporter::Exceptions.new
33
+ @reporter << Reporter::Summary.new
34
+ @reporter << Reporter::ErrorCount.new
35
+ end
36
+
37
+ def call
38
+ reporter.init(self)
39
+ begin
40
+ run_config
41
+ rescue FailFast
42
+ end
43
+ reporter.report
44
+ reporter.find(Reporter::ErrorCount).report
45
+ end
46
+
47
+ private
48
+
49
+ def run_config
50
+ config.each_scope do |scope|
51
+ @scope = scope
52
+ @hooks = Support::Hooks.for(scope.config)
53
+ @client = scope.get_client
54
+ reporter.before_all
55
+ @hooks.fire_before_all(@scope, @client)
56
+ reporter.before_all_done
57
+ reporter.before_scope
58
+ run_scope
59
+ reporter.scope_done
60
+ reporter.after_all
61
+ @hooks.fire_after_all(@scope, @client)
62
+ reporter.after_all_done
63
+ end
64
+ end
65
+
66
+ def run_scope
67
+ scope.each_specification_file do |spec_file|
68
+ @specification = load_specification(spec_file)
69
+ if @specification
70
+ reporter.before_specification
71
+ run_specification
72
+ reporter.specification_done
73
+ reporter.spec_file_done
74
+ elsif failfast?
75
+ raise FailFast
42
76
  end
43
77
  end
44
- self
45
78
  end
46
79
 
47
- def rspec_service!(on, service, client, scope)
80
+ def load_specification(spec_file)
81
+ @spec_file = spec_file
82
+ reporter.before_spec_file
83
+ Webspicy.specification(spec_file.load, spec_file, scope)
84
+ rescue *PASSTHROUGH_EXCEPTIONS
85
+ raise
86
+ rescue Exception => e
87
+ reporter.spec_file_error(e)
88
+ nil
89
+ end
90
+
91
+ def run_specification
92
+ scope.each_service(specification) do |service|
93
+ @service = service
94
+ reporter.before_service
95
+ run_service
96
+ reporter.service_done
97
+ end
98
+ end
99
+
100
+ def run_service
48
101
  scope.each_testcase(service) do |test_case|
49
- str = "#{service} #{test_case}"
50
- str = colorize_highlight(str)
51
- on.describe(str) do
52
- include_examples 'a successful test case invocation', client, test_case
53
- end
102
+ @test_case = test_case
103
+ reporter.before_test_case
104
+ run_test_case
105
+ reporter.test_case_done
54
106
  end
55
107
  end
56
108
 
57
- def rspec_config!
58
- RSpec.shared_examples "a successful test case invocation" do |client, test_case|
59
-
60
- around(:each) do |example|
61
- client.around(test_case) do
62
- client.before(test_case)
63
- test_case.instrument(client)
64
- client.instrument(test_case)
65
- @response = client.call(test_case)
66
- @invocation = Tester::Invocation.new(test_case, @response, client)
67
- example.run
68
- client.after(test_case, @invocation)
69
- @invocation
70
- end
71
- end
109
+ def run_test_case
110
+ hooks.fire_around(test_case, client) do
111
+ reporter.before_each
112
+ hooks.fire_before_each(test_case, client)
113
+ reporter.before_each_done
72
114
 
73
- let(:invocation) do
74
- @invocation
75
- end
115
+ reporter.before_instrument
116
+ client.instrument(test_case)
117
+ reporter.instrument_done
76
118
 
77
- it "meets its specification" do
78
- raise "Test not ran" unless invocation.done?
79
- invocation.rspec_assert!(self)
80
- end
119
+ reporter.before_invocation
120
+ @invocation = client.call(test_case)
121
+ reporter.invocation_done
122
+
123
+ reporter.before_assertions
124
+ check_invocation
125
+ reporter.assertions_done
126
+
127
+ reporter.after_each
128
+ hooks.fire_after_each(test_case, @invocation, client)
129
+ reporter.after_each_done
130
+
131
+ raise FailFast if !result.success? and failfast?
81
132
  end
82
133
  end
83
134
 
135
+ def check_invocation
136
+ @result = Result.from(self)
137
+ end
138
+
84
139
  end # class Tester
85
140
  end # module Webspicy
141
+ require_relative 'tester/reporter'
86
142
  require_relative 'tester/client'
87
143
  require_relative 'tester/invocation'
144
+ require_relative 'tester/result'
145
+ require_relative 'tester/failure'
88
146
  require_relative 'tester/assertions'
89
147
  require_relative 'tester/asserter'
148
+ require_relative 'tester/file_checker'
@@ -78,14 +78,19 @@ module Webspicy
78
78
  expected = id
79
79
  id, path = path, ''
80
80
  end
81
- unless @assertions.idFD(@target, path, id, expected)
82
- _! "Expected #{_s(@target, path)} to meet FD #{expected.inspect}"
81
+ element = @assertions.element_with_id(@target, path, id)
82
+ unless element
83
+ _! "Expected an element with id #{id} to contain the key(s) and value(s) #{expected}, but there is no element with that id"
84
+ end
85
+
86
+ unless @assertions.idFD(element, expected)
87
+ _! "Expected #{_s(@target, path)} to contain the key(s) and value(s) #{expected}"
83
88
  end
84
89
  end
85
90
 
86
91
  def pathFD(path, expected)
87
92
  unless @assertions.pathFD(@target, path, expected)
88
- _! "#{expected.inspect} vs. #{_s(@target, path)}"
93
+ _! "Expected #{_s(@target, path)} to contain the key(s) and value(s) #{expected}"
89
94
  end
90
95
  end
91
96
 
@@ -121,7 +126,7 @@ module Webspicy
121
126
  end
122
127
 
123
128
  def _!(msg)
124
- raise msg
129
+ raise Failure, msg
125
130
  end
126
131
 
127
132
  end # class Asserter
@@ -63,16 +63,15 @@ module Webspicy
63
63
  (ids.to_set & expected.to_set).empty?
64
64
  end
65
65
 
66
- def idFD(target, path, id, expected = NO_ARG)
67
- if expected == NO_ARG
68
- expected = id
69
- id, path = path, ''
70
- end
66
+ def element_with_id(target, path, id)
71
67
  target = extract_path(target, path)
72
- found = an_array(target).find{|t| t[:id] == id }
73
- expected.keys.all?{|k|
74
- value_equal(expected[k], found[k])
75
- }
68
+ an_array(target).find { |t| t[:id] == id }
69
+ end
70
+
71
+ def idFD(element, expected)
72
+ expected.keys.all? do |k|
73
+ value_equal(expected[k], element[k])
74
+ end
76
75
  end
77
76
 
78
77
  def pathFD(target, path, expected)
@@ -1,63 +1,43 @@
1
1
  module Webspicy
2
2
  class Tester
3
3
  class Client
4
+ extend Forwardable
4
5
 
5
6
  def initialize(scope)
6
7
  @scope = scope
7
8
  end
8
9
  attr_reader :scope
9
10
 
10
- def config
11
- scope.config
12
- end
11
+ def_delegators :@scope, *[
12
+ :config
13
+ ]
13
14
 
14
- def around(*args, &bl)
15
- args << self
16
- ls = config.listeners(:around_each)
17
- if ls.size == 0
18
- bl.call
19
- elsif ls.size > 1
20
- _around(ls.first, ls[1..-1], args, &bl)
21
- else
22
- ls.first.call(*args, &bl)
15
+ def find_and_call(method, url, mutation)
16
+ unless tc = scope.find_test_case(method, url)
17
+ raise Error, "No such service `#{method} #{url}`"
23
18
  end
19
+ mutated = tc.mutate(mutation)
20
+ instrumented = instrument(mutated)
21
+ call(instrumented)
24
22
  end
25
23
 
26
- def _around(head, tail, args, &bl)
27
- head.call(*args) do
28
- if tail.empty?
29
- bl.call
30
- else
31
- _around(tail.first, tail[1..-1], args, &bl)
32
- end
24
+ def instrument(test_case)
25
+ service = test_case.service
26
+ service.preconditions.each do |pre|
27
+ pre.instrument(test_case, self) if pre.respond_to?(:instrument)
33
28
  end
34
- end
35
- private :_around
36
-
37
- def instrument(*args, &bl)
38
- args << self
29
+ service.postconditions.each do |post|
30
+ post.instrument(test_case, self) if post.respond_to?(:instrument)
31
+ end if test_case.example?
32
+ service.errconditions.each do |post|
33
+ post.instrument(test_case, self) if post.respond_to?(:instrument)
34
+ end if test_case.counterexample?
39
35
  config.listeners(:instrument).each do |i|
40
- i.call(*args, &bl)
41
- end
42
- end
43
-
44
- def before(*args, &bl)
45
- args << self
46
- config.listeners(:before_each).each do |beach|
47
- beach.call(*args, &bl)
48
- end
49
- end
50
-
51
- def after(*args, &bl)
52
- args << self
53
- config.listeners(:after_each).each do |aeach|
54
- aeach.call(*args, &bl)
36
+ i.call(test_case, self)
55
37
  end
38
+ test_case
56
39
  end
57
40
 
58
41
  end # class Client
59
42
  end # class Tester
60
43
  end # module Webspicy
61
- require_relative 'client/support'
62
- require_relative 'client/http_client'
63
- require_relative 'client/rack_test_client'