webspicy 0.18.0 → 0.20.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/bin/webspicy +6 -1
  3. data/doc/1-black-box-scene.md +109 -0
  4. data/doc/2-black-box-testing.md +27 -0
  5. data/doc/3-specification-importance.md +41 -0
  6. data/doc/4-sequence-diagram.md +82 -0
  7. data/lib/webspicy.rb +17 -9
  8. data/lib/webspicy/configuration.rb +48 -8
  9. data/lib/webspicy/configuration/scope.rb +22 -8
  10. data/lib/webspicy/formaldoc.fio +5 -5
  11. data/lib/webspicy/specification.rb +5 -2
  12. data/lib/webspicy/specification/condition.rb +48 -0
  13. data/lib/webspicy/specification/err.rb +18 -0
  14. data/lib/webspicy/specification/oldies.rb +4 -0
  15. data/lib/webspicy/specification/oldies/bridge.rb +32 -0
  16. data/lib/webspicy/specification/{errcondition.rb → oldies/errcondition.rb} +6 -0
  17. data/lib/webspicy/specification/{postcondition.rb → oldies/postcondition.rb} +6 -0
  18. data/lib/webspicy/specification/{precondition.rb → oldies/precondition.rb} +6 -2
  19. data/lib/webspicy/specification/post.rb +20 -0
  20. data/lib/webspicy/specification/post/missing_condition_impl.rb +15 -0
  21. data/lib/webspicy/specification/post/unexpected_condition_impl.rb +15 -0
  22. data/lib/webspicy/specification/pre.rb +19 -0
  23. data/lib/webspicy/specification/{precondition → pre}/global_request_headers.rb +4 -4
  24. data/lib/webspicy/specification/{precondition → pre}/robust_to_invalid_input.rb +4 -4
  25. data/lib/webspicy/specification/service.rb +34 -5
  26. data/lib/webspicy/specification/test_case.rb +11 -9
  27. data/lib/webspicy/support.rb +31 -0
  28. data/lib/webspicy/support/hooks.rb +65 -0
  29. data/lib/webspicy/support/world.rb +47 -0
  30. data/lib/webspicy/tester.rb +202 -3
  31. data/lib/webspicy/tester/assertions.rb +2 -2
  32. data/lib/webspicy/tester/client.rb +4 -50
  33. data/lib/webspicy/tester/fakeses.rb +41 -0
  34. data/lib/webspicy/tester/fakeses/email.rb +38 -0
  35. data/lib/webspicy/tester/fakesmtp.rb +39 -0
  36. data/lib/webspicy/tester/fakesmtp/email.rb +27 -0
  37. data/lib/webspicy/tester/file_checker.rb +22 -0
  38. data/lib/webspicy/tester/invocation.rb +15 -48
  39. data/lib/webspicy/tester/reporter.rb +85 -0
  40. data/lib/webspicy/tester/reporter/composite.rb +38 -0
  41. data/lib/webspicy/tester/reporter/documentation.rb +74 -0
  42. data/lib/webspicy/tester/reporter/error_count.rb +25 -0
  43. data/lib/webspicy/tester/reporter/exceptions.rb +62 -0
  44. data/lib/webspicy/tester/reporter/file_progress.rb +25 -0
  45. data/lib/webspicy/tester/reporter/file_summary.rb +43 -0
  46. data/lib/webspicy/tester/reporter/progress.rb +30 -0
  47. data/lib/webspicy/tester/reporter/summary.rb +63 -0
  48. data/lib/webspicy/tester/result.rb +142 -0
  49. data/lib/webspicy/tester/result/assert_met.rb +29 -0
  50. data/lib/webspicy/tester/result/check.rb +33 -0
  51. data/lib/webspicy/tester/result/errcondition_met.rb +29 -0
  52. data/lib/webspicy/tester/result/error_schema_met.rb +25 -0
  53. data/lib/webspicy/tester/result/invocation_succeeded.rb +13 -0
  54. data/lib/webspicy/tester/result/output_schema_met.rb +25 -0
  55. data/lib/webspicy/tester/result/postcondition_met.rb +29 -0
  56. data/lib/webspicy/tester/result/response_header_met.rb +43 -0
  57. data/lib/webspicy/tester/result/response_status_met.rb +25 -0
  58. data/lib/webspicy/version.rb +2 -2
  59. data/lib/webspicy/web.rb +4 -0
  60. data/lib/webspicy/web/client.rb +15 -0
  61. data/lib/webspicy/{tester → web}/client/http_client.rb +34 -14
  62. data/lib/webspicy/{tester → web}/client/rack_test_client.rb +3 -3
  63. data/lib/webspicy/{tester → web}/client/support.rb +2 -2
  64. data/lib/webspicy/web/invocation.rb +69 -0
  65. data/lib/webspicy/web/mocker.rb +90 -0
  66. data/lib/webspicy/{mocker → web/mocker}/config.ru +3 -2
  67. data/lib/webspicy/{openapi.rb → web/openapi.rb} +0 -0
  68. data/lib/webspicy/web/openapi/generator.rb +129 -0
  69. data/spec/unit/specification/{precondition → pre}/test_global_request_headers.rb +9 -4
  70. data/spec/unit/specification/test_condition.rb +44 -0
  71. data/spec/unit/support/hooks/test_fire_after_each.rb +53 -0
  72. data/spec/unit/{tester/client/test_around.rb → support/hooks/test_fire_around.rb} +15 -10
  73. data/spec/unit/support/hooks/test_fire_before_each.rb +53 -0
  74. data/spec/unit/support/world/fixtures/array.json +8 -0
  75. data/spec/unit/support/world/fixtures/queue.rb +1 -0
  76. data/spec/unit/support/world/fixtures/single.json +11 -0
  77. data/spec/unit/support/world/fixtures/yaml.yml +3 -0
  78. data/spec/unit/support/world/test_world.rb +56 -0
  79. data/spec/unit/test_configuration.rb +49 -0
  80. data/spec/unit/tester/fakeses/test_email.rb +40 -0
  81. data/spec/unit/web/mocker/test_mocker.rb +35 -0
  82. data/spec/unit/web/openapi/test_generator.rb +31 -0
  83. metadata +80 -65
  84. data/examples/restful/Gemfile +0 -5
  85. data/examples/restful/Rakefile +0 -25
  86. data/examples/restful/app.rb +0 -180
  87. data/examples/restful/webspicy/config.rb +0 -24
  88. data/examples/restful/webspicy/formaldef/todo/_one/delete.yml +0 -55
  89. data/examples/restful/webspicy/formaldef/todo/_one/get.simpler.yml +0 -46
  90. data/examples/restful/webspicy/formaldef/todo/_one/get.yml +0 -50
  91. data/examples/restful/webspicy/formaldef/todo/_one/patch.yml +0 -66
  92. data/examples/restful/webspicy/formaldef/todo/_one/put.yml +0 -65
  93. data/examples/restful/webspicy/formaldef/todo/get.yml +0 -36
  94. data/examples/restful/webspicy/formaldef/todo/options.yml +0 -32
  95. data/examples/restful/webspicy/formaldef/todo/post.csv.yml +0 -43
  96. data/examples/restful/webspicy/formaldef/todo/post.file.yml +0 -40
  97. data/examples/restful/webspicy/formaldef/todo/post.yml +0 -51
  98. data/examples/restful/webspicy/formaldef/todo/todos.csv +0 -4
  99. data/examples/restful/webspicy/rack.rb +0 -7
  100. data/examples/restful/webspicy/real.rb +0 -8
  101. data/examples/restful/webspicy/schema.fio +0 -20
  102. data/examples/restful/webspicy/support/must_be_an_admin.rb +0 -20
  103. data/examples/restful/webspicy/support/must_be_authenticated.rb +0 -48
  104. data/examples/restful/webspicy/support/todo_not_removed.rb +0 -21
  105. data/examples/restful/webspicy/support/todo_removed.rb +0 -20
  106. data/examples/single_spec/spec.yml +0 -59
  107. data/examples/website/config.rb +0 -2
  108. data/examples/website/schema.fio +0 -1
  109. data/examples/website/specification/get-http.yml +0 -30
  110. data/examples/website/specification/get-https.yml +0 -30
  111. data/lib/webspicy/checker.rb +0 -10
  112. data/lib/webspicy/mocker.rb +0 -88
  113. data/lib/webspicy/openapi/generator.rb +0 -127
  114. data/lib/webspicy/rspec/checker.rb +0 -2
  115. data/lib/webspicy/rspec/checker/rspec_checker.rb +0 -24
  116. data/lib/webspicy/rspec/support/rspec_runnable.rb +0 -27
  117. data/lib/webspicy/rspec/tester.rb +0 -4
  118. data/lib/webspicy/rspec/tester/rspec_asserter.rb +0 -121
  119. data/lib/webspicy/rspec/tester/rspec_matchers.rb +0 -114
  120. data/lib/webspicy/rspec/tester/rspec_tester.rb +0 -63
  121. data/spec/unit/mocker/test_mocker.rb +0 -32
  122. data/spec/unit/openapi/test_generator.rb +0 -28
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2eaf855f36546e0f538b47c50d6cb612a741807057f82172a93bff42542b326f
4
- data.tar.gz: b2e892ebf4d30e04d843df175d01ac77e8e65e145b7e6a88b4eccc0c24614ea5
3
+ metadata.gz: 438c8ad5287da851bcbfe71f39fb3f53288949e16e4ba0e5c9fbdb12874c141c
4
+ data.tar.gz: 1b575f4fa2d1b434885fe8a13f41c9975fb9e1d9528cc44cf0d08fa7b36e2306
5
5
  SHA512:
6
- metadata.gz: e6c77544f40a49028508709e8be6e7f7b95d979212416bcde782cdb5dd361f0091433d68e38c8908800851c629053c34f2c102ae41be8dea660f5ee51a839177
7
- data.tar.gz: 8fb1d33bf31723292d39a2451a79fcca4e67c5fc57f37ade00e40290fbe12e3ebbd718fc619e7c2128fdae34e7c8e44a087649fb4199d8d3505d42ae3b0e982f
6
+ metadata.gz: 8d0d6d54fedfa5bb9ad1ba6aa1178b8e33db677ff1db5f3f56da73a6df75ee0523f08e6f04916669ce1fe13149c86ebdf826780e5520332c2d0211c95ee25f94
7
+ data.tar.gz: d33ffb74de8d5d8e0f57af996b49aeb177beff8403f9ab5c22231cc77228112efa47c8373b1d55e957832f414c32b5cf42dbeca0ca58d27ffb037ecbf8c045d8
data/bin/webspicy CHANGED
@@ -9,6 +9,7 @@
9
9
  #/
10
10
  #/ Options:
11
11
  #/ -h, --help Show this help message
12
+ #/ -k, --insecure Allow insecure server connections when using SSL
12
13
  #/ -v, --version Show webspicy version
13
14
  #/ --debug Same as LOG_LEVEL=DEBUG, takes precedence
14
15
  #/
@@ -19,6 +20,7 @@
19
20
  #/ - METHOD=GET|POST|DELETE... execute only tests matching the HTTP verb
20
21
  #/ - TAG=... execute only tests matching the given tag
21
22
  #/ - FAILFAST=yes|no stop executing tests on first failure
23
+ #/ - INSECURE=yes|no allow insecure server connections when using SSL
22
24
  #/
23
25
  require 'webspicy'
24
26
  require 'webspicy/tester'
@@ -37,6 +39,9 @@ ARGV.options do |opts|
37
39
  puts "webspicy v#{Webspicy::VERSION}, (c) Enspirit SRL"
38
40
  exit(0)
39
41
  }
42
+ opts.on('-k', '--insecure') {
43
+ ENV['INSECURE'] = 'yes'
44
+ }
40
45
  opts.on("--debug") {
41
46
  ENV['LOG_LEVEL'] = 'DEBUG'
42
47
  Webspicy::LOGGER.level = Logger.const_get(ENV['LOG_LEVEL'])
@@ -51,4 +56,4 @@ end
51
56
 
52
57
  config = Webspicy::Configuration.dress(ARGV[0])
53
58
  res = Webspicy::Tester.new(config).call
54
- abort("Some tests failed") unless res == 0
59
+ abort("#{res} errors occured") unless res == 0
@@ -0,0 +1,109 @@
1
+ # Setting the black-box scene
2
+
3
+ This document sets the black-box testing & specification scene. We start with the testing vocabulary (that is probably well known) and gradually introduce specification concepts (that are a bit less).
4
+
5
+ ## Test object
6
+
7
+ The `Test object` is the software under test, generally a backend component exposing web services. As `webspicy` is also used for documentation & specification, we sometimes call it the `Specification object` or `Documentation object`.
8
+
9
+ The test object is seen as a black-box: its specification and testing are in terms of observable behavior, not driven by the internal structure or implementation.
10
+
11
+ ## Test item
12
+
13
+ A `Test item` (resp. `Specification item`) is an individual element to be tested (resp. specified). `webspicy` is generally used to test web services invoked thanks to HTTP. If we abstract from HTTP details - and see it as the request/response invocation protocol only - test items are high-level software operations called with input/output data.
14
+
15
+ ## Test case
16
+
17
+ A `Test case` is a complete setting for invoking a test item, together with assertions on the result. The setting includes a `Current system state` and a `Request` (that contains `Input` data). The result covers the `Resulting system state`, and a `Response` (that contains `Output` data).
18
+
19
+ ## Test suite
20
+
21
+ A `Test suite` of an object is the set of all its test cases.
22
+
23
+ ## System state
24
+
25
+ Most of the time we are interested in (deterministic) *stateful* testing. That is, we consider that the behavior of a test item is not only influenced by the `Input` data but also by the current system state.
26
+
27
+ The `System state` covers the content of all stateful components the test object interacts with in its environment: files, databases, caches, buses, etc.
28
+
29
+ For effective testing, one needs to be able to control and check the system state easily.
30
+
31
+ ## Current system state
32
+
33
+ A `Current system state` is the state of the system just before invoking a test item.
34
+
35
+ ## Resulting system state
36
+
37
+ A `Resulting system state` is the state of the system just after invoking a test item from an current state.
38
+
39
+ ## Request
40
+
41
+ A `Request` is what is sent to a test object to invoke a test item. When test items are web services, it is a HTTP request specifically.
42
+
43
+ ## Response
44
+
45
+ A `Response` is what is received from the test object in return from a test item invocation request. When test items are web services, it is a HTTP response specifically.
46
+
47
+ ## Input
48
+
49
+ The `Input` is the data on which a high-level operation is executed. In a similar way that a high-level operation abstracts from web services details, the input abstracts from HTTP request details.
50
+
51
+ ## Output
52
+
53
+ The `Output` is the data returned by a high-level operation after execution. In a similar way that a high-level operation abstracts from web services details, the output abstracts from HTTP response details.
54
+
55
+ ## Precondition
56
+
57
+ A `Precondition` is a necessary condition to be met for the test item to execute successfully when invoked. A precondition can refer to the current state, the request, and/or the input.
58
+
59
+ The set of preconditions of an item is sometimes written `PRE`.
60
+
61
+ ## Postcondition
62
+
63
+ A `Postcondition` is a condition that is guaranteed by the test item (if implemented correctly) when invoked while its preconditions holds. A postcondition can refer to the current state (before execution), the request, the input, the resulting state, the response, and/or the output.
64
+
65
+ ## Input schema
66
+
67
+ The `Input schema` is a formal definition of the set of valid input data of a test item (hence the underlying high-level operation). The input schema is a `webspicy` formal shortcut for a precondition stating that "input data must be valid, that is ...".
68
+
69
+ (The input schema can also be seen as the signature of the high-level operation.)
70
+
71
+ ## Output schema
72
+
73
+ The `Output schema` is a formal definition of the set of possible output data of a test item (idem). The output schema is a `webspicy` formal shortcut for a postcondition stating that "output data is valid, that is ...".
74
+
75
+ (The output schema can also be seen as the return type of the high-level operation.)
76
+
77
+ ## Ideal specification
78
+
79
+ An `Ideal specification` is a formal identification of an item, together with the set of its pre and post conditions. By nature the ideal specification covers the input and output schema.
80
+
81
+ ## Example
82
+
83
+ An `Example` is a positive test case. That is, it is the description of an invocation setting that meets all preconditions of a test item together with assertions on the successful result.
84
+
85
+ ## Counterexample
86
+
87
+ A `Counterexample` is a negative test case. That is, it is the description of an invocation setting that violates at least one precondition, together with assertions on the error result.
88
+
89
+ ## Errcondition
90
+
91
+ An `Errcondition` is a condition that is guaranteed by the test item (if implemented correctly) when invoked while at least one precondition is violated. An errcondition can refer to the current state (before execution), the request, the input, the resulting state, the response, and/or the output.
92
+
93
+ An item is said to be robust when it has strong errconditions: it guarantees some behavior and results even in case of execution failures.
94
+
95
+ ## Error schema
96
+
97
+ The `Error schema` is a formal definition of the set of possible output data of a test item when it fails to execute successfully, typically because a precondition is violated. The error schema is a `webspicy` formal shortcut for an errcondition stating that "error data must be valid, that is ...".
98
+
99
+ ## Exceptional specification
100
+
101
+ An `Exceptional specification` is a formal identification of an item, together with the set of its pre and err conditions. By nature the exceptional specification covers the input and err schema.
102
+
103
+ ## Item specification
104
+
105
+ An `Item specification` is the set of its ideal and exception specications. Therefore it includes all schema, pre, post and err conditions.
106
+
107
+ ## Specification
108
+
109
+ A `Specification` of an object is the set of all its item specifications.
@@ -0,0 +1,27 @@
1
+ # Black-box testing
2
+
3
+ We define here what we call `black-box testing`, using the vocabulary installed previously.
4
+
5
+ ## Objective
6
+
7
+ Black-box testing aims at checking whether a test object (actually its implementation) meets its specification. It does so by looking at the object's observable behavior only.
8
+
9
+ ## Method
10
+
11
+ Testing is never exhaustive and gives no correctness guarantees. As one says, testing can only show the presence of defects, not their absence.
12
+
13
+ In practice, black-box testing meets its objective by checking whether:
14
+
15
+ * all positive test cases (i.e. examples) yield successful executions meeting the postconditions.
16
+
17
+ * all negative test cases (i.e. counterexamples) yield unsuccessful executions meeting the errconditions.
18
+
19
+ ## Prerequisites
20
+
21
+ Applying black-box testing to web services requires at least:
22
+
23
+ * A test item: a web service endpoint
24
+ * A specification: pre, post and err conditions of the web service
25
+ * Test cases: examples and counterexamples of web service usage
26
+
27
+ The second element, i.e. the *specification*, is both important and useful. Modern test frameworks leave the specification implicit (not written at all) or informal (written as code documentation). We think this is both a mistake and a lost opportunity. We will explain why in next section.
@@ -0,0 +1,41 @@
1
+ # On the importance of the specification
2
+
3
+ Modern software development methods and frameworks, especially Test-Driven (TDD) and Behavior-Driven Development (BDD), use tests extensively. As their name suggests, they use tests to *drive* the development. In fact, it is a major contribution of Agile methods to software development: driving the development by tests invites the developer to improve his code through testing and thereby also improves test coverage as a side effect of the development itself.
4
+
5
+ However this is not the traditional objective of software testing, which is to test the implementation's conformance to a specification. This yields strange situations where a lot of tests are written while the specification is not:
6
+
7
+ * Most of the time the specification is implicit: it is simply not written at all,
8
+ * Sometimes the specification is written as informal comments on the tested method or class,
9
+ * Some test frameworks even suggest that the set of tests *is* the specification, which is notably wrong.
10
+
11
+ ## What is a specification?
12
+
13
+ As a reminder, a `Specification` is the set of `PRE-`, `POST-` and `ERR-` `conditions` defining the contract between the test item (e.g. a web service) and its consumer. In short,
14
+
15
+ - if the consumer calls the service while the `PRE` are met, then the service guarantees that the `POST` are met.
16
+ - if the consumer calls the service while a `PRE` is not met, and the service is robust, then it guarantees that the `ERR` are met.
17
+
18
+ ## Advantages of an explicit specification
19
+
20
+ Having a explicit specification has traditional advantages:
21
+
22
+ - It documents the software, generally in a better and more consise way than a full test suite
23
+ - It enables reasoning about correctness: if you don't know the contractual conditions you can't actually know whether the test suite passes by chance of because the software implementation is correct
24
+
25
+ Beyond those, specifications can also be used somewhat "against the grain", in the test item / specification / test-case triangle. This is the reason why `webspicy` exists, as we want to explore the following advantages in the specific context of API testing:
26
+
27
+ - A specification can be used to make tests easier to write
28
+ * by instrumenting positive test cases so that preconditions are met with almost no effort from the tester
29
+ * by generating current system states automatically, instead of having to script them
30
+
31
+ - A specification can be used to generate test cases automatically
32
+ * generating examples aims at testing correctness
33
+ * generating counterexamples aims at testing robustness & security
34
+
35
+ - A specification can be used as a roadmap
36
+ * documenting unchecked preconditions highlights robustness weakenesses and calls for counterexamples
37
+ * documenting unmet postconditions highlights bugs and call for examples
38
+
39
+ - A specification may capture best practices and conventions
40
+ * Pre and post-conditions can sometimes be reused across different softwares, when they apply to conventions such as RESTful APIs
41
+
@@ -0,0 +1,82 @@
1
+ ```
2
+ @startuml
3
+ participant Tester
4
+ participant Client
5
+ participant Result
6
+ participant PrePost
7
+ participant Config
8
+ Tester -> Config : before_all(Scope, Client)
9
+ group * test case
10
+ Tester -> Config : before_each(TestCase, Client)
11
+ Tester -> Client : instrument(TestCase)
12
+ group * pre/post/err
13
+ Client -> PrePost : instrument(TestCase, Client)
14
+ end
15
+ Client -> Config : instrument(TestCase, Client)
16
+ Tester -> Client : call(TestCase)
17
+ Tester -> Result : check_response!
18
+ Tester -> Result : check_output!
19
+ Tester -> Result : check_assertions!
20
+ Tester -> Result : check_postconditions!
21
+ group * post/err
22
+ Result -> PrePost : check(Invocation)
23
+ end
24
+ Tester -> Config : after_each(TestCase, Client)
25
+ end
26
+ Tester -> Config : after_all(Scope, Client)
27
+ @enduml
28
+ ```
29
+
30
+ ┌──────┐ ┌──────┐ ┌──────┐ ┌───────┐ ┌──────┐
31
+ │Tester│ │Client│ │Result│ │PrePost│ │Config│
32
+ └──┬───┘ └──┬───┘ └──┬───┘ └───┬───┘ └──┬───┘
33
+ │ │ │ │ │
34
+ │ │ before_all(Scope, Client) │ │
35
+ │───────────────────────────────────────────────────────────────────────────>│
36
+ │ │ │ │ │
37
+ ╔══════════════╤═══════════════════╪═════════════════╪══════════════════╪═════════════════╪═════════════╗
38
+ ║ * TEST CASE │ │ │ │ │ ║
39
+ ╟──────────────┘ │ before_each(TestCase, Client) │ │ ║
40
+ ║ │───────────────────────────────────────────────────────────────────────────>│ ║
41
+ ║ │ │ │ │ │ ║
42
+ ║ │ instrument(TestCase)│ │ │ │ ║
43
+ ║ │────────────────────>│ │ │ │ ║
44
+ ║ │ │ │ │ │ ║
45
+ ║ │ ╔════════════╪════╤════════════╪══════════════════╪═════════════╗ │ ║
46
+ ║ │ ║ * PRE/POST/ERR │ │ │ ║ │ ║
47
+ ║ │ ╟─────────────────┘ │ │ ║ │ ║
48
+ ║ │ ║ │ instrument(TestCase, Client) │ ║ │ ║
49
+ ║ │ ║ │───────────────────────────────────>│ ║ │ ║
50
+ ║ │ ╚════════════╪═════════════════╪══════════════════╪═════════════╝ │ ║
51
+ ║ │ │ │ │ │ ║
52
+ ║ │ │ instrument(TestCase, Client) │ ║
53
+ ║ │ │─────────────────────────────────────────────────────>│ ║
54
+ ║ │ │ │ │ │ ║
55
+ ║ │ call(TestCase) │ │ │ │ ║
56
+ ║ │────────────────────>│ │ │ │ ║
57
+ ║ │ │ │ │ │ ║
58
+ ║ │ check_response! │ │ │ ║
59
+ ║ │──────────────────────────────────────>│ │ │ ║
60
+ ║ │ check_output! │ │ │ ║
61
+ ║ │──────────────────────────────────────>│ │ │ ║
62
+ ║ │ check_assertions! │ │ │ ║
63
+ ║ │──────────────────────────────────────>│ │ │ ║
64
+ ║ │ check_postconditions! │ │ │ ║
65
+ ║ │──────────────────────────────────────>│ │ │ ║
66
+ ║ │ │ │ │ │ ║
67
+ ║ │ │ ╔═════════════╤═════════════════╪═════════════╗ │ ║
68
+ ║ │ │ ║ * POST/ERR │ │ ║ │ ║
69
+ ║ │ │ ╟─────────────┘ │ ║ │ ║
70
+ ║ │ │ ║ │ check(Invocation)│ ║ │ ║
71
+ ║ │ │ ║ │─────────────────>│ ║ │ ║
72
+ ║ │ │ ╚════════════╪══════════════════╪═════════════╝ │ ║
73
+ ║ │ │ │ │ │ ║
74
+ ║ │ │ after_each(TestCase, Client) │ │ ║
75
+ ║ │───────────────────────────────────────────────────────────────────────────>│ ║
76
+ ╚════════════╪═════════════════════╪═════════════════╪══════════════════╪═════════════════╪═════════════╝
77
+ │ │ │ │ │
78
+ │ │ after_all(Scope, Client) │ │
79
+ │───────────────────────────────────────────────────────────────────────────>│
80
+ ┌──┴───┐ ┌──┴───┐ ┌──┴───┐ ┌───┴───┐ ┌──┴───┐
81
+ │Tester│ │Client│ │Result│ │PrePost│ │Config│
82
+ └──────┘ └──────┘ └──────┘ └───────┘ └──────┘
data/lib/webspicy.rb CHANGED
@@ -5,11 +5,11 @@ require 'finitio'
5
5
  require 'logger'
6
6
  require 'ostruct'
7
7
  require 'yaml'
8
- require 'rspec'
9
8
  require 'rack/test'
10
9
  require 'mustermann'
11
10
  require 'colorized_string'
12
11
  require 'securerandom'
12
+ require 'forwardable'
13
13
  module Webspicy
14
14
 
15
15
  ###
@@ -20,20 +20,22 @@ module Webspicy
20
20
  require 'webspicy/support'
21
21
  require 'webspicy/specification'
22
22
  require 'webspicy/configuration'
23
- require 'webspicy/checker'
24
23
  require 'webspicy/tester'
24
+ require 'webspicy/web'
25
+
26
+ class Error < StandardError; end
27
+ class TimeoutError < Error; end
25
28
 
26
29
  ###
27
30
  ### Backward compatibility
28
31
  ###
29
32
  Client = Tester::Client
30
- HttpClient = Tester::HttpClient
31
- RackTestClient = Tester::RackTestClient
33
+ HttpClient = Web::HttpClient
34
+ RackTestClient = Web::RackTestClient
32
35
  Resource = Specification
33
- Precondition = Specification::Precondition
34
- Postcondition = Specification::Postcondition
35
36
  FileUpload = Specification::FileUpload
36
37
  Scope = Configuration::Scope
38
+ Checker = Tester::FileChecker
37
39
 
38
40
  ###
39
41
  ### About folders
@@ -52,6 +54,12 @@ module Webspicy
52
54
  FIO
53
55
  FORMALDOC = Finitio.system(Path.dir/("webspicy/formaldoc.fio"))
54
56
 
57
+ ###
58
+ ### Exceptions that we let pass during testing
59
+ ###
60
+
61
+ PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, SystemExit]
62
+
55
63
  # Returns a default scope instance.
56
64
  def default_scope
57
65
  Configuration::Scope.new(Configuration.new)
@@ -90,9 +98,9 @@ module Webspicy
90
98
  module_function :test_case
91
99
 
92
100
  def handle_finitio_error(ex, scope)
93
- msg = "#{ex.message}:\n #{ex.root_cause.message}"
94
- msg = Support::Colorize.colorize_error(msg, scope.config)
95
- fatal(msg)
101
+ # msg = "#{ex.message}:\n #{ex.root_cause.message}"
102
+ # msg = Support::Colorize.colorize_error(msg, scope.config)
103
+ # fatal(msg)
96
104
  raise
97
105
  end
98
106
  module_function :handle_finitio_error
@@ -3,15 +3,29 @@ module Webspicy
3
3
 
4
4
  LISTENER_KINDS = [ :before_all, :before_each, :after_all, :after_each, :around_each ]
5
5
 
6
- def initialize(folder = Path.pwd, parent = nil)
7
- @folder = folder
6
+ class << self
7
+ attr_accessor :default_folder
8
+
9
+ def with_default_folder(f)
10
+ old = default_folder
11
+ @default_folder = f
12
+ yield.tap{
13
+ @default_folder = old
14
+ }
15
+ end
16
+ end
17
+
18
+ def initialize(folder = Configuration.default_folder || Path.pwd, parent = nil)
19
+ @folder = folder.expand_path
8
20
  @parent = parent
9
21
  @children = []
10
22
  @preconditions = []
11
23
  @postconditions = []
12
24
  @errconditions = []
25
+ @insecure = default_insecure
13
26
  @listeners = Hash.new{|h,k| h[k] = [] }
14
27
  @rspec_options = default_rspec_options
28
+ @failfast = default_failfast
15
29
  @run_examples = default_run_examples
16
30
  @run_counterexamples = default_run_counterexamples
17
31
  @run_generated_counterexamples = default_run_generated_counterexamples
@@ -24,14 +38,16 @@ module Webspicy
24
38
  :success => :green
25
39
  }
26
40
  @scope_factory = ->(config){ Scope.new(config) }
27
- @client = Tester::HttpClient
28
- Path.require_tree(folder/'support') if (folder/'support').exists?
41
+ @client = Web::HttpClient
42
+ Path.require_tree(@folder/'support') if (@folder/'support').exists?
43
+ @world = Support::World.new(folder/'world', self)
29
44
  yield(self) if block_given?
30
45
  end
31
46
  attr_accessor :folder
32
47
  protected :folder=
33
48
 
34
49
  attr_accessor :colors
50
+ attr_reader :world
35
51
 
36
52
  def self.dress(arg, &bl)
37
53
  case arg
@@ -44,9 +60,11 @@ module Webspicy
44
60
  when ->(f){ Path(f).exists? }
45
61
  arg = Path(arg)
46
62
  if arg.file? && arg.ext == ".rb"
47
- c = Kernel.instance_eval arg.read, arg.to_s
48
- yield(c) if block_given?
49
- c
63
+ Configuration.with_default_folder(arg.parent) do
64
+ c = Kernel.instance_eval arg.read, arg.to_s
65
+ yield(c) if block_given?
66
+ c
67
+ end
50
68
  elsif arg.file? && arg.ext == '.yml'
51
69
  folder = arg.backfind("[config.rb]")
52
70
  if folder && folder.exists?
@@ -82,7 +100,7 @@ module Webspicy
82
100
  config.each_scope(&bl)
83
101
  end
84
102
  else
85
- yield factor_scope
103
+ yield(factor_scope)
86
104
  end
87
105
  end
88
106
 
@@ -377,6 +395,28 @@ module Webspicy
377
395
  attr_accessor :rspec_options
378
396
  protected :rspec_options=
379
397
 
398
+ # Returns the default value for failfast
399
+ #
400
+ # The following environment variables <-> option are supported:
401
+ #
402
+ # - FAILFAST=yes <-> true
403
+ #
404
+ def default_failfast
405
+ ENV['FAILFAST'] == 'yes' || ENV['FAILFAST'] == "1"
406
+ end
407
+ attr_accessor :failfast
408
+
409
+ # Returns the default value to use for insecure.
410
+ #
411
+ # The following environment variables <-> option are supported:
412
+ #
413
+ # - INSECURE=yes <-> true
414
+ #
415
+ def default_insecure
416
+ ENV['INSECURE'] == 'yes' || ENV['INSECURE'] == "1"
417
+ end
418
+ attr_accessor :insecure
419
+
380
420
  # Returns the default rspec options.
381
421
  #
382
422
  # By default rspec colors are enabled and the format set to 'documentation'.