theorem 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 433696da9dd29c04e52b86d753dbbc73aba33fdb3e2a14a1c7e03dc4f7f1c2c9
4
- data.tar.gz: 42f41fc806176cecbcce43441ace89126e751bed7ab23b103b91477451b243c5
3
+ metadata.gz: '09d0acbeb340968a19cb27122ba6c852b66b2cc4347fdfca7f6ba6a9f7781ce0'
4
+ data.tar.gz: aef702741a59ff1a11f08e6225b0804b30b813ed0cc8eb5184cdd50c54587901
5
5
  SHA512:
6
- metadata.gz: 336ef51f1a4b3bfe1b34da109e8ab193a10aff873a3e4151c4d3a754be479bec455aae731886f8a35c769effc1f89b439ef66055b6a62b096b5fa53ec966c3a4
7
- data.tar.gz: e5454f1c81f833480f2b5eccbe9f516f9a4826c7ae743f47c1e4112123b0c558220b2dde81eb918929622aca286ba67b8679f3a3cc1a71dee14313a80e8f1b9c
6
+ metadata.gz: 6055b29fd80529f514b73363a18582122a5e23b312fda318b3d83a546bb7d8b062fcaf87f57858340e432bc1327a183f1b494c107d0970861b9548974f34996f
7
+ data.tar.gz: 40c0dc5d7660db43bdf6606aae81a43c48cb5015dd29eacd63a4f2afc2e0daa2d70b34f1ce04560f190d2dd282caef3bc67d4c37236c5bcda3abfce5bed65aad
data/src/harness.rb CHANGED
@@ -16,26 +16,4 @@ module Theorem
16
16
  filtered_registry(options)
17
17
  end
18
18
  end
19
-
20
- # module retry harness
21
- module RetryHarness
22
- include Control::Harness
23
-
24
- on_run do |tests|
25
- arr_of_tests = tests.map { |test| { test: test, index: 0 } }
26
-
27
- final_results = []
28
- arr_of_tests.each do |test|
29
- test[:index] += 1
30
- results = test[:test].run!
31
- if results.any?(&:failed?) && test[:index] <= 3
32
- puts "Retrying iteration: #{test[:index]}\n#{results.map(&:full_name).join("\n")}"
33
- redo
34
- end
35
- final_results.concat results
36
- end
37
-
38
- final_results
39
- end
40
- end
41
19
  end
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../theorem/reporter'
4
+
5
+ module Theorem
6
+ module StringRefinements
7
+ refine String do
8
+ # colorization
9
+ def colorize(color_code)
10
+ "\e[#{color_code}m#{self}\e[0m"
11
+ end
12
+
13
+ def red
14
+ colorize(31)
15
+ end
16
+
17
+ def green
18
+ colorize(32)
19
+ end
20
+
21
+ def yellow
22
+ colorize(33)
23
+ end
24
+
25
+ def blue
26
+ colorize(34)
27
+ end
28
+
29
+ def pink
30
+ colorize(35)
31
+ end
32
+
33
+ def light_blue
34
+ colorize(36)
35
+ end
36
+ end
37
+ end
38
+
39
+ # Default Stdout reporter
40
+ module StdoutReporter
41
+ extend Control::Reporter
42
+ using StringRefinements
43
+
44
+ subscribe :test_finished do |test|
45
+ print test.failed? ? 'x'.red : '.'.green
46
+ end
47
+
48
+ subscribe :suite_finished do |results, duration|
49
+ puts "\n"
50
+ report_summary(results)
51
+
52
+ failed_tests = results.select(&:failed?)
53
+
54
+ report_failures(failed_tests)
55
+
56
+ puts "\nTotal time: #{duration} seconds"
57
+ puts "Total tests: #{results.size}\n"
58
+ end
59
+
60
+ def inflate_percentiles(tests)
61
+ sorted = tests.reject { |t| t.duration.nil? }.sort_by(&:duration)
62
+ tests.each_with_object([]) do |test, arr|
63
+ _, below = sorted.partition do |duration_test|
64
+ if test.duration.nil?
65
+ true
66
+ else
67
+ test.duration >= duration_test.duration
68
+ end
69
+ end
70
+ hash = {}
71
+ hash[:percentile] = (below.size.to_f / sorted.size.to_f) * 100
72
+ hash[:test] = test
73
+ arr << hash
74
+ end
75
+ end
76
+
77
+ def report_summary(tests)
78
+ inflated = inflate_percentiles(tests)
79
+
80
+ top_20 = 80 / (100.0 * (tests.size + 1).to_f)
81
+ lowest_20 = 20 / (100.0 * (tests.size + 1).to_f)
82
+
83
+ inflated.each do |test|
84
+ icon = test[:test].failed? ? '❌'.red : '✓'.green
85
+ puts "#{icon} #{test[:test].full_name.blue} : #{duration(test)}"
86
+ end
87
+ end
88
+
89
+ def duration(test)
90
+ return 'Not run' if test[:test].duration.nil?
91
+
92
+ str = "#{format('%<num>0.10f', num: test[:test].duration)} seconds"
93
+ rank = ((test[:percentile] / 100) * (test[:test].duration + 1)) * 100
94
+ if rank < 5
95
+ str.red
96
+ elsif rank > 95
97
+ str.green
98
+ elsif rank < 10
99
+ str.yellow
100
+ else
101
+ str
102
+ end
103
+ end
104
+
105
+ def report_failures(tests)
106
+ tests.each do |failure|
107
+ puts "\n\nFailure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
108
+ end
109
+ end
110
+ end
111
+ end
@@ -9,6 +9,10 @@ module Theorem
9
9
  @ctx = ctx
10
10
  end
11
11
 
12
+ def name
13
+ @test.name
14
+ end
15
+
12
16
  def run!
13
17
  @test.run!(@ctx)
14
18
  end
@@ -54,6 +58,10 @@ module Theorem
54
58
  end
55
59
  end
56
60
 
61
+ def empty?
62
+ @state.empty?
63
+ end
64
+
57
65
  def reverse_prepare(&block)
58
66
  @state.unshift block
59
67
  end
@@ -4,16 +4,14 @@ module Theorem
4
4
  module Control
5
5
  # error class
6
6
  class CompletedTest
7
- attr_reader :test, :error
7
+ attr_reader :test, :duration
8
+ attr_accessor :error, :notary
8
9
 
9
- def initialize(test, error = nil, notary:)
10
+ def initialize(test, error = nil, notary:, duration: nil)
10
11
  @test = test
11
12
  @error = error
12
13
  @notary = notary
13
- end
14
-
15
- def notary
16
- @notary
14
+ @duration = duration
17
15
  end
18
16
 
19
17
  def full_name
@@ -11,9 +11,21 @@ module Theorem
11
11
  mod.define_singleton_method :included do |inner|
12
12
  inner.define_singleton_method :run! do |options: {}|
13
13
  tests = inner.instance_exec options, &mod.test_loader
14
+
15
+ inner.suite_started_subscribers.each do |subscriber|
16
+ subscriber.call tests.map(&:tests).flatten.map do |test|
17
+ { name: test.name, metadata: test.metadata }
18
+ end
19
+ end
20
+
21
+ starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
14
22
  results = inner.instance_exec tests, options, &mod.run_loader
15
- inner.completed_suite_subscribers.each do |subscriber|
16
- subscriber.call(results)
23
+ ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
24
+
25
+ duration = ending - starting
26
+
27
+ inner.suite_finished_subscribers.each do |subscriber|
28
+ subscriber.call(results, duration)
17
29
  end
18
30
  exit results.any?(&:failed?) ? 1 : 0
19
31
  end
@@ -22,19 +34,6 @@ module Theorem
22
34
 
23
35
  # harness helpers
24
36
  module ClassMethods
25
- DEFAULT_LOADER = ->(options) do
26
- directory = options[:directory] || '.'
27
-
28
- ExtendedDir.require_all("./#{directory}")
29
-
30
- registry
31
- end
32
-
33
- DEFAULT_RUNNER = ->(tests, options) do
34
- tests.each_with_object([]) do |test, memo|
35
- memo.concat test.run!
36
- end
37
- end
38
37
 
39
38
  def load_tests(&block)
40
39
  @on_load_tests = block
@@ -45,11 +44,31 @@ module Theorem
45
44
  end
46
45
 
47
46
  def run_loader
48
- @on_run || DEFAULT_RUNNER
47
+ @on_run || default_runner
49
48
  end
50
49
 
51
50
  def test_loader
52
- @on_load_tests || DEFAULT_LOADER
51
+ @on_load_tests || default_loader
52
+ end
53
+
54
+ private
55
+
56
+ def default_loader
57
+ lambda do |options|
58
+ directory = options[:directory] || '.'
59
+
60
+ ExtendedDir.require_all("./#{directory}")
61
+
62
+ registry
63
+ end
64
+ end
65
+
66
+ def default_runner
67
+ lambda do |tests, options|
68
+ tests.each_with_object([]) do |test, memo|
69
+ memo.concat test.run!
70
+ end
71
+ end
53
72
  end
54
73
  end
55
74
  end
@@ -39,9 +39,9 @@ module Theorem
39
39
  mod.add_to_registry(klass)
40
40
  end
41
41
 
42
- mod.const_set(:Beaker, Beaker)
43
- mod.const_set(:Test, Test)
44
- mod.const_set(:CompletedTest, CompletedTest)
42
+ mod.const_set(:Beaker, Beaker) unless mod.const_defined?(:Beaker)
43
+ mod.const_set(:Test, Test) unless mod.const_defined?(:Test)
44
+ mod.const_set(:CompletedTest, CompletedTest) unless mod.const_defined?(:CompletedTest)
45
45
  mod.extend(Registry)
46
46
  end
47
47
  end
@@ -28,31 +28,17 @@ module Theorem
28
28
  registry << klass
29
29
  end
30
30
 
31
- def on_extra_event(&block)
32
- @extra_events ||= []
33
- @extra_events << block
34
- end
35
-
36
- def extra_event_subscribers
37
- @extra_events || []
38
- end
39
-
40
- def on_completed_test(&block)
41
- @completed_tests ||= []
42
- @completed_tests << block
43
- end
31
+ %i[suite_started test_started test_finished suite_finished].each do |method|
32
+ define_method method do |&block|
33
+ instance_variable_set("@#{method}_subscribers", []) unless instance_variable_get("@#{method}_subscribers")
34
+ instance_variable_get("@#{method}_subscribers").append(block)
35
+ end
44
36
 
45
- def completed_test_subscribers
46
- @completed_tests || []
47
- end
37
+ define_method "#{method}_subscribers" do
38
+ return [] unless instance_variable_get("@#{method}_subscribers")
48
39
 
49
- def on_completed_suite(&block)
50
- @completed_suites ||= []
51
- @completed_suites << block
52
- end
53
-
54
- def completed_suite_subscribers
55
- @completed_suites || []
40
+ instance_variable_get("@#{method}_subscribers")
41
+ end
56
42
  end
57
43
  end
58
44
  end
data/src/theorem/test.rb CHANGED
@@ -110,6 +110,9 @@ module Theorem
110
110
  before_failures = run_before_all_beakers(test_case)
111
111
 
112
112
  if before_failures.any?
113
+ before_failures.each do |failure|
114
+ publish_test_completion(failure)
115
+ end
113
116
  return before_failures
114
117
  end
115
118
 
@@ -118,29 +121,41 @@ module Theorem
118
121
 
119
122
  results = []
120
123
  @tests.each do |test|
124
+ test_start = clock_time
125
+
126
+ publish_test_start(test)
127
+
121
128
  error ||= run_before_each_beakers(test_case)
122
- before_test_case = test_case.clone
123
129
 
130
+ before_test_case = test_case.clone
124
131
  error ||= run_test(test, before_test_case)
125
-
126
132
  error ||= run_after_each_beakers(before_test_case)
127
133
 
128
134
  notary = test_case.notary.merge(test.notary)
129
135
 
130
- completed_test = CompletedTest.new(test, error, notary: notary.dump)
131
- publish_test_completion(completed_test)
136
+ duration = clock_time - test_start
137
+
138
+ completed_test = CompletedTest.new(test, error, duration: duration, notary: notary.dump)
139
+
140
+ # publish_early if there are no after_all beakers
141
+ publish_test_completion(completed_test) if @after_all.empty?
142
+
132
143
  results << completed_test
133
144
  end
134
145
 
135
- after_failures = run_after_all_beakers(duplicate_test_case)
146
+ after_failures = run_after_all_beakers(results, duplicate_test_case)
136
147
 
137
148
  if after_failures.any?
149
+ after_failures.each do |failure|
150
+ publish_test_completion(failure)
151
+ end
138
152
  return after_failures
139
153
  end
140
154
 
141
155
  results.each do |completed_test|
142
156
  # merge any after_all notations
143
157
  completed_test.notary.merge!(duplicate_test_case.notary.dump)
158
+ publish_test_completion(completed_test) unless @after_all.empty?
144
159
  end
145
160
 
146
161
  results
@@ -148,6 +163,10 @@ module Theorem
148
163
 
149
164
  private
150
165
 
166
+ def clock_time
167
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
168
+ end
169
+
151
170
  def run_test(test, test_case)
152
171
  if @around.empty?
153
172
  begin
@@ -163,7 +182,7 @@ module Theorem
163
182
  end
164
183
  end
165
184
 
166
- def run_after_all_beakers(test_case)
185
+ def run_after_all_beakers(results, test_case)
167
186
  @after_all.run!(test_case)
168
187
 
169
188
  @parent_after_all&.each do |beaker|
@@ -174,9 +193,12 @@ module Theorem
174
193
  rescue Exception => error
175
194
  Theorem.handle_exception(error)
176
195
 
177
- @tests.map do |test|
178
- CompletedTest.new(test, error, notary: test_case.notary)
196
+ results.each do |test|
197
+ test.error = error
198
+ test.notary = test_case.notary
179
199
  end
200
+
201
+ results
180
202
  end
181
203
 
182
204
  def run_after_each_beakers(test_case)
@@ -219,11 +241,16 @@ module Theorem
219
241
  end
220
242
 
221
243
  def publish_test_completion(completed_test)
222
- control.completed_test_subscribers.each do |subscriber|
244
+ control.test_finished_subscribers.each do |subscriber|
223
245
  subscriber.call(completed_test)
224
246
  end
225
247
  end
226
248
 
249
+ def publish_test_start(test)
250
+ control.test_started_subscribers.each do |subscriber|
251
+ subscriber.call(test)
252
+ end
253
+ end
227
254
  end
228
255
  end
229
256
  end
data/src/theorem.rb CHANGED
@@ -1,8 +1,7 @@
1
1
  require_relative 'theorem/hypothesis'
2
2
  require_relative 'harness'
3
3
  require_relative 'experiment'
4
- require_relative 'stdout_reporter'
5
-
4
+ require_relative 'publishers/stdout_reporter'
6
5
  require_relative 'theorem/harness'
7
6
  require 'json'
8
7
 
@@ -45,18 +44,6 @@ module Theorem
45
44
  mod.run!(options: options)
46
45
  end
47
46
 
48
- module JsonReporter
49
- extend Control::Reporter
50
-
51
- subscribe :on_completed_suite do |results|
52
- results = results.map do |result|
53
- { name: result.full_name, failed: result.failed? }
54
- end
55
- puts results.to_json
56
- puts "\n\n"
57
- end
58
- end
59
-
60
47
  module Hypothesis
61
48
  include Control::Hypothesis
62
49
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: theorem
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sean Gregory
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-04-09 00:00:00.000000000 Z
11
+ date: 2022-04-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: extended_dir
@@ -80,6 +80,34 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: simplecov
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: simplecov-cobertura
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
83
111
  description: simple and extensible test library toolkit
84
112
  email: sean.christopher.gregory@gmail.com
85
113
  executables:
@@ -90,7 +118,7 @@ files:
90
118
  - bin/theorize
91
119
  - src/experiment.rb
92
120
  - src/harness.rb
93
- - src/stdout_reporter.rb
121
+ - src/publishers/stdout_reporter.rb
94
122
  - src/theorem.rb
95
123
  - src/theorem/beaker.rb
96
124
  - src/theorem/completed_test.rb
@@ -1,70 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative './theorem/reporter'
4
-
5
- module Theorem
6
- module StringRefinements
7
- refine String do
8
- # colorization
9
- def colorize(color_code)
10
- "\e[#{color_code}m#{self}\e[0m"
11
- end
12
-
13
- def red
14
- colorize(31)
15
- end
16
-
17
- def green
18
- colorize(32)
19
- end
20
-
21
- def yellow
22
- colorize(33)
23
- end
24
-
25
- def blue
26
- colorize(34)
27
- end
28
-
29
- def pink
30
- colorize(35)
31
- end
32
-
33
- def light_blue
34
- colorize(36)
35
- end
36
- end
37
- end
38
-
39
- # Default Stdout reporter
40
- module StdoutReporter
41
- extend Control::Reporter
42
- using StringRefinements
43
-
44
- subscribe :on_completed_test do |test|
45
- print test.failed? ? 'x'.red : '.'.green
46
- end
47
-
48
- subscribe :on_completed_suite do |results|
49
- puts "\n"
50
- report_summary(results)
51
-
52
- failed_tests = results.select(&:failed?)
53
-
54
- report_failures(failed_tests)
55
- end
56
-
57
- def report_summary(tests)
58
- tests.each do |test|
59
- icon = test.failed? ? '❌'.red : '✓'.green
60
- puts "#{icon} #{test.full_name.blue}"
61
- end
62
- end
63
-
64
- def report_failures(tests)
65
- tests.each do |failure|
66
- puts "Failure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
67
- end
68
- end
69
- end
70
- end