pbt 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.standard.yml +1 -1
- data/CHANGELOG.md +21 -0
- data/CLAUDE.md +2 -2
- data/README.md +116 -36
- data/benchmark/FIBER.md +97 -0
- data/benchmark/fiber_experiment.rb +39 -0
- data/benchmark/fiber_scheduler/Gemfile +7 -0
- data/benchmark/fiber_scheduler/experiment.rb +131 -0
- data/benchmark/fiber_scheduler/spec/experiment_spec.rb +38 -0
- data/docs/plans/fiber-experiment.md +12 -0
- data/docs/plans/generator-statistics.md +16 -0
- data/docs/plans/stateful-pbt-redesign.md +415 -0
- data/lib/pbt/arbitrary/arbitrary.rb +65 -12
- data/lib/pbt/arbitrary/arbitrary_methods.rb +51 -24
- data/lib/pbt/arbitrary/array_arbitrary.rb +46 -12
- data/lib/pbt/arbitrary/choose_arbitrary.rb +28 -1
- data/lib/pbt/arbitrary/constant.rb +0 -1
- data/lib/pbt/arbitrary/constant_arbitrary.rb +6 -1
- data/lib/pbt/arbitrary/constant_from_arbitrary.rb +39 -0
- data/lib/pbt/arbitrary/filter_arbitrary.rb +41 -3
- data/lib/pbt/arbitrary/fixed_hash_arbitrary.rb +17 -1
- data/lib/pbt/arbitrary/float_arbitrary.rb +31 -0
- data/lib/pbt/arbitrary/frequency_arbitrary.rb +61 -0
- data/lib/pbt/arbitrary/integer_arbitrary.rb +12 -1
- data/lib/pbt/arbitrary/map_arbitrary.rb +41 -3
- data/lib/pbt/arbitrary/one_of_arbitrary.rb +47 -14
- data/lib/pbt/arbitrary/sample.rb +97 -0
- data/lib/pbt/arbitrary/tuple_arbitrary.rb +23 -1
- data/lib/pbt/check/case.rb +1 -1
- data/lib/pbt/check/configuration.rb +5 -3
- data/lib/pbt/check/property.rb +24 -5
- data/lib/pbt/check/runner_iterator.rb +84 -10
- data/lib/pbt/check/runner_methods.rb +47 -17
- data/lib/pbt/check/tosser.rb +1 -1
- data/lib/pbt/reporter/run_details.rb +2 -2
- data/lib/pbt/reporter/run_details_reporter.rb +13 -0
- data/lib/pbt/reporter/run_execution.rb +47 -4
- data/lib/pbt/stateful/property.rb +8 -102
- data/lib/pbt/statistics.rb +28 -0
- data/lib/pbt/version.rb +1 -1
- data/lib/pbt.rb +45 -9
- metadata +16 -3
|
@@ -2,27 +2,60 @@
|
|
|
2
2
|
|
|
3
3
|
module Pbt
|
|
4
4
|
module Arbitrary
|
|
5
|
-
#
|
|
5
|
+
# Uniformly selects one of the given arbitraries and delegates generation/shrinking to it.
|
|
6
6
|
class OneOfArbitrary < Arbitrary
|
|
7
|
-
# @param
|
|
8
|
-
def initialize(
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
# @param arbitraries [Array<Arbitrary>] Non-empty list of arbitraries to choose from.
|
|
8
|
+
def initialize(arbitraries)
|
|
9
|
+
raise ArgumentError, "one_of expects an Array of arbitraries" unless arbitraries.is_a?(Array)
|
|
10
|
+
raise EmptyDomainError, "one_of requires at least one arbitrary" if arbitraries.empty?
|
|
11
|
+
|
|
12
|
+
unless arbitraries.all? { |arb| arb.respond_to?(:generate) && arb.respond_to?(:shrink) }
|
|
13
|
+
raise ArgumentError, "one_of expects arbitraries (use Pbt.constant_from for constant values)"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
@arbitraries = arbitraries.dup
|
|
11
17
|
end
|
|
12
18
|
|
|
13
19
|
# @see Arbitrary#generate
|
|
14
|
-
def generate(rng)
|
|
15
|
-
|
|
20
|
+
def generate(rng = Random.new)
|
|
21
|
+
select(rng).generate(rng)
|
|
16
22
|
end
|
|
17
23
|
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
# Preserve the selected branch for shrinking, even after subsequent generation.
|
|
25
|
+
#
|
|
26
|
+
# @see Arbitrary#sample
|
|
27
|
+
def sample(rng = Random.new)
|
|
28
|
+
Sample.capture(select(rng), rng)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @private
|
|
32
|
+
def contextual?
|
|
33
|
+
true
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @see Arbitrary#cover?
|
|
37
|
+
def cover?(value)
|
|
38
|
+
unknown = false
|
|
39
|
+
@arbitraries.each do |arb|
|
|
40
|
+
covered = Arbitrary.cover_result(arb, value)
|
|
41
|
+
return true if covered == true
|
|
42
|
+
|
|
43
|
+
unknown = true if covered.nil?
|
|
25
44
|
end
|
|
45
|
+
unknown ? nil : false
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# A bare value cannot identify its source arbitrary. Use `sample` or `Pbt.property` to shrink.
|
|
49
|
+
#
|
|
50
|
+
# @see Arbitrary#shrink
|
|
51
|
+
def shrink(_current)
|
|
52
|
+
[].each
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def select(rng)
|
|
58
|
+
@arbitraries[rng.rand(@arbitraries.size)]
|
|
26
59
|
end
|
|
27
60
|
end
|
|
28
61
|
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pbt
|
|
4
|
+
module Arbitrary
|
|
5
|
+
# A generated value and its lazy shrink tree. Each sample owns its provenance.
|
|
6
|
+
class Sample
|
|
7
|
+
attr_reader :value
|
|
8
|
+
|
|
9
|
+
def initialize(value, &shrinker)
|
|
10
|
+
@value = value
|
|
11
|
+
@shrinker = shrinker
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def shrink
|
|
15
|
+
Enumerator.new { |y| @shrinker.call.each { |child| y << child } }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def map(&mapper)
|
|
19
|
+
Sample.new(mapper.call(value)) do
|
|
20
|
+
shrink.lazy.map { |child| child.map(&mapper) }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def filter(&predicate)
|
|
25
|
+
Sample.new(value) do
|
|
26
|
+
shrink.lazy.select { |child| predicate.call(child.value) }.map { |child| child.filter(&predicate) }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Compatibility with user-defined generate/shrink objects.
|
|
31
|
+
def self.capture(arbitrary, rng)
|
|
32
|
+
if arbitrary.respond_to?(:sample)
|
|
33
|
+
arbitrary.sample(rng)
|
|
34
|
+
else
|
|
35
|
+
from_value(arbitrary, arbitrary.generate(rng))
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.from_value(arbitrary, value)
|
|
40
|
+
Sample.new(value) do
|
|
41
|
+
arbitrary.shrink(value).lazy.map { |child| from_value(arbitrary, child) }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.contextual?(arbitrary)
|
|
46
|
+
arbitrary.respond_to?(:contextual?) && arbitrary.contextual?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.tuple(samples)
|
|
50
|
+
Sample.new(samples.map(&:value)) do
|
|
51
|
+
Enumerator.new do |y|
|
|
52
|
+
samples.each_with_index do |sample, index|
|
|
53
|
+
sample.shrink.each do |child|
|
|
54
|
+
next_samples = samples.dup
|
|
55
|
+
next_samples[index] = child
|
|
56
|
+
y << tuple(next_samples)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def self.array(samples, length_arbitrary, min_length)
|
|
64
|
+
Sample.new(samples.map(&:value)) do
|
|
65
|
+
Enumerator.new do |y|
|
|
66
|
+
if samples.size > min_length
|
|
67
|
+
emitted_n_minus_1 = false
|
|
68
|
+
length_arbitrary.shrink(samples.size).each do |length|
|
|
69
|
+
emitted_n_minus_1 = true if length == samples.size - 1
|
|
70
|
+
if length.zero?
|
|
71
|
+
y << array([], length_arbitrary, min_length)
|
|
72
|
+
else
|
|
73
|
+
samples.each_cons(length) { |items| y << array(items, length_arbitrary, min_length) }
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
if samples.size >= 3
|
|
78
|
+
indices = emitted_n_minus_1 ? (1...(samples.size - 1)) : (0...samples.size)
|
|
79
|
+
indices.each do |i|
|
|
80
|
+
y << array([*samples[...i], *samples[i + 1..]], length_arbitrary, min_length)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
samples.each_with_index do |sample, index|
|
|
86
|
+
sample.shrink.each do |child|
|
|
87
|
+
next_samples = samples.dup
|
|
88
|
+
next_samples[index] = child
|
|
89
|
+
y << array(next_samples, length_arbitrary, min_length)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -10,10 +10,32 @@ module Pbt
|
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
# @see Arbitrary#generate
|
|
13
|
-
def generate(rng)
|
|
13
|
+
def generate(rng = Random.new)
|
|
14
14
|
@arbs.map { |arb| arb.generate(rng) }
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
def contextual?
|
|
18
|
+
@arbs.any? { |arb| Sample.contextual?(arb) }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def sample(rng = Random.new)
|
|
22
|
+
Sample.tuple(@arbs.map { |arb| Sample.capture(arb, rng) })
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @see Arbitrary#cover?
|
|
26
|
+
def cover?(value)
|
|
27
|
+
return false unless value.is_a?(Array) && value.size == @arbs.size
|
|
28
|
+
|
|
29
|
+
unknown = false
|
|
30
|
+
@arbs.each_with_index do |arb, idx|
|
|
31
|
+
covered = Arbitrary.cover_result(arb, value[idx])
|
|
32
|
+
return false if covered == false
|
|
33
|
+
|
|
34
|
+
unknown = true if covered.nil?
|
|
35
|
+
end
|
|
36
|
+
unknown ? nil : true
|
|
37
|
+
end
|
|
38
|
+
|
|
17
39
|
# @see Arbitrary#shrink
|
|
18
40
|
def shrink(current)
|
|
19
41
|
# This is not the most comprehensive but allows a reasonable number of entries in the shrink.
|
data/lib/pbt/check/case.rb
CHANGED
|
@@ -9,19 +9,21 @@ module Pbt
|
|
|
9
9
|
:num_runs,
|
|
10
10
|
:seed,
|
|
11
11
|
:thread_report_on_exception,
|
|
12
|
-
|
|
12
|
+
:max_skips
|
|
13
13
|
) do
|
|
14
14
|
# @param verbose [Boolean] Whether to print verbose output. Default is `false`.
|
|
15
|
-
# @param worker [Symbol] The concurrency method to use.
|
|
15
|
+
# @param worker [Symbol] The concurrency method to use. `:ractor` and `:none` are supported. Default is `:none`.
|
|
16
16
|
# @param num_runs [Integer] The number of runs to perform. Default is `100`.
|
|
17
17
|
# @param seed [Integer] The seed to use for random number generation. It's useful to reproduce failed test with the seed you'd pick up from failure messages. Default is a random seed.
|
|
18
18
|
# @param thread_report_on_exception [Boolean] Whether to report exceptions in threads. It's useful to suppress error logs on Ractor that reports many errors. Default is `false`.
|
|
19
|
+
# @param max_skips [Integer, nil] Maximum number of skipped cases allowed via `Pbt.assume`. Defaults to `num_runs * 10` when `nil`.
|
|
19
20
|
def initialize(
|
|
20
21
|
verbose: false,
|
|
21
22
|
worker: :none,
|
|
22
23
|
num_runs: 100,
|
|
23
24
|
seed: Random.new_seed,
|
|
24
|
-
thread_report_on_exception: false
|
|
25
|
+
thread_report_on_exception: false,
|
|
26
|
+
max_skips: nil
|
|
25
27
|
)
|
|
26
28
|
super
|
|
27
29
|
end
|
data/lib/pbt/check/property.rb
CHANGED
|
@@ -5,10 +5,14 @@ module Pbt
|
|
|
5
5
|
# Represents a property to be tested.
|
|
6
6
|
# This class holds an arbitrary to generate values and a predicate to test them.
|
|
7
7
|
class Property
|
|
8
|
-
# @param arb [
|
|
8
|
+
# @param arb [Arbitrary]
|
|
9
|
+
# @param keyword_args [Boolean] Whether generated Hash values should be splatted as keyword arguments.
|
|
10
|
+
# @param splat_args [Boolean] Whether generated tuple values should be splatted as positional arguments.
|
|
9
11
|
# @param predicate [Proc] Predicate proc to test the generated values. Library users write this.
|
|
10
|
-
def initialize(arb, &predicate)
|
|
12
|
+
def initialize(arb, keyword_args: false, splat_args: false, &predicate)
|
|
11
13
|
@arb = arb
|
|
14
|
+
@keyword_args = keyword_args
|
|
15
|
+
@splat_args = splat_args
|
|
12
16
|
@predicate = predicate
|
|
13
17
|
end
|
|
14
18
|
|
|
@@ -16,10 +20,19 @@ module Pbt
|
|
|
16
20
|
#
|
|
17
21
|
# @param rng [Random] Random number generator.
|
|
18
22
|
# @return [Object]
|
|
19
|
-
def generate(rng)
|
|
23
|
+
def generate(rng = Random.new)
|
|
20
24
|
@arb.generate(rng)
|
|
21
25
|
end
|
|
22
26
|
|
|
27
|
+
# @private Generate with provenance only when a composite needs it.
|
|
28
|
+
def generate_for_check(rng)
|
|
29
|
+
if Arbitrary::Sample.contextual?(@arb)
|
|
30
|
+
Arbitrary::Sample.capture(@arb, rng)
|
|
31
|
+
else
|
|
32
|
+
generate(rng)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
23
36
|
# Shrink the `val` to a smaller one.
|
|
24
37
|
# This is used to find the smallest failing case after a failure.
|
|
25
38
|
#
|
|
@@ -34,8 +47,10 @@ module Pbt
|
|
|
34
47
|
# @param val [Object]
|
|
35
48
|
# @return [void]
|
|
36
49
|
def run(val)
|
|
37
|
-
if
|
|
50
|
+
if @keyword_args
|
|
38
51
|
@predicate.call(**val)
|
|
52
|
+
elsif @splat_args
|
|
53
|
+
@predicate.call(*val)
|
|
39
54
|
else
|
|
40
55
|
@predicate.call(val)
|
|
41
56
|
end
|
|
@@ -47,7 +62,11 @@ module Pbt
|
|
|
47
62
|
# @param val [Object]
|
|
48
63
|
# @return [Ractor]
|
|
49
64
|
def run_in_ractor(val)
|
|
50
|
-
|
|
65
|
+
if @splat_args
|
|
66
|
+
Ractor.new(*val, &@predicate)
|
|
67
|
+
else
|
|
68
|
+
Ractor.new(val, &@predicate)
|
|
69
|
+
end
|
|
51
70
|
end
|
|
52
71
|
end
|
|
53
72
|
end
|
|
@@ -12,13 +12,20 @@ module Pbt
|
|
|
12
12
|
class RunnerIterator < DelegateClass(Array)
|
|
13
13
|
attr_reader :run_execution
|
|
14
14
|
|
|
15
|
-
# @param source_values [Enumerator] Enumerator of generated values by arbitrary.
|
|
15
|
+
# @param source_values [Enumerator] Enumerator of generated values by arbitrary.
|
|
16
16
|
# @param property [Property] Property to test. This is used to shrink the failed case.
|
|
17
17
|
# @param verbose [Boolean] Controls the verbosity of the output.
|
|
18
|
-
|
|
18
|
+
# @param target_runs [Integer, nil] Number of non-skipped runs to complete before stopping.
|
|
19
|
+
# @param max_skips [Integer, nil] Maximum number of skipped initial runs allowed via `Pbt.assume`.
|
|
20
|
+
def initialize(source_values, property, verbose, target_runs: nil, max_skips: nil)
|
|
19
21
|
@run_execution = Reporter::RunExecution.new(verbose)
|
|
20
22
|
@property = property
|
|
21
|
-
@
|
|
23
|
+
@source_stream = source_values
|
|
24
|
+
@target_runs = target_runs
|
|
25
|
+
@max_skips = max_skips
|
|
26
|
+
@initial_failure = nil
|
|
27
|
+
@initial_offset = 0
|
|
28
|
+
@next_values = @target_runs ? next_initial_batch(@target_runs) : source_values
|
|
22
29
|
enumerator = Enumerator.new do |y|
|
|
23
30
|
loop do
|
|
24
31
|
y.yield @next_values.next
|
|
@@ -32,26 +39,93 @@ module Pbt
|
|
|
32
39
|
#
|
|
33
40
|
# @return [Boolean]
|
|
34
41
|
def has_next?
|
|
35
|
-
@
|
|
36
|
-
|
|
37
|
-
|
|
42
|
+
return false if @run_execution.too_many_skips?
|
|
43
|
+
|
|
44
|
+
begin
|
|
45
|
+
@next_values.peek
|
|
46
|
+
true
|
|
47
|
+
rescue StopIteration
|
|
48
|
+
if @target_runs && @initial_failure.nil?
|
|
49
|
+
remaining = @target_runs - @run_execution.num_successes
|
|
50
|
+
if remaining.positive?
|
|
51
|
+
@next_values = next_initial_batch(remaining)
|
|
52
|
+
@next_values.peek
|
|
53
|
+
return true
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
false
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Map a pass-local index to the stream index during initial runs, or keep local index during shrinking.
|
|
61
|
+
#
|
|
62
|
+
# @param local_index [Integer]
|
|
63
|
+
# @return [Integer]
|
|
64
|
+
def case_index(local_index)
|
|
65
|
+
(@target_runs && @initial_failure.nil?) ? (@initial_offset + local_index) : local_index
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Handle a case discarded by `Pbt.assume`.
|
|
69
|
+
# During shrinking, skipped candidates are ignored without counting toward `max_skips`.
|
|
70
|
+
#
|
|
71
|
+
# @return [Boolean] whether the skip budget was exceeded and execution should stop
|
|
72
|
+
def handle_skip
|
|
73
|
+
return false if @initial_failure
|
|
74
|
+
|
|
75
|
+
@run_execution.record_skip
|
|
76
|
+
if @max_skips && @run_execution.num_skips > @max_skips
|
|
77
|
+
error = TooManySkipsError.new(
|
|
78
|
+
"Too many skipped runs: #{@run_execution.num_skips} skip(s) while attempting #{@target_runs} run(s) (#{@run_execution.num_successes} passed)"
|
|
79
|
+
)
|
|
80
|
+
@run_execution.record_too_many_skips(error)
|
|
81
|
+
return true
|
|
82
|
+
end
|
|
83
|
+
|
|
38
84
|
false
|
|
39
85
|
end
|
|
40
86
|
|
|
41
87
|
# Handle result of a test.
|
|
42
|
-
# When a test is successful, it records the success.
|
|
43
|
-
# When a test
|
|
88
|
+
# When a test is successful, it records the success and returns false.
|
|
89
|
+
# When a test fails with an accepted failure, it records the failure, sets up the next shrink values,
|
|
90
|
+
# and returns true. When a shrink candidate fails with an unrelated error according to `property#same_failure?`,
|
|
91
|
+
# it is ignored and returns false so shrinking continues on the current level.
|
|
92
|
+
# Properties implementing `shrink_case(c)` must return an `Enumerator` (since `#has_next?` calls `#peek`).
|
|
44
93
|
#
|
|
45
94
|
# @param c [Case]
|
|
46
|
-
# @return [
|
|
95
|
+
# @return [Boolean] whether the failure was accepted
|
|
47
96
|
def handle_result(c)
|
|
48
97
|
if c.exception
|
|
98
|
+
if @initial_failure && @property.respond_to?(:same_failure?) && !@property.same_failure?(@initial_failure, c.exception)
|
|
99
|
+
return false
|
|
100
|
+
end
|
|
101
|
+
|
|
49
102
|
# failed run
|
|
103
|
+
@initial_failure ||= c.exception
|
|
50
104
|
@run_execution.record_failure(c)
|
|
51
|
-
@next_values =
|
|
105
|
+
@next_values =
|
|
106
|
+
if @property.respond_to?(:shrink_case)
|
|
107
|
+
@property.shrink_case(c)
|
|
108
|
+
elsif c.sample
|
|
109
|
+
c.sample.shrink
|
|
110
|
+
else
|
|
111
|
+
@property.shrink(c.val)
|
|
112
|
+
end
|
|
113
|
+
true
|
|
52
114
|
else
|
|
53
115
|
# successful run
|
|
54
116
|
@run_execution.record_success(c)
|
|
117
|
+
false
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def next_initial_batch(count)
|
|
124
|
+
@initial_offset = @run_execution.num_successes + @run_execution.num_skips
|
|
125
|
+
Enumerator.new(count) do |y|
|
|
126
|
+
count.times do
|
|
127
|
+
y.yield @source_stream.next
|
|
128
|
+
end
|
|
55
129
|
end
|
|
56
130
|
end
|
|
57
131
|
end
|
|
@@ -7,6 +7,9 @@ require "pbt/reporter/run_details_reporter"
|
|
|
7
7
|
|
|
8
8
|
module Pbt
|
|
9
9
|
module Check
|
|
10
|
+
# Exceptions that must never be treated as property failures or swallowed during shrinking.
|
|
11
|
+
FATAL_EXCEPTIONS = [SystemExit, SignalException, NoMemoryError].freeze
|
|
12
|
+
|
|
10
13
|
# Module to be
|
|
11
14
|
module RunnerMethods
|
|
12
15
|
include Check::Tosser
|
|
@@ -40,15 +43,15 @@ module Pbt
|
|
|
40
43
|
config[:worker] = :none
|
|
41
44
|
end
|
|
42
45
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
y.yield initial_values.next
|
|
47
|
-
end
|
|
46
|
+
max_skips = config[:max_skips]
|
|
47
|
+
if !max_skips.nil? && !(max_skips.is_a?(Integer) && max_skips >= 0)
|
|
48
|
+
raise Pbt::InvalidConfiguration, "max_skips must be a non-negative Integer or nil"
|
|
48
49
|
end
|
|
49
50
|
|
|
51
|
+
initial_values = toss(property, config[:seed])
|
|
52
|
+
|
|
50
53
|
setup_for_ractor(config, property) do
|
|
51
|
-
run_it(property,
|
|
54
|
+
run_it(property, initial_values, config).to_run_details(config)
|
|
52
55
|
end
|
|
53
56
|
end
|
|
54
57
|
|
|
@@ -78,11 +81,18 @@ module Pbt
|
|
|
78
81
|
# Run the property test for each value.
|
|
79
82
|
#
|
|
80
83
|
# @param property [Property] Property to test.
|
|
81
|
-
# @param source_values [Enumerator]
|
|
84
|
+
# @param source_values [Enumerator] Infinite enumerator of values to test.
|
|
82
85
|
# @param config [Hash] Configuration parameters.
|
|
83
86
|
# @return [RunExecution] Result of the test.
|
|
84
87
|
def run_it(property, source_values, config)
|
|
85
|
-
|
|
88
|
+
max_skips = config[:max_skips].nil? ? config[:num_runs] * 10 : config[:max_skips]
|
|
89
|
+
runner = Check::RunnerIterator.new(
|
|
90
|
+
source_values,
|
|
91
|
+
property,
|
|
92
|
+
config[:verbose],
|
|
93
|
+
target_runs: config[:num_runs],
|
|
94
|
+
max_skips: max_skips
|
|
95
|
+
)
|
|
86
96
|
while runner.has_next?
|
|
87
97
|
case config[:worker]
|
|
88
98
|
in :none
|
|
@@ -99,15 +109,18 @@ module Pbt
|
|
|
99
109
|
# @return [void]
|
|
100
110
|
def run_it_in_sequential(property, runner)
|
|
101
111
|
runner.each_with_index do |val, index|
|
|
102
|
-
c =
|
|
112
|
+
c = build_case(val, runner.case_index(index))
|
|
103
113
|
begin
|
|
104
|
-
property.run(val)
|
|
114
|
+
property.run(c.val)
|
|
105
115
|
runner.handle_result(c)
|
|
106
|
-
|
|
116
|
+
rescue PreconditionNotMet
|
|
117
|
+
break if runner.handle_skip
|
|
118
|
+
rescue *FATAL_EXCEPTIONS
|
|
119
|
+
raise
|
|
120
|
+
# Catch all other exceptions including RSpec's ExpectationNotMet (It inherits Exception).
|
|
107
121
|
rescue Exception => e # standard:disable Lint/RescueException:
|
|
108
122
|
c.exception = e
|
|
109
|
-
runner.handle_result(c)
|
|
110
|
-
break # Ignore the rest of the cases. Just pick up the first failure.
|
|
123
|
+
break if runner.handle_result(c) # Ignore the rest of the cases when failure is accepted.
|
|
111
124
|
end
|
|
112
125
|
end
|
|
113
126
|
end
|
|
@@ -117,14 +130,31 @@ module Pbt
|
|
|
117
130
|
# @return [void]
|
|
118
131
|
def run_it_in_ractors(property, runner)
|
|
119
132
|
runner.map.with_index { |val, index|
|
|
120
|
-
|
|
133
|
+
c = build_case(val, runner.case_index(index))
|
|
134
|
+
c.ractor = property.run_in_ractor(c.val)
|
|
135
|
+
c
|
|
121
136
|
}.each do |c|
|
|
122
137
|
wait_ractor_result(c.ractor)
|
|
123
138
|
runner.handle_result(c)
|
|
124
139
|
rescue => e
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
140
|
+
cause = unwrap_ractor_exception(e)
|
|
141
|
+
if cause.is_a?(PreconditionNotMet)
|
|
142
|
+
break if runner.handle_skip
|
|
143
|
+
|
|
144
|
+
next
|
|
145
|
+
end
|
|
146
|
+
raise cause if FATAL_EXCEPTIONS.any? { |klass| cause.is_a?(klass) }
|
|
147
|
+
|
|
148
|
+
c.exception = cause
|
|
149
|
+
break if runner.handle_result(c) # Ignore the rest of the cases when failure is accepted.
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def build_case(value, index)
|
|
154
|
+
if value.is_a?(Arbitrary::Sample)
|
|
155
|
+
Case.new(val: value.value, sample: value, index: index)
|
|
156
|
+
else
|
|
157
|
+
Case.new(val: value, index: index)
|
|
128
158
|
end
|
|
129
159
|
end
|
|
130
160
|
|
data/lib/pbt/check/tosser.rb
CHANGED
|
@@ -25,7 +25,7 @@ module Pbt
|
|
|
25
25
|
# @param arb [Arbitrary] Arbitrary to generate a value.
|
|
26
26
|
# @param rng [Random] Random number generator.
|
|
27
27
|
def toss_next(arb, rng)
|
|
28
|
-
arb.generate(rng)
|
|
28
|
+
arb.respond_to?(:generate_for_check) ? arb.generate_for_check(rng) : arb.generate(rng)
|
|
29
29
|
end
|
|
30
30
|
end
|
|
31
31
|
end
|
|
@@ -6,6 +6,7 @@ module Pbt
|
|
|
6
6
|
RunDetails = Struct.new(
|
|
7
7
|
:failed,
|
|
8
8
|
:num_runs,
|
|
9
|
+
:num_skips,
|
|
9
10
|
:num_shrinks,
|
|
10
11
|
:seed,
|
|
11
12
|
:counterexample,
|
|
@@ -15,8 +16,7 @@ module Pbt
|
|
|
15
16
|
:failures,
|
|
16
17
|
:verbose,
|
|
17
18
|
:execution_summary,
|
|
18
|
-
:run_configuration
|
|
19
|
-
keyword_init: true
|
|
19
|
+
:run_configuration
|
|
20
20
|
)
|
|
21
21
|
end
|
|
22
22
|
end
|
|
@@ -15,6 +15,10 @@ module Pbt
|
|
|
15
15
|
# @raise [PropertyFailure]
|
|
16
16
|
def report_run_details
|
|
17
17
|
if @run_details.failed
|
|
18
|
+
if @run_details.error_instance.is_a?(TooManySkipsError)
|
|
19
|
+
raise TooManySkipsError, format_too_many_skips_message
|
|
20
|
+
end
|
|
21
|
+
|
|
18
22
|
message = []
|
|
19
23
|
message << format_error_message
|
|
20
24
|
message << error_backtrace
|
|
@@ -28,6 +32,15 @@ module Pbt
|
|
|
28
32
|
|
|
29
33
|
private
|
|
30
34
|
|
|
35
|
+
# @return [String]
|
|
36
|
+
def format_too_many_skips_message
|
|
37
|
+
<<~MSG.chomp
|
|
38
|
+
#{@run_details.error_message}
|
|
39
|
+
seed: #{@run_details.seed}
|
|
40
|
+
Consider narrowing your arbitrary instead of relying heavily on Pbt.assume.
|
|
41
|
+
MSG
|
|
42
|
+
end
|
|
43
|
+
|
|
31
44
|
# @return [String]
|
|
32
45
|
def format_error_message
|
|
33
46
|
<<~MSG.chomp
|