pbt 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.standard.yml +1 -1
- data/CHANGELOG.md +15 -0
- data/CLAUDE.md +2 -2
- data/README.md +94 -17
- data/benchmark/FIBER.md +97 -0
- data/benchmark/fiber_experiment.rb +39 -0
- data/benchmark/fiber_scheduler/Gemfile +7 -0
- data/benchmark/fiber_scheduler/experiment.rb +131 -0
- data/benchmark/fiber_scheduler/spec/experiment_spec.rb +38 -0
- data/docs/plans/fiber-experiment.md +12 -0
- data/docs/plans/generator-statistics.md +16 -0
- data/docs/plans/stateful-pbt-redesign.md +415 -0
- data/lib/pbt/arbitrary/arbitrary.rb +63 -10
- data/lib/pbt/arbitrary/arbitrary_methods.rb +51 -24
- data/lib/pbt/arbitrary/array_arbitrary.rb +45 -11
- data/lib/pbt/arbitrary/choose_arbitrary.rb +27 -0
- data/lib/pbt/arbitrary/constant.rb +0 -1
- data/lib/pbt/arbitrary/constant_arbitrary.rb +5 -0
- data/lib/pbt/arbitrary/constant_from_arbitrary.rb +39 -0
- data/lib/pbt/arbitrary/filter_arbitrary.rb +40 -2
- data/lib/pbt/arbitrary/fixed_hash_arbitrary.rb +16 -0
- data/lib/pbt/arbitrary/float_arbitrary.rb +31 -0
- data/lib/pbt/arbitrary/frequency_arbitrary.rb +61 -0
- data/lib/pbt/arbitrary/integer_arbitrary.rb +11 -0
- data/lib/pbt/arbitrary/map_arbitrary.rb +40 -2
- data/lib/pbt/arbitrary/one_of_arbitrary.rb +46 -13
- data/lib/pbt/arbitrary/sample.rb +97 -0
- data/lib/pbt/arbitrary/tuple_arbitrary.rb +22 -0
- data/lib/pbt/check/case.rb +1 -1
- data/lib/pbt/check/configuration.rb +5 -3
- data/lib/pbt/check/property.rb +23 -4
- data/lib/pbt/check/runner_iterator.rb +84 -10
- data/lib/pbt/check/runner_methods.rb +47 -17
- data/lib/pbt/check/tosser.rb +1 -1
- data/lib/pbt/reporter/run_details.rb +2 -2
- data/lib/pbt/reporter/run_details_reporter.rb +13 -0
- data/lib/pbt/reporter/run_execution.rb +47 -4
- data/lib/pbt/stateful/property.rb +5 -1
- data/lib/pbt/statistics.rb +28 -0
- data/lib/pbt/version.rb +1 -1
- data/lib/pbt.rb +45 -9
- metadata +16 -3
|
@@ -6,12 +6,13 @@ module Pbt
|
|
|
6
6
|
module Reporter
|
|
7
7
|
# Represents the result of a single run of a property test.
|
|
8
8
|
class RunExecution
|
|
9
|
+
attr_reader :num_successes, :num_skips
|
|
10
|
+
|
|
9
11
|
# A tree node of the execution for verbose output.
|
|
10
12
|
ExecutionTreeNode = Struct.new(
|
|
11
13
|
:result,
|
|
12
14
|
:value,
|
|
13
|
-
:children
|
|
14
|
-
keyword_init: true
|
|
15
|
+
:children
|
|
15
16
|
)
|
|
16
17
|
|
|
17
18
|
# @param verbose [Boolean] Whether to print verbose output.
|
|
@@ -19,8 +20,11 @@ module Pbt
|
|
|
19
20
|
@verbose = verbose
|
|
20
21
|
@path_to_failure = []
|
|
21
22
|
@failure = nil
|
|
23
|
+
@too_many_skips_error = nil
|
|
22
24
|
@failures = []
|
|
23
25
|
@num_successes = 0
|
|
26
|
+
@num_skips = 0
|
|
27
|
+
@num_runs_on_failure = nil
|
|
24
28
|
@root_execution_trees = []
|
|
25
29
|
@current_level_execution_trees = @root_execution_trees
|
|
26
30
|
end
|
|
@@ -29,6 +33,7 @@ module Pbt
|
|
|
29
33
|
#
|
|
30
34
|
# @param c [Pbt::Check::Case]
|
|
31
35
|
def record_failure(c)
|
|
36
|
+
@num_runs_on_failure ||= @num_successes + 1
|
|
32
37
|
if @verbose
|
|
33
38
|
current_tree = append_execution_tree_node(:failure, c.val)
|
|
34
39
|
@current_level_execution_trees = current_tree.children
|
|
@@ -52,11 +57,31 @@ module Pbt
|
|
|
52
57
|
@num_successes += 1
|
|
53
58
|
end
|
|
54
59
|
|
|
60
|
+
# Record a skipped initial run (from `Pbt.assume`).
|
|
61
|
+
#
|
|
62
|
+
# @return [void]
|
|
63
|
+
def record_skip
|
|
64
|
+
@num_skips += 1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Record a failure caused by exceeding `max_skips`.
|
|
68
|
+
#
|
|
69
|
+
# @param error [Pbt::TooManySkipsError]
|
|
70
|
+
# @return [void]
|
|
71
|
+
def record_too_many_skips(error)
|
|
72
|
+
@too_many_skips_error = error
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# @return [Boolean]
|
|
76
|
+
def too_many_skips?
|
|
77
|
+
!@too_many_skips_error.nil?
|
|
78
|
+
end
|
|
79
|
+
|
|
55
80
|
# Whether the test was successful.
|
|
56
81
|
#
|
|
57
82
|
# @return [Boolean]
|
|
58
83
|
def success?
|
|
59
|
-
!@failure
|
|
84
|
+
!@failure && !@too_many_skips_error
|
|
60
85
|
end
|
|
61
86
|
|
|
62
87
|
# Convert execution to run details.
|
|
@@ -68,6 +93,7 @@ module Pbt
|
|
|
68
93
|
RunDetails.new(
|
|
69
94
|
failed: false,
|
|
70
95
|
num_runs: @num_successes,
|
|
96
|
+
num_skips: @num_skips,
|
|
71
97
|
num_shrinks: 0,
|
|
72
98
|
seed: config[:seed],
|
|
73
99
|
counterexample: nil,
|
|
@@ -79,10 +105,27 @@ module Pbt
|
|
|
79
105
|
execution_summary: @root_execution_trees,
|
|
80
106
|
run_configuration: config
|
|
81
107
|
)
|
|
108
|
+
elsif @too_many_skips_error
|
|
109
|
+
RunDetails.new(
|
|
110
|
+
failed: true,
|
|
111
|
+
num_runs: @num_successes,
|
|
112
|
+
num_skips: @num_skips,
|
|
113
|
+
num_shrinks: 0,
|
|
114
|
+
seed: config[:seed],
|
|
115
|
+
counterexample: nil,
|
|
116
|
+
counterexample_path: nil,
|
|
117
|
+
error_message: @too_many_skips_error.message,
|
|
118
|
+
error_instance: @too_many_skips_error,
|
|
119
|
+
failures: @failures,
|
|
120
|
+
verbose: @verbose,
|
|
121
|
+
execution_summary: @root_execution_trees,
|
|
122
|
+
run_configuration: config
|
|
123
|
+
)
|
|
82
124
|
else
|
|
83
125
|
RunDetails.new(
|
|
84
126
|
failed: true,
|
|
85
|
-
num_runs: @
|
|
127
|
+
num_runs: @num_runs_on_failure,
|
|
128
|
+
num_skips: @num_skips,
|
|
86
129
|
num_shrinks: @path_to_failure.size - 1,
|
|
87
130
|
seed: config[:seed],
|
|
88
131
|
counterexample: @value,
|
|
@@ -16,7 +16,7 @@ module Pbt
|
|
|
16
16
|
verify!
|
|
17
17
|
].freeze
|
|
18
18
|
|
|
19
|
-
Step = Struct.new(:command, :args
|
|
19
|
+
Step = Struct.new(:command, :args) do
|
|
20
20
|
def inspect
|
|
21
21
|
"#<Pbt::Stateful::Step command=#{command_label}, args=#{args.inspect}>"
|
|
22
22
|
end
|
|
@@ -140,6 +140,8 @@ module Pbt
|
|
|
140
140
|
result:,
|
|
141
141
|
sut:
|
|
142
142
|
)
|
|
143
|
+
rescue *Check::FATAL_EXCEPTIONS
|
|
144
|
+
raise
|
|
143
145
|
rescue Exception => e # standard:disable Lint/RescueException:
|
|
144
146
|
raise e.class,
|
|
145
147
|
"stateful step #{index} (#{command_name(command)}): #{e.message} [args=#{args.inspect}]",
|
|
@@ -256,6 +258,8 @@ module Pbt
|
|
|
256
258
|
return args if applicable?(command, state, args, context:)
|
|
257
259
|
end
|
|
258
260
|
|
|
261
|
+
NoApplicableArgs
|
|
262
|
+
rescue Pbt::Arbitrary::EmptyDomainError
|
|
259
263
|
NoApplicableArgs
|
|
260
264
|
end
|
|
261
265
|
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pbt
|
|
4
|
+
# Distribution of an independent, seeded sample of generated values.
|
|
5
|
+
class Statistics
|
|
6
|
+
attr_reader :seed, :num_runs, :counts
|
|
7
|
+
|
|
8
|
+
def initialize(arbitrary, num_runs:, seed:, &classifier)
|
|
9
|
+
unless num_runs.is_a?(Integer) && num_runs >= 0
|
|
10
|
+
raise ArgumentError, "num_runs must be a nonnegative Integer"
|
|
11
|
+
end
|
|
12
|
+
@seed = seed
|
|
13
|
+
@num_runs = num_runs
|
|
14
|
+
@counts = Hash.new(0)
|
|
15
|
+
rng = Random.new(seed)
|
|
16
|
+
num_runs.times do
|
|
17
|
+
value = arbitrary.generate(rng)
|
|
18
|
+
label = classifier ? classifier.call(value) : value
|
|
19
|
+
@counts[label] += 1
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Percentages are not rounded. Empty samples return an empty Hash.
|
|
24
|
+
def percentages
|
|
25
|
+
counts.transform_values { |count| count * 100.0 / num_runs }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
data/lib/pbt/version.rb
CHANGED
data/lib/pbt.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "pbt/version"
|
|
4
|
+
require_relative "pbt/statistics"
|
|
4
5
|
require_relative "pbt/arbitrary/arbitrary_methods"
|
|
5
6
|
require_relative "pbt/check/runner_methods"
|
|
6
7
|
require_relative "pbt/check/property"
|
|
@@ -11,6 +12,12 @@ module Pbt
|
|
|
11
12
|
# Represents a property-based test failure.
|
|
12
13
|
class PropertyFailure < StandardError; end
|
|
13
14
|
|
|
15
|
+
# Represents a property-based test failure caused by exceeding the skip budget via `Pbt.assume`.
|
|
16
|
+
class TooManySkipsError < PropertyFailure; end
|
|
17
|
+
|
|
18
|
+
# Internal control-flow exception raised by `Pbt.assume` when a precondition is not met.
|
|
19
|
+
class PreconditionNotMet < StandardError; end
|
|
20
|
+
|
|
14
21
|
# Represents an invalid configuration.
|
|
15
22
|
class InvalidConfiguration < StandardError; end
|
|
16
23
|
|
|
@@ -18,7 +25,19 @@ module Pbt
|
|
|
18
25
|
extend Check::RunnerMethods
|
|
19
26
|
extend Check::ConfigurationMethods
|
|
20
27
|
|
|
21
|
-
#
|
|
28
|
+
# Discard the current generated test case when `condition` is falsy.
|
|
29
|
+
# Discarded cases do not count toward `num_runs` and are skipped during shrinking.
|
|
30
|
+
# Exceeding `max_skips` fails the property with `Pbt::TooManySkipsError`.
|
|
31
|
+
#
|
|
32
|
+
# @param condition [Object]
|
|
33
|
+
# @return [void]
|
|
34
|
+
# @raise [Pbt::PreconditionNotMet]
|
|
35
|
+
def self.assume(condition)
|
|
36
|
+
raise PreconditionNotMet unless condition
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` or `Pbt.check`.
|
|
40
|
+
# Predicates fail by raising an exception (including RSpec/Minitest assertions); returning `false` alone does not fail.
|
|
22
41
|
# Be aware that using both positional and keyword arguments is not supported.
|
|
23
42
|
#
|
|
24
43
|
# @example Basic usage
|
|
@@ -26,23 +45,33 @@ module Pbt
|
|
|
26
45
|
# # your test code here
|
|
27
46
|
# end
|
|
28
47
|
#
|
|
29
|
-
# @example Use multiple arbitraries
|
|
30
|
-
# Pbt.property(Pbt.
|
|
48
|
+
# @example Use multiple positional arbitraries
|
|
49
|
+
# Pbt.property(Pbt.ascii_string, Pbt.symbol) do |str, sym|
|
|
31
50
|
# # your test code here
|
|
32
51
|
# end
|
|
33
52
|
#
|
|
34
|
-
# @example Use
|
|
35
|
-
# Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x
|
|
53
|
+
# @example Use keyword arbitraries (under default `worker: :none`)
|
|
54
|
+
# Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x:, y:|
|
|
36
55
|
# # your test code here
|
|
37
56
|
# end
|
|
38
57
|
#
|
|
58
|
+
# @example Use keyword arbitraries with `worker: :ractor` (Ractor passes the Hash as a single argument)
|
|
59
|
+
# Pbt.property(x: Pbt.integer, y: Pbt.integer) do |h|
|
|
60
|
+
# # h[:x], h[:y]
|
|
61
|
+
# end
|
|
62
|
+
#
|
|
39
63
|
# @param args [Array<Arbitrary>] Arbitraries to generate values. You can pass one or more arbitraries.
|
|
40
64
|
# @param kwargs [Hash<Symbol,Arbitrary>] Arbitraries to generate values. You can pass arbitraries with keyword arguments.
|
|
41
65
|
# @param predicate [Proc] Test code that receives generated values and runs the test.
|
|
42
|
-
# @return [Property]
|
|
66
|
+
# @return [Check::Property]
|
|
43
67
|
def self.property(*args, **kwargs, &predicate)
|
|
44
68
|
arb = to_arbitrary(args, kwargs)
|
|
45
|
-
Check::Property.new(
|
|
69
|
+
Check::Property.new(
|
|
70
|
+
arb,
|
|
71
|
+
keyword_args: args.empty? && !kwargs.empty?,
|
|
72
|
+
splat_args: args.size > 1,
|
|
73
|
+
&predicate
|
|
74
|
+
)
|
|
46
75
|
end
|
|
47
76
|
|
|
48
77
|
# Create a stateful property-based test backed by a model and a SUT factory.
|
|
@@ -54,8 +83,8 @@ module Pbt
|
|
|
54
83
|
#
|
|
55
84
|
# Each command object is expected to provide:
|
|
56
85
|
# - `name`
|
|
57
|
-
# - `arguments`
|
|
58
|
-
# - `applicable?(state
|
|
86
|
+
# - `arguments(state)` -> an arbitrary
|
|
87
|
+
# - `applicable?(state, args)` -> bool
|
|
59
88
|
# - `next_state(state, args)`
|
|
60
89
|
# - `run!(sut, args)` -> result
|
|
61
90
|
# - `verify!(before_state:, after_state:, args:, result:, sut:)`
|
|
@@ -68,6 +97,13 @@ module Pbt
|
|
|
68
97
|
Stateful::Property.new(model:, sut:, max_steps:)
|
|
69
98
|
end
|
|
70
99
|
|
|
100
|
+
# Inspect an independent sample of generated values, optionally grouped by a classifier.
|
|
101
|
+
# Values/labels used as Hash keys must not be mutated during or after sampling.
|
|
102
|
+
# @return [Statistics] Counts, percentages and the seed used for replay.
|
|
103
|
+
def self.statistics(arbitrary, num_runs: 100, seed: Random.new_seed, &classifier)
|
|
104
|
+
Statistics.new(arbitrary, num_runs: num_runs, seed: seed, &classifier)
|
|
105
|
+
end
|
|
106
|
+
|
|
71
107
|
class << self
|
|
72
108
|
private
|
|
73
109
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pbt
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- ohbarye
|
|
@@ -23,12 +23,20 @@ files:
|
|
|
23
23
|
- LICENSE.txt
|
|
24
24
|
- README.md
|
|
25
25
|
- Rakefile
|
|
26
|
+
- benchmark/FIBER.md
|
|
26
27
|
- benchmark/README.md
|
|
27
28
|
- benchmark/example.txt
|
|
28
29
|
- benchmark/failure_simple.rb
|
|
30
|
+
- benchmark/fiber_experiment.rb
|
|
31
|
+
- benchmark/fiber_scheduler/Gemfile
|
|
32
|
+
- benchmark/fiber_scheduler/experiment.rb
|
|
33
|
+
- benchmark/fiber_scheduler/spec/experiment_spec.rb
|
|
29
34
|
- benchmark/success_cpu_bound.rb
|
|
30
35
|
- benchmark/success_io_bound.rb
|
|
31
36
|
- benchmark/success_simple.rb
|
|
37
|
+
- docs/plans/fiber-experiment.md
|
|
38
|
+
- docs/plans/generator-statistics.md
|
|
39
|
+
- docs/plans/stateful-pbt-redesign.md
|
|
32
40
|
- lib/pbt.rb
|
|
33
41
|
- lib/pbt/arbitrary/arbitrary.rb
|
|
34
42
|
- lib/pbt/arbitrary/arbitrary_methods.rb
|
|
@@ -36,11 +44,15 @@ files:
|
|
|
36
44
|
- lib/pbt/arbitrary/choose_arbitrary.rb
|
|
37
45
|
- lib/pbt/arbitrary/constant.rb
|
|
38
46
|
- lib/pbt/arbitrary/constant_arbitrary.rb
|
|
47
|
+
- lib/pbt/arbitrary/constant_from_arbitrary.rb
|
|
39
48
|
- lib/pbt/arbitrary/filter_arbitrary.rb
|
|
40
49
|
- lib/pbt/arbitrary/fixed_hash_arbitrary.rb
|
|
50
|
+
- lib/pbt/arbitrary/float_arbitrary.rb
|
|
51
|
+
- lib/pbt/arbitrary/frequency_arbitrary.rb
|
|
41
52
|
- lib/pbt/arbitrary/integer_arbitrary.rb
|
|
42
53
|
- lib/pbt/arbitrary/map_arbitrary.rb
|
|
43
54
|
- lib/pbt/arbitrary/one_of_arbitrary.rb
|
|
55
|
+
- lib/pbt/arbitrary/sample.rb
|
|
44
56
|
- lib/pbt/arbitrary/tuple_arbitrary.rb
|
|
45
57
|
- lib/pbt/check/case.rb
|
|
46
58
|
- lib/pbt/check/configuration.rb
|
|
@@ -52,6 +64,7 @@ files:
|
|
|
52
64
|
- lib/pbt/reporter/run_details_reporter.rb
|
|
53
65
|
- lib/pbt/reporter/run_execution.rb
|
|
54
66
|
- lib/pbt/stateful/property.rb
|
|
67
|
+
- lib/pbt/statistics.rb
|
|
55
68
|
- lib/pbt/version.rb
|
|
56
69
|
- sig/pbt.rbs
|
|
57
70
|
homepage: https://github.com/ohbarye/pbt
|
|
@@ -68,14 +81,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
68
81
|
requirements:
|
|
69
82
|
- - ">="
|
|
70
83
|
- !ruby/object:Gem::Version
|
|
71
|
-
version: '3.
|
|
84
|
+
version: '3.3'
|
|
72
85
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
73
86
|
requirements:
|
|
74
87
|
- - ">="
|
|
75
88
|
- !ruby/object:Gem::Version
|
|
76
89
|
version: '0'
|
|
77
90
|
requirements: []
|
|
78
|
-
rubygems_version:
|
|
91
|
+
rubygems_version: 4.0.20
|
|
79
92
|
specification_version: 4
|
|
80
93
|
summary: Property-Based Testing tool for Ruby, utilizing Ractor for parallelizing
|
|
81
94
|
test cases.
|