pbt 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +1 -1
  3. data/CHANGELOG.md +15 -0
  4. data/CLAUDE.md +2 -2
  5. data/README.md +94 -17
  6. data/benchmark/FIBER.md +97 -0
  7. data/benchmark/fiber_experiment.rb +39 -0
  8. data/benchmark/fiber_scheduler/Gemfile +7 -0
  9. data/benchmark/fiber_scheduler/experiment.rb +131 -0
  10. data/benchmark/fiber_scheduler/spec/experiment_spec.rb +38 -0
  11. data/docs/plans/fiber-experiment.md +12 -0
  12. data/docs/plans/generator-statistics.md +16 -0
  13. data/docs/plans/stateful-pbt-redesign.md +415 -0
  14. data/lib/pbt/arbitrary/arbitrary.rb +63 -10
  15. data/lib/pbt/arbitrary/arbitrary_methods.rb +51 -24
  16. data/lib/pbt/arbitrary/array_arbitrary.rb +45 -11
  17. data/lib/pbt/arbitrary/choose_arbitrary.rb +27 -0
  18. data/lib/pbt/arbitrary/constant.rb +0 -1
  19. data/lib/pbt/arbitrary/constant_arbitrary.rb +5 -0
  20. data/lib/pbt/arbitrary/constant_from_arbitrary.rb +39 -0
  21. data/lib/pbt/arbitrary/filter_arbitrary.rb +40 -2
  22. data/lib/pbt/arbitrary/fixed_hash_arbitrary.rb +16 -0
  23. data/lib/pbt/arbitrary/float_arbitrary.rb +31 -0
  24. data/lib/pbt/arbitrary/frequency_arbitrary.rb +61 -0
  25. data/lib/pbt/arbitrary/integer_arbitrary.rb +11 -0
  26. data/lib/pbt/arbitrary/map_arbitrary.rb +40 -2
  27. data/lib/pbt/arbitrary/one_of_arbitrary.rb +46 -13
  28. data/lib/pbt/arbitrary/sample.rb +97 -0
  29. data/lib/pbt/arbitrary/tuple_arbitrary.rb +22 -0
  30. data/lib/pbt/check/case.rb +1 -1
  31. data/lib/pbt/check/configuration.rb +5 -3
  32. data/lib/pbt/check/property.rb +23 -4
  33. data/lib/pbt/check/runner_iterator.rb +84 -10
  34. data/lib/pbt/check/runner_methods.rb +47 -17
  35. data/lib/pbt/check/tosser.rb +1 -1
  36. data/lib/pbt/reporter/run_details.rb +2 -2
  37. data/lib/pbt/reporter/run_details_reporter.rb +13 -0
  38. data/lib/pbt/reporter/run_execution.rb +47 -4
  39. data/lib/pbt/stateful/property.rb +5 -1
  40. data/lib/pbt/statistics.rb +28 -0
  41. data/lib/pbt/version.rb +1 -1
  42. data/lib/pbt.rb +45 -9
  43. metadata +16 -3
@@ -6,12 +6,13 @@ module Pbt
6
6
  module Reporter
7
7
  # Represents the result of a single run of a property test.
8
8
  class RunExecution
9
+ attr_reader :num_successes, :num_skips
10
+
9
11
  # A tree node of the execution for verbose output.
10
12
  ExecutionTreeNode = Struct.new(
11
13
  :result,
12
14
  :value,
13
- :children,
14
- keyword_init: true
15
+ :children
15
16
  )
16
17
 
17
18
  # @param verbose [Boolean] Whether to print verbose output.
@@ -19,8 +20,11 @@ module Pbt
19
20
  @verbose = verbose
20
21
  @path_to_failure = []
21
22
  @failure = nil
23
+ @too_many_skips_error = nil
22
24
  @failures = []
23
25
  @num_successes = 0
26
+ @num_skips = 0
27
+ @num_runs_on_failure = nil
24
28
  @root_execution_trees = []
25
29
  @current_level_execution_trees = @root_execution_trees
26
30
  end
@@ -29,6 +33,7 @@ module Pbt
29
33
  #
30
34
  # @param c [Pbt::Check::Case]
31
35
  def record_failure(c)
36
+ @num_runs_on_failure ||= @num_successes + 1
32
37
  if @verbose
33
38
  current_tree = append_execution_tree_node(:failure, c.val)
34
39
  @current_level_execution_trees = current_tree.children
@@ -52,11 +57,31 @@ module Pbt
52
57
  @num_successes += 1
53
58
  end
54
59
 
60
+ # Record a skipped initial run (from `Pbt.assume`).
61
+ #
62
+ # @return [void]
63
+ def record_skip
64
+ @num_skips += 1
65
+ end
66
+
67
+ # Record a failure caused by exceeding `max_skips`.
68
+ #
69
+ # @param error [Pbt::TooManySkipsError]
70
+ # @return [void]
71
+ def record_too_many_skips(error)
72
+ @too_many_skips_error = error
73
+ end
74
+
75
+ # @return [Boolean]
76
+ def too_many_skips?
77
+ !@too_many_skips_error.nil?
78
+ end
79
+
55
80
  # Whether the test was successful.
56
81
  #
57
82
  # @return [Boolean]
58
83
  def success?
59
- !@failure
84
+ !@failure && !@too_many_skips_error
60
85
  end
61
86
 
62
87
  # Convert execution to run details.
@@ -68,6 +93,7 @@ module Pbt
68
93
  RunDetails.new(
69
94
  failed: false,
70
95
  num_runs: @num_successes,
96
+ num_skips: @num_skips,
71
97
  num_shrinks: 0,
72
98
  seed: config[:seed],
73
99
  counterexample: nil,
@@ -79,10 +105,27 @@ module Pbt
79
105
  execution_summary: @root_execution_trees,
80
106
  run_configuration: config
81
107
  )
108
+ elsif @too_many_skips_error
109
+ RunDetails.new(
110
+ failed: true,
111
+ num_runs: @num_successes,
112
+ num_skips: @num_skips,
113
+ num_shrinks: 0,
114
+ seed: config[:seed],
115
+ counterexample: nil,
116
+ counterexample_path: nil,
117
+ error_message: @too_many_skips_error.message,
118
+ error_instance: @too_many_skips_error,
119
+ failures: @failures,
120
+ verbose: @verbose,
121
+ execution_summary: @root_execution_trees,
122
+ run_configuration: config
123
+ )
82
124
  else
83
125
  RunDetails.new(
84
126
  failed: true,
85
- num_runs: @path_to_failure[0] + 1,
127
+ num_runs: @num_runs_on_failure,
128
+ num_skips: @num_skips,
86
129
  num_shrinks: @path_to_failure.size - 1,
87
130
  seed: config[:seed],
88
131
  counterexample: @value,
@@ -16,7 +16,7 @@ module Pbt
16
16
  verify!
17
17
  ].freeze
18
18
 
19
- Step = Struct.new(:command, :args, keyword_init: true) do
19
+ Step = Struct.new(:command, :args) do
20
20
  def inspect
21
21
  "#<Pbt::Stateful::Step command=#{command_label}, args=#{args.inspect}>"
22
22
  end
@@ -140,6 +140,8 @@ module Pbt
140
140
  result:,
141
141
  sut:
142
142
  )
143
+ rescue *Check::FATAL_EXCEPTIONS
144
+ raise
143
145
  rescue Exception => e # standard:disable Lint/RescueException:
144
146
  raise e.class,
145
147
  "stateful step #{index} (#{command_name(command)}): #{e.message} [args=#{args.inspect}]",
@@ -256,6 +258,8 @@ module Pbt
256
258
  return args if applicable?(command, state, args, context:)
257
259
  end
258
260
 
261
+ NoApplicableArgs
262
+ rescue Pbt::Arbitrary::EmptyDomainError
259
263
  NoApplicableArgs
260
264
  end
261
265
 
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pbt
4
+ # Distribution of an independent, seeded sample of generated values.
5
+ class Statistics
6
+ attr_reader :seed, :num_runs, :counts
7
+
8
+ def initialize(arbitrary, num_runs:, seed:, &classifier)
9
+ unless num_runs.is_a?(Integer) && num_runs >= 0
10
+ raise ArgumentError, "num_runs must be a nonnegative Integer"
11
+ end
12
+ @seed = seed
13
+ @num_runs = num_runs
14
+ @counts = Hash.new(0)
15
+ rng = Random.new(seed)
16
+ num_runs.times do
17
+ value = arbitrary.generate(rng)
18
+ label = classifier ? classifier.call(value) : value
19
+ @counts[label] += 1
20
+ end
21
+ end
22
+
23
+ # Percentages are not rounded. Empty samples return an empty Hash.
24
+ def percentages
25
+ counts.transform_values { |count| count * 100.0 / num_runs }
26
+ end
27
+ end
28
+ end
data/lib/pbt/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pbt
4
- VERSION = "0.7.0"
4
+ VERSION = "0.8.0"
5
5
  end
data/lib/pbt.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "pbt/version"
4
+ require_relative "pbt/statistics"
4
5
  require_relative "pbt/arbitrary/arbitrary_methods"
5
6
  require_relative "pbt/check/runner_methods"
6
7
  require_relative "pbt/check/property"
@@ -11,6 +12,12 @@ module Pbt
11
12
  # Represents a property-based test failure.
12
13
  class PropertyFailure < StandardError; end
13
14
 
15
+ # Represents a property-based test failure caused by exceeding the skip budget via `Pbt.assume`.
16
+ class TooManySkipsError < PropertyFailure; end
17
+
18
+ # Internal control-flow exception raised by `Pbt.assume` when a precondition is not met.
19
+ class PreconditionNotMet < StandardError; end
20
+
14
21
  # Represents an invalid configuration.
15
22
  class InvalidConfiguration < StandardError; end
16
23
 
@@ -18,7 +25,19 @@ module Pbt
18
25
  extend Check::RunnerMethods
19
26
  extend Check::ConfigurationMethods
20
27
 
21
- # Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` method.
28
+ # Discard the current generated test case when `condition` is falsy.
29
+ # Discarded cases do not count toward `num_runs` and are skipped during shrinking.
30
+ # Exceeding `max_skips` fails the property with `Pbt::TooManySkipsError`.
31
+ #
32
+ # @param condition [Object]
33
+ # @return [void]
34
+ # @raise [Pbt::PreconditionNotMet]
35
+ def self.assume(condition)
36
+ raise PreconditionNotMet unless condition
37
+ end
38
+
39
+ # Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` or `Pbt.check`.
40
+ # Predicates fail by raising an exception (including RSpec/Minitest assertions); returning `false` alone does not fail.
22
41
  # Be aware that using both positional and keyword arguments is not supported.
23
42
  #
24
43
  # @example Basic usage
@@ -26,23 +45,33 @@ module Pbt
26
45
  # # your test code here
27
46
  # end
28
47
  #
29
- # @example Use multiple arbitraries
30
- # Pbt.property(Pbt.string, Pbt.symbol) do |str, sym|
48
+ # @example Use multiple positional arbitraries
49
+ # Pbt.property(Pbt.ascii_string, Pbt.symbol) do |str, sym|
31
50
  # # your test code here
32
51
  # end
33
52
  #
34
- # @example Use hash arbitraries
35
- # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x, y|
53
+ # @example Use keyword arbitraries (under default `worker: :none`)
54
+ # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x:, y:|
36
55
  # # your test code here
37
56
  # end
38
57
  #
58
+ # @example Use keyword arbitraries with `worker: :ractor` (Ractor passes the Hash as a single argument)
59
+ # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |h|
60
+ # # h[:x], h[:y]
61
+ # end
62
+ #
39
63
  # @param args [Array<Arbitrary>] Arbitraries to generate values. You can pass one or more arbitraries.
40
64
  # @param kwargs [Hash<Symbol,Arbitrary>] Arbitraries to generate values. You can pass arbitraries with keyword arguments.
41
65
  # @param predicate [Proc] Test code that receives generated values and runs the test.
42
- # @return [Property]
66
+ # @return [Check::Property]
43
67
  def self.property(*args, **kwargs, &predicate)
44
68
  arb = to_arbitrary(args, kwargs)
45
- Check::Property.new(arb, &predicate)
69
+ Check::Property.new(
70
+ arb,
71
+ keyword_args: args.empty? && !kwargs.empty?,
72
+ splat_args: args.size > 1,
73
+ &predicate
74
+ )
46
75
  end
47
76
 
48
77
  # Create a stateful property-based test backed by a model and a SUT factory.
@@ -54,8 +83,8 @@ module Pbt
54
83
  #
55
84
  # Each command object is expected to provide:
56
85
  # - `name`
57
- # - `arguments` (an arbitrary) or `arguments(state)`
58
- # - `applicable?(state)` -> bool or `applicable?(state, args)` -> bool
86
+ # - `arguments(state)` -> an arbitrary
87
+ # - `applicable?(state, args)` -> bool
59
88
  # - `next_state(state, args)`
60
89
  # - `run!(sut, args)` -> result
61
90
  # - `verify!(before_state:, after_state:, args:, result:, sut:)`
@@ -68,6 +97,13 @@ module Pbt
68
97
  Stateful::Property.new(model:, sut:, max_steps:)
69
98
  end
70
99
 
100
+ # Inspect an independent sample of generated values, optionally grouped by a classifier.
101
+ # Values/labels used as Hash keys must not be mutated during or after sampling.
102
+ # @return [Statistics] Counts, percentages and the seed used for replay.
103
+ def self.statistics(arbitrary, num_runs: 100, seed: Random.new_seed, &classifier)
104
+ Statistics.new(arbitrary, num_runs: num_runs, seed: seed, &classifier)
105
+ end
106
+
71
107
  class << self
72
108
  private
73
109
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pbt
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.0
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - ohbarye
@@ -23,12 +23,20 @@ files:
23
23
  - LICENSE.txt
24
24
  - README.md
25
25
  - Rakefile
26
+ - benchmark/FIBER.md
26
27
  - benchmark/README.md
27
28
  - benchmark/example.txt
28
29
  - benchmark/failure_simple.rb
30
+ - benchmark/fiber_experiment.rb
31
+ - benchmark/fiber_scheduler/Gemfile
32
+ - benchmark/fiber_scheduler/experiment.rb
33
+ - benchmark/fiber_scheduler/spec/experiment_spec.rb
29
34
  - benchmark/success_cpu_bound.rb
30
35
  - benchmark/success_io_bound.rb
31
36
  - benchmark/success_simple.rb
37
+ - docs/plans/fiber-experiment.md
38
+ - docs/plans/generator-statistics.md
39
+ - docs/plans/stateful-pbt-redesign.md
32
40
  - lib/pbt.rb
33
41
  - lib/pbt/arbitrary/arbitrary.rb
34
42
  - lib/pbt/arbitrary/arbitrary_methods.rb
@@ -36,11 +44,15 @@ files:
36
44
  - lib/pbt/arbitrary/choose_arbitrary.rb
37
45
  - lib/pbt/arbitrary/constant.rb
38
46
  - lib/pbt/arbitrary/constant_arbitrary.rb
47
+ - lib/pbt/arbitrary/constant_from_arbitrary.rb
39
48
  - lib/pbt/arbitrary/filter_arbitrary.rb
40
49
  - lib/pbt/arbitrary/fixed_hash_arbitrary.rb
50
+ - lib/pbt/arbitrary/float_arbitrary.rb
51
+ - lib/pbt/arbitrary/frequency_arbitrary.rb
41
52
  - lib/pbt/arbitrary/integer_arbitrary.rb
42
53
  - lib/pbt/arbitrary/map_arbitrary.rb
43
54
  - lib/pbt/arbitrary/one_of_arbitrary.rb
55
+ - lib/pbt/arbitrary/sample.rb
44
56
  - lib/pbt/arbitrary/tuple_arbitrary.rb
45
57
  - lib/pbt/check/case.rb
46
58
  - lib/pbt/check/configuration.rb
@@ -52,6 +64,7 @@ files:
52
64
  - lib/pbt/reporter/run_details_reporter.rb
53
65
  - lib/pbt/reporter/run_execution.rb
54
66
  - lib/pbt/stateful/property.rb
67
+ - lib/pbt/statistics.rb
55
68
  - lib/pbt/version.rb
56
69
  - sig/pbt.rbs
57
70
  homepage: https://github.com/ohbarye/pbt
@@ -68,14 +81,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
68
81
  requirements:
69
82
  - - ">="
70
83
  - !ruby/object:Gem::Version
71
- version: '3.1'
84
+ version: '3.3'
72
85
  required_rubygems_version: !ruby/object:Gem::Requirement
73
86
  requirements:
74
87
  - - ">="
75
88
  - !ruby/object:Gem::Version
76
89
  version: '0'
77
90
  requirements: []
78
- rubygems_version: 3.6.9
91
+ rubygems_version: 4.0.20
79
92
  specification_version: 4
80
93
  summary: Property-Based Testing tool for Ruby, utilizing Ractor for parallelizing
81
94
  test cases.