pbt 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +1 -1
  3. data/CHANGELOG.md +21 -0
  4. data/CLAUDE.md +2 -2
  5. data/README.md +116 -36
  6. data/benchmark/FIBER.md +97 -0
  7. data/benchmark/fiber_experiment.rb +39 -0
  8. data/benchmark/fiber_scheduler/Gemfile +7 -0
  9. data/benchmark/fiber_scheduler/experiment.rb +131 -0
  10. data/benchmark/fiber_scheduler/spec/experiment_spec.rb +38 -0
  11. data/docs/plans/fiber-experiment.md +12 -0
  12. data/docs/plans/generator-statistics.md +16 -0
  13. data/docs/plans/stateful-pbt-redesign.md +415 -0
  14. data/lib/pbt/arbitrary/arbitrary.rb +65 -12
  15. data/lib/pbt/arbitrary/arbitrary_methods.rb +51 -24
  16. data/lib/pbt/arbitrary/array_arbitrary.rb +46 -12
  17. data/lib/pbt/arbitrary/choose_arbitrary.rb +28 -1
  18. data/lib/pbt/arbitrary/constant.rb +0 -1
  19. data/lib/pbt/arbitrary/constant_arbitrary.rb +6 -1
  20. data/lib/pbt/arbitrary/constant_from_arbitrary.rb +39 -0
  21. data/lib/pbt/arbitrary/filter_arbitrary.rb +41 -3
  22. data/lib/pbt/arbitrary/fixed_hash_arbitrary.rb +17 -1
  23. data/lib/pbt/arbitrary/float_arbitrary.rb +31 -0
  24. data/lib/pbt/arbitrary/frequency_arbitrary.rb +61 -0
  25. data/lib/pbt/arbitrary/integer_arbitrary.rb +12 -1
  26. data/lib/pbt/arbitrary/map_arbitrary.rb +41 -3
  27. data/lib/pbt/arbitrary/one_of_arbitrary.rb +47 -14
  28. data/lib/pbt/arbitrary/sample.rb +97 -0
  29. data/lib/pbt/arbitrary/tuple_arbitrary.rb +23 -1
  30. data/lib/pbt/check/case.rb +1 -1
  31. data/lib/pbt/check/configuration.rb +5 -3
  32. data/lib/pbt/check/property.rb +24 -5
  33. data/lib/pbt/check/runner_iterator.rb +84 -10
  34. data/lib/pbt/check/runner_methods.rb +47 -17
  35. data/lib/pbt/check/tosser.rb +1 -1
  36. data/lib/pbt/reporter/run_details.rb +2 -2
  37. data/lib/pbt/reporter/run_details_reporter.rb +13 -0
  38. data/lib/pbt/reporter/run_execution.rb +47 -4
  39. data/lib/pbt/stateful/property.rb +8 -102
  40. data/lib/pbt/statistics.rb +28 -0
  41. data/lib/pbt/version.rb +1 -1
  42. data/lib/pbt.rb +45 -9
  43. metadata +16 -3
@@ -6,12 +6,13 @@ module Pbt
6
6
  module Reporter
7
7
  # Represents the result of a single run of a property test.
8
8
  class RunExecution
9
+ attr_reader :num_successes, :num_skips
10
+
9
11
  # A tree node of the execution for verbose output.
10
12
  ExecutionTreeNode = Struct.new(
11
13
  :result,
12
14
  :value,
13
- :children,
14
- keyword_init: true
15
+ :children
15
16
  )
16
17
 
17
18
  # @param verbose [Boolean] Whether to print verbose output.
@@ -19,8 +20,11 @@ module Pbt
19
20
  @verbose = verbose
20
21
  @path_to_failure = []
21
22
  @failure = nil
23
+ @too_many_skips_error = nil
22
24
  @failures = []
23
25
  @num_successes = 0
26
+ @num_skips = 0
27
+ @num_runs_on_failure = nil
24
28
  @root_execution_trees = []
25
29
  @current_level_execution_trees = @root_execution_trees
26
30
  end
@@ -29,6 +33,7 @@ module Pbt
29
33
  #
30
34
  # @param c [Pbt::Check::Case]
31
35
  def record_failure(c)
36
+ @num_runs_on_failure ||= @num_successes + 1
32
37
  if @verbose
33
38
  current_tree = append_execution_tree_node(:failure, c.val)
34
39
  @current_level_execution_trees = current_tree.children
@@ -52,11 +57,31 @@ module Pbt
52
57
  @num_successes += 1
53
58
  end
54
59
 
60
+ # Record a skipped initial run (from `Pbt.assume`).
61
+ #
62
+ # @return [void]
63
+ def record_skip
64
+ @num_skips += 1
65
+ end
66
+
67
+ # Record a failure caused by exceeding `max_skips`.
68
+ #
69
+ # @param error [Pbt::TooManySkipsError]
70
+ # @return [void]
71
+ def record_too_many_skips(error)
72
+ @too_many_skips_error = error
73
+ end
74
+
75
+ # @return [Boolean]
76
+ def too_many_skips?
77
+ !@too_many_skips_error.nil?
78
+ end
79
+
55
80
  # Whether the test was successful.
56
81
  #
57
82
  # @return [Boolean]
58
83
  def success?
59
- !@failure
84
+ !@failure && !@too_many_skips_error
60
85
  end
61
86
 
62
87
  # Convert execution to run details.
@@ -68,6 +93,7 @@ module Pbt
68
93
  RunDetails.new(
69
94
  failed: false,
70
95
  num_runs: @num_successes,
96
+ num_skips: @num_skips,
71
97
  num_shrinks: 0,
72
98
  seed: config[:seed],
73
99
  counterexample: nil,
@@ -79,10 +105,27 @@ module Pbt
79
105
  execution_summary: @root_execution_trees,
80
106
  run_configuration: config
81
107
  )
108
+ elsif @too_many_skips_error
109
+ RunDetails.new(
110
+ failed: true,
111
+ num_runs: @num_successes,
112
+ num_skips: @num_skips,
113
+ num_shrinks: 0,
114
+ seed: config[:seed],
115
+ counterexample: nil,
116
+ counterexample_path: nil,
117
+ error_message: @too_many_skips_error.message,
118
+ error_instance: @too_many_skips_error,
119
+ failures: @failures,
120
+ verbose: @verbose,
121
+ execution_summary: @root_execution_trees,
122
+ run_configuration: config
123
+ )
82
124
  else
83
125
  RunDetails.new(
84
126
  failed: true,
85
- num_runs: @path_to_failure[0] + 1,
127
+ num_runs: @num_runs_on_failure,
128
+ num_skips: @num_skips,
86
129
  num_shrinks: @path_to_failure.size - 1,
87
130
  seed: config[:seed],
88
131
  counterexample: @value,
@@ -16,7 +16,7 @@ module Pbt
16
16
  verify!
17
17
  ].freeze
18
18
 
19
- Step = Struct.new(:command, :args, keyword_init: true) do
19
+ Step = Struct.new(:command, :args) do
20
20
  def inspect
21
21
  "#<Pbt::Stateful::Step command=#{command_label}, args=#{args.inspect}>"
22
22
  end
@@ -50,7 +50,7 @@ module Pbt
50
50
  #
51
51
  # @param rng [Random]
52
52
  # @return [Array<Step>]
53
- def generate(rng)
53
+ def generate(rng = Random.new)
54
54
  length = rng.rand(0..@max_steps)
55
55
  state = @model.initial_state
56
56
  sequence = []
@@ -140,6 +140,8 @@ module Pbt
140
140
  result:,
141
141
  sut:
142
142
  )
143
+ rescue *Check::FATAL_EXCEPTIONS
144
+ raise
143
145
  rescue Exception => e # standard:disable Lint/RescueException:
144
146
  raise e.class,
145
147
  "stateful step #{index} (#{command_name(command)}): #{e.message} [args=#{args.inspect}]",
@@ -207,11 +209,6 @@ module Pbt
207
209
  "Pbt.stateful command protocol mismatch for #{command.class} " \
208
210
  "(name=#{safe_command_label(command)}, missing: #{missing_methods.join(", ")}, context=#{context})"
209
211
  end
210
-
211
- validate_command_signature!(command, :arguments, valid_counts: [0, 1], expectation: "arguments or arguments(state)",
212
- context:)
213
- validate_command_signature!(command, :applicable?, valid_counts: [1, 2],
214
- expectation: "applicable?(state) or applicable?(state, args)", context:)
215
212
  end
216
213
 
217
214
  # @param command [Object]
@@ -252,12 +249,6 @@ module Pbt
252
249
  # @param context [String]
253
250
  # @return [Object]
254
251
  def generate_applicable_args(command, state, rng, context:)
255
- unless arg_aware_command?(command)
256
- return NoApplicableArgs unless applicable?(command, state, nil, context:)
257
-
258
- return arbitrary_for(command, state, context:).generate(rng)
259
- end
260
-
261
252
  arbitrary = arbitrary_for(command, state, context:)
262
253
 
263
254
  ARG_AWARE_GENERATION_ATTEMPTS.times do
@@ -267,6 +258,8 @@ module Pbt
267
258
  return args if applicable?(command, state, args, context:)
268
259
  end
269
260
 
261
+ NoApplicableArgs
262
+ rescue Pbt::Arbitrary::EmptyDomainError
270
263
  NoApplicableArgs
271
264
  end
272
265
 
@@ -275,15 +268,7 @@ module Pbt
275
268
  # @param context [String]
276
269
  # @return [Object]
277
270
  def arguments_for(command, state, context:)
278
- method = command.method(:arguments)
279
-
280
- if supports_argument_count?(method, 1)
281
- command.arguments(state)
282
- elsif supports_argument_count?(method, 0)
283
- command.arguments
284
- else
285
- raise_invalid_signature!(command, :arguments, "arguments or arguments(state)", context)
286
- end
271
+ command.arguments(state)
287
272
  end
288
273
 
289
274
  # @param command [Object]
@@ -303,8 +288,6 @@ module Pbt
303
288
  def generate_args_for(command, arbitrary, rng)
304
289
  arbitrary.generate(rng)
305
290
  rescue Pbt::Arbitrary::EmptyDomainError
306
- raise unless state_aware_arg_aware_command?(command)
307
-
308
291
  NoApplicableArgs
309
292
  end
310
293
 
@@ -314,84 +297,7 @@ module Pbt
314
297
  # @param context [String]
315
298
  # @return [Boolean]
316
299
  def applicable?(command, state, args, context:)
317
- method = command.method(:applicable?)
318
-
319
- if supports_argument_count?(method, 2)
320
- command.applicable?(state, args)
321
- elsif supports_argument_count?(method, 1)
322
- command.applicable?(state)
323
- else
324
- raise_invalid_signature!(command, :applicable?, "applicable?(state) or applicable?(state, args)", context)
325
- end
326
- end
327
-
328
- # @param command [Object]
329
- # @return [Boolean]
330
- def arg_aware_command?(command)
331
- supports_argument_count?(command.method(:applicable?), 2)
332
- end
333
-
334
- # @param command [Object]
335
- # @return [Boolean]
336
- def state_aware_arg_aware_command?(command)
337
- arg_aware_command?(command) && supports_argument_count?(command.method(:arguments), 1)
338
- end
339
-
340
- # @param command [Object]
341
- # @param method_name [Symbol]
342
- # @param valid_counts [Array<Integer>]
343
- # @param expectation [String]
344
- # @param context [String]
345
- # @return [void]
346
- def validate_command_signature!(command, method_name, valid_counts:, expectation:, context:)
347
- method = command.method(method_name)
348
- return if valid_counts.any? { |count| supports_argument_count?(method, count) }
349
-
350
- raise_invalid_signature!(command, method_name, expectation, context)
351
- end
352
-
353
- # @param command [Object]
354
- # @param method_name [Symbol]
355
- # @param expectation [String]
356
- # @param context [String]
357
- # @return [void]
358
- def raise_invalid_signature!(command, method_name, expectation, context)
359
- raise Pbt::InvalidConfiguration,
360
- "Pbt.stateful command protocol mismatch for #{command.class} " \
361
- "(name=#{safe_command_label(command)}, invalid #{method_name} signature; expected #{expectation}, context=#{context})"
362
- end
363
-
364
- # @param method [Method]
365
- # @param count [Integer]
366
- # @return [Boolean]
367
- def supports_argument_count?(method, count)
368
- return false if method.parameters.any? { |kind, _name| keyword_parameter?(kind) }
369
-
370
- required = 0
371
- optional = 0
372
- rest = false
373
-
374
- method.parameters.each do |kind, _name|
375
- case kind
376
- when :req
377
- required += 1
378
- when :opt
379
- optional += 1
380
- when :rest
381
- rest = true
382
- end
383
- end
384
-
385
- return false if count < required
386
- return true if rest
387
-
388
- count <= required + optional
389
- end
390
-
391
- # @param kind [Symbol]
392
- # @return [Boolean]
393
- def keyword_parameter?(kind)
394
- %i[keyreq key keyrest].include?(kind)
300
+ command.applicable?(state, args)
395
301
  end
396
302
 
397
303
  # @param sequence [Array<Hash, Step>]
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pbt
4
+ # Distribution of an independent, seeded sample of generated values.
5
+ class Statistics
6
+ attr_reader :seed, :num_runs, :counts
7
+
8
+ def initialize(arbitrary, num_runs:, seed:, &classifier)
9
+ unless num_runs.is_a?(Integer) && num_runs >= 0
10
+ raise ArgumentError, "num_runs must be a nonnegative Integer"
11
+ end
12
+ @seed = seed
13
+ @num_runs = num_runs
14
+ @counts = Hash.new(0)
15
+ rng = Random.new(seed)
16
+ num_runs.times do
17
+ value = arbitrary.generate(rng)
18
+ label = classifier ? classifier.call(value) : value
19
+ @counts[label] += 1
20
+ end
21
+ end
22
+
23
+ # Percentages are not rounded. Empty samples return an empty Hash.
24
+ def percentages
25
+ counts.transform_values { |count| count * 100.0 / num_runs }
26
+ end
27
+ end
28
+ end
data/lib/pbt/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pbt
4
- VERSION = "0.6.0"
4
+ VERSION = "0.8.0"
5
5
  end
data/lib/pbt.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "pbt/version"
4
+ require_relative "pbt/statistics"
4
5
  require_relative "pbt/arbitrary/arbitrary_methods"
5
6
  require_relative "pbt/check/runner_methods"
6
7
  require_relative "pbt/check/property"
@@ -11,6 +12,12 @@ module Pbt
11
12
  # Represents a property-based test failure.
12
13
  class PropertyFailure < StandardError; end
13
14
 
15
+ # Represents a property-based test failure caused by exceeding the skip budget via `Pbt.assume`.
16
+ class TooManySkipsError < PropertyFailure; end
17
+
18
+ # Internal control-flow exception raised by `Pbt.assume` when a precondition is not met.
19
+ class PreconditionNotMet < StandardError; end
20
+
14
21
  # Represents an invalid configuration.
15
22
  class InvalidConfiguration < StandardError; end
16
23
 
@@ -18,7 +25,19 @@ module Pbt
18
25
  extend Check::RunnerMethods
19
26
  extend Check::ConfigurationMethods
20
27
 
21
- # Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` method.
28
+ # Discard the current generated test case when `condition` is falsy.
29
+ # Discarded cases do not count toward `num_runs` and are skipped during shrinking.
30
+ # Exceeding `max_skips` fails the property with `Pbt::TooManySkipsError`.
31
+ #
32
+ # @param condition [Object]
33
+ # @return [void]
34
+ # @raise [Pbt::PreconditionNotMet]
35
+ def self.assume(condition)
36
+ raise PreconditionNotMet unless condition
37
+ end
38
+
39
+ # Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` or `Pbt.check`.
40
+ # Predicates fail by raising an exception (including RSpec/Minitest assertions); returning `false` alone does not fail.
22
41
  # Be aware that using both positional and keyword arguments is not supported.
23
42
  #
24
43
  # @example Basic usage
@@ -26,23 +45,33 @@ module Pbt
26
45
  # # your test code here
27
46
  # end
28
47
  #
29
- # @example Use multiple arbitraries
30
- # Pbt.property(Pbt.string, Pbt.symbol) do |str, sym|
48
+ # @example Use multiple positional arbitraries
49
+ # Pbt.property(Pbt.ascii_string, Pbt.symbol) do |str, sym|
31
50
  # # your test code here
32
51
  # end
33
52
  #
34
- # @example Use hash arbitraries
35
- # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x, y|
53
+ # @example Use keyword arbitraries (under default `worker: :none`)
54
+ # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x:, y:|
36
55
  # # your test code here
37
56
  # end
38
57
  #
58
+ # @example Use keyword arbitraries with `worker: :ractor` (Ractor passes the Hash as a single argument)
59
+ # Pbt.property(x: Pbt.integer, y: Pbt.integer) do |h|
60
+ # # h[:x], h[:y]
61
+ # end
62
+ #
39
63
  # @param args [Array<Arbitrary>] Arbitraries to generate values. You can pass one or more arbitraries.
40
64
  # @param kwargs [Hash<Symbol,Arbitrary>] Arbitraries to generate values. You can pass arbitraries with keyword arguments.
41
65
  # @param predicate [Proc] Test code that receives generated values and runs the test.
42
- # @return [Property]
66
+ # @return [Check::Property]
43
67
  def self.property(*args, **kwargs, &predicate)
44
68
  arb = to_arbitrary(args, kwargs)
45
- Check::Property.new(arb, &predicate)
69
+ Check::Property.new(
70
+ arb,
71
+ keyword_args: args.empty? && !kwargs.empty?,
72
+ splat_args: args.size > 1,
73
+ &predicate
74
+ )
46
75
  end
47
76
 
48
77
  # Create a stateful property-based test backed by a model and a SUT factory.
@@ -54,8 +83,8 @@ module Pbt
54
83
  #
55
84
  # Each command object is expected to provide:
56
85
  # - `name`
57
- # - `arguments` (an arbitrary) or `arguments(state)`
58
- # - `applicable?(state)` -> bool or `applicable?(state, args)` -> bool
86
+ # - `arguments(state)` -> an arbitrary
87
+ # - `applicable?(state, args)` -> bool
59
88
  # - `next_state(state, args)`
60
89
  # - `run!(sut, args)` -> result
61
90
  # - `verify!(before_state:, after_state:, args:, result:, sut:)`
@@ -68,6 +97,13 @@ module Pbt
68
97
  Stateful::Property.new(model:, sut:, max_steps:)
69
98
  end
70
99
 
100
+ # Inspect an independent sample of generated values, optionally grouped by a classifier.
101
+ # Values/labels used as Hash keys must not be mutated during or after sampling.
102
+ # @return [Statistics] Counts, percentages and the seed used for replay.
103
+ def self.statistics(arbitrary, num_runs: 100, seed: Random.new_seed, &classifier)
104
+ Statistics.new(arbitrary, num_runs: num_runs, seed: seed, &classifier)
105
+ end
106
+
71
107
  class << self
72
108
  private
73
109
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pbt
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - ohbarye
@@ -23,12 +23,20 @@ files:
23
23
  - LICENSE.txt
24
24
  - README.md
25
25
  - Rakefile
26
+ - benchmark/FIBER.md
26
27
  - benchmark/README.md
27
28
  - benchmark/example.txt
28
29
  - benchmark/failure_simple.rb
30
+ - benchmark/fiber_experiment.rb
31
+ - benchmark/fiber_scheduler/Gemfile
32
+ - benchmark/fiber_scheduler/experiment.rb
33
+ - benchmark/fiber_scheduler/spec/experiment_spec.rb
29
34
  - benchmark/success_cpu_bound.rb
30
35
  - benchmark/success_io_bound.rb
31
36
  - benchmark/success_simple.rb
37
+ - docs/plans/fiber-experiment.md
38
+ - docs/plans/generator-statistics.md
39
+ - docs/plans/stateful-pbt-redesign.md
32
40
  - lib/pbt.rb
33
41
  - lib/pbt/arbitrary/arbitrary.rb
34
42
  - lib/pbt/arbitrary/arbitrary_methods.rb
@@ -36,11 +44,15 @@ files:
36
44
  - lib/pbt/arbitrary/choose_arbitrary.rb
37
45
  - lib/pbt/arbitrary/constant.rb
38
46
  - lib/pbt/arbitrary/constant_arbitrary.rb
47
+ - lib/pbt/arbitrary/constant_from_arbitrary.rb
39
48
  - lib/pbt/arbitrary/filter_arbitrary.rb
40
49
  - lib/pbt/arbitrary/fixed_hash_arbitrary.rb
50
+ - lib/pbt/arbitrary/float_arbitrary.rb
51
+ - lib/pbt/arbitrary/frequency_arbitrary.rb
41
52
  - lib/pbt/arbitrary/integer_arbitrary.rb
42
53
  - lib/pbt/arbitrary/map_arbitrary.rb
43
54
  - lib/pbt/arbitrary/one_of_arbitrary.rb
55
+ - lib/pbt/arbitrary/sample.rb
44
56
  - lib/pbt/arbitrary/tuple_arbitrary.rb
45
57
  - lib/pbt/check/case.rb
46
58
  - lib/pbt/check/configuration.rb
@@ -52,6 +64,7 @@ files:
52
64
  - lib/pbt/reporter/run_details_reporter.rb
53
65
  - lib/pbt/reporter/run_execution.rb
54
66
  - lib/pbt/stateful/property.rb
67
+ - lib/pbt/statistics.rb
55
68
  - lib/pbt/version.rb
56
69
  - sig/pbt.rbs
57
70
  homepage: https://github.com/ohbarye/pbt
@@ -68,14 +81,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
68
81
  requirements:
69
82
  - - ">="
70
83
  - !ruby/object:Gem::Version
71
- version: '3.1'
84
+ version: '3.3'
72
85
  required_rubygems_version: !ruby/object:Gem::Requirement
73
86
  requirements:
74
87
  - - ">="
75
88
  - !ruby/object:Gem::Version
76
89
  version: '0'
77
90
  requirements: []
78
- rubygems_version: 3.6.9
91
+ rubygems_version: 4.0.20
79
92
  specification_version: 4
80
93
  summary: Property-Based Testing tool for Ruby, utilizing Ractor for parallelizing
81
94
  test cases.