pbt 0.0.1 → 0.1.0

data/lib/pbt/check/configuration.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Pbt
+  module Check
+    # Configuration for Pbt.
+    Configuration = Struct.new(
+      :verbose,
+      :worker,
+      :num_runs,
+      :seed,
+      :thread_report_on_exception,
+      keyword_init: true
+    ) do
+      # @param verbose [Boolean] Whether to print verbose output. Default is `false`.
+      # @param worker [Symbol] The concurrency method to use. :ractor`, `:thread`, `:process` and `:none` are supported. Default is `:ractor`.
+      # @param num_runs [Integer] The number of runs to perform. Default is `100`.
+      # @param seed [Integer] The seed to use for random number generation. It's useful to reproduce failed test with the seed you'd pick up from failure messages. Default is a random seed.
+      # @param thread_report_on_exception [Boolean] Whether to report exceptions in threads. It's useful to suppress error logs on Ractor that reports many errors. Default is `false`.
+      def initialize(
+        verbose: false,
+        worker: :ractor,
+        num_runs: 100,
+        seed: Random.new.seed,
+        thread_report_on_exception: false
+      )
+        super
+      end
+    end
+
+    module ConfigurationMethods
+      # Return the current configuration.
+      # If you modify the configuration, it will affect all future property-based tests.
+      #
+      # @example
+      #   config = Pbt.configuration
+      #   config.num_runs = 20
+      #
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      # Return the current configuration.
+      # If you modify the configuration, it will affect all future property-based tests.
+      #
+      # @example
+      #   Pbt.configure do |config|
+      #     config.num_runs = 20
+      #   end
+      #
+      # @yield [configuration] The current configuration.
+      # @yieldparam configuration [Configuration]
+      def configure
+        yield(configuration)
+      end
+    end
+  end
+end

data/lib/pbt/check/property.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Pbt
+  module Check
+    # Represents a property to be tested.
+    # This class holds an arbitrary to generate values and a predicate to test them.
+    class Property
+      # @param arb [Array<Arbitrary>]
+      # @param predicate [Proc] Predicate proc to test the generated values. Library users write this.
+      def initialize(arb, &predicate)
+        @arb = arb
+        @predicate = predicate
+      end
+
+      # Generate a next value to test.
+      #
+      # @param rng [Random] Random number generator.
+      # @return [Object]
+      def generate(rng)
+        @arb.generate(rng)
+      end
+
+      # Shrink the `val` to a smaller one.
+      # This is used to find the smallest failing case after a failure.
+      #
+      # @param val [Object]
+      # @return [Enumerator<Object>]
+      def shrink(val)
+        @arb.shrink(val)
+      end
+
+      # Run the predicate with the generated `val`.
+      #
+      # @param val [Object]
+      # @return [void]
+      def run(val)
+        @predicate.call(val)
+      end
+
+      # Run the predicate with the generated `val` in a Ractor.
+      # This is used only for parallel testing with Ractors.
+      #
+      # @param val [Object]
+      # @return [Ractor]
+      def run_in_ractor(val)
+        Ractor.new(val, &@predicate)
+      end
+    end
+  end
+end

data/lib/pbt/check/runner_iterator.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "pbt/reporter/run_execution"
+
+module Pbt
+  module Check
+    # This class is an iterator of the generated/shrunken values.
+    #
+    # @private
+    # @!attribute [r] run_execution [Reporter::RunExecution]
+    class RunnerIterator < DelegateClass(Array)
+      attr_reader :run_execution
+
+      # @param source_values [Enumerator] Enumerator of generated values by arbitrary. This is used to determine initial N (=`num_runs`) cases.
+      # @param property [Property] Property to test. This is used to shrink the failed case.
+      # @param verbose [Boolean] Controls the verbosity of the output.
+      def initialize(source_values, property, verbose)
+        @run_execution = Reporter::RunExecution.new(verbose)
+        @property = property
+        @next_values = source_values
+        enumerator = Enumerator.new do |y|
+          loop do
+            y.yield @next_values.next
+          end
+        end
+        super(enumerator) # delegate `#each` and etc. to enumerator
+      end
+
+      # Check if there is a next value to test.
+      # If there is no next value, it returns false. Otherwise true.
+      #
+      # @return [Boolean]
+      def has_next?
+        @next_values.peek
+        true
+      rescue StopIteration
+        false
+      end
+
+      # Handle result of a test.
+      # When a test is successful, it records the success.
+      # When a test is failed, it records the failure and set up the next values to test with property#shrink.
+      #
+      # @param c [Case]
+      # @return [void]
+      def handle_result(c)
+        if c.exception
+          # failed run
+          @run_execution.record_failure(c)
+          @next_values = @property.shrink(c.val)
+        else
+          # successful run
+          @run_execution.record_success
+        end
+      end
+    end
+  end
+end

data/lib/pbt/check/runner_methods.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "pbt/check/runner_iterator"
+require "pbt/check/tosser"
+require "pbt/check/case"
+require "pbt/reporter/run_details_reporter"
+
+module Pbt
+  module Check
+    # Module to be
+    module RunnerMethods
+      include Check::Tosser
+
+      # Run a property based test and report its result.
+      #
+      # @see Check::Configuration
+      # @param options [Hash] Optional parameters to customize the execution.
+      # @param property [Proc] Proc that returns Property instance.
+      # @return [void]
+      # @raise [PropertyFailure]
+      def assert(**options, &property)
+        out = check(**options, &property)
+        Reporter::RunDetailsReporter.new(out).report_run_details
+      end
+
+      # Run a property based test and return its result.
+      # This doesn't throw contrary to `assert`.
+      # Use `assert` unless you want to handle the result.
+      #
+      # @see RunnerMethods#assert
+      # @see Check::Configuration
+      # @param options [Hash] Optional parameters to customize the execution.
+      # @param property [Proc] Proc that returns Property instance.
+      # @return [RunDetails]
+      def check(**options, &property)
+        property = property.call
+        config = Pbt.configuration.to_h.merge(options.to_h)
+
+        initial_values = toss(property, config[:seed])
+        source_values = Enumerator.new(config[:num_runs]) do |y|
+          config[:num_runs].times do
+            y.yield initial_values.next
+          end
+        end
+
+        suppress_exception_report_for_ractor(config) do
+          run_it(property, source_values, config).to_run_details(config)
+        end
+      end
+
+      private
+
+      # If using Ractor, so many exception reports happen in Ractor and a console gets too messy. Suppress them to avoid that.
+      #
+      # @param config [Hash] Configuration parameters.
+      # @param block [Proc]
+      def suppress_exception_report_for_ractor(config, &block)
+        if config[:worker] == :ractor
+          original_report_on_exception = Thread.report_on_exception
+          Thread.report_on_exception = config[:thread_report_on_exception]
+        end
+
+        yield
+      ensure
+        Thread.report_on_exception = original_report_on_exception if config[:worker] == :ractor
+      end
+
+      # Run the property test for each value.
+      #
+      # @param property [Proc] Property to test.
+      # @param source_values [Enumerator] Enumerator of values to test.
+      # @param config [Hash] Configuration parameters.
+      # @return [RunExecution] Result of the test.
+      def run_it(property, source_values, config)
+        runner = Check::RunnerIterator.new(source_values, property, config[:verbose])
+        while runner.has_next?
+          case config[:worker]
+          in :ractor
+            run_it_in_ractors(property, runner)
+          in :process
+            run_it_in_processes(property, runner)
+          in :thread
+            run_it_in_threads(property, runner)
+          in :none
+            run_it_in_sequential(property, runner)
+          end
+        end
+        runner.run_execution
+      end
+
+      # @param property [Proc]
+      # @param runner [RunnerIterator]
+      # @return [void]
+      def run_it_in_ractors(property, runner)
+        runner.map.with_index { |val, index|
+          Case.new(val:, index:, ractor: property.run_in_ractor(val))
+        }.each do |c|
+          c.ractor.take
+          runner.handle_result(c)
+        rescue => e
+          c.exception = e.cause # Ractor error is wrapped in a Ractor::RemoteError. We need to get the cause.
+          runner.handle_result(c)
+          break # Ignore the rest of the cases. Just pick up the first failure.
+        end
+      end
+
+      # @param property [Proc]
+      # @param runner [RunnerIterator]
+      # @return [void]
+      def run_it_in_threads(property, runner)
+        require_parallel
+
+        Parallel.map_with_index(runner, in_threads: Parallel.processor_count) do |val, index|
+          Case.new(val:, index:).tap do |c|
+            property.run(val)
+          rescue => e
+            c.exception = e
+            # It's possible to break this loop here by raising `Parallel::Break`.
+            # But if it raises, we cannot fetch all cases' result. So this loop continues until the end.
+          end
+        end.each do |c|
+          runner.handle_result(c)
+          break if c.exception
+          # Ignore the rest of the cases. Just pick up the first failure.
+        end
+      end
+
+      # @param property [Proc]
+      # @param runner [RunnerIterator]
+      # @return [void]
+      def run_it_in_processes(property, runner)
+        require_parallel
+
+        Parallel.map_with_index(runner, in_processes: Parallel.processor_count) do |val, index|
+          Case.new(val:, index:).tap do |c|
+            property.run(val)
+          rescue => e
+            c.exception = e
+            # It's possible to break this loop here by raising `Parallel::Break`.
+            # But if it raises, we cannot fetch all cases' result. So this loop continues until the end.
+          end
+        end.each do |c|
+          runner.handle_result(c)
+          break if c.exception
+          # Ignore the rest of the cases. Just pick up the first failure.
+        end
+      end
+
+      # @param property [Proc]
+      # @param runner [RunnerIterator]
+      # @return [void]
+      def run_it_in_sequential(property, runner)
+        runner.each_with_index do |val, index|
+          c = Case.new(val:, index:)
+          begin
+            property.run(val)
+            runner.handle_result(c)
+          rescue => e
+            c.exception = e
+            runner.handle_result(c)
+            break # Ignore the rest of the cases. Just pick up the first failure.
+          end
+        end
+      end
+
+      # Load Parallel gem. If it's not installed, raise an error.
+      # @see https://github.com/grosser/parallel
+      # @raise [InvalidConfiguration]
+      def require_parallel
+        require "parallel"
+      rescue LoadError
+        raise InvalidConfiguration,
+          "Parallel gem (https://github.com/grosser/parallel) is required to use worker `:process` or `:thread`. Please add `gem 'parallel'` to your Gemfile."
+      end
+    end
+  end
+end

data/lib/pbt/check/tosser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Pbt
+  module Check
+    # Module to be included in classes that need to generate values to test.
+    module Tosser
+      # Generate values.
+      #
+      # @param arb [Arbitrary] Arbitrary to generate a value.
+      # @param seed [Integer] Random number generator's seed.
+      # @return [Enumerator]
+      def toss(arb, seed)
+        Enumerator.new do |enum|
+          rng = Random.new(seed)
+          loop do
+            enum.yield toss_next(arb, rng)
+          end
+        end
+      end
+
+      private
+
+      # Generate next value.
+      #
+      # @param arb [Arbitrary] Arbitrary to generate a value.
+      # @param rng [Random] Random number generator.
+      def toss_next(arb, rng)
+        arb.generate(rng)
+      end
+    end
+  end
+end

data/lib/pbt/reporter/run_details.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Pbt
+  module Reporter
+    # Details of a single run of a property test.
+    RunDetails = Struct.new(
+      :failed,
+      :num_runs,
+      :num_shrinks,
+      :seed,
+      :counterexample,
+      :counterexample_path,
+      :error_message,
+      :error_instance,
+      :failures,
+      :verbose,
+      :run_configuration,
+      keyword_init: true
+    )
+  end
+end

data/lib/pbt/reporter/run_details_reporter.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Pbt
+  module Reporter
+    # Reporter for the run details of a property test.
+    class RunDetailsReporter
+      # @param run_details [Pbt::Reporter::RunExecution]
+      def initialize(run_details)
+        @run_details = run_details
+      end
+
+      # Report the run details of a property test.
+      # If the property test failed, raise a PropertyFailure.
+      #
+      # @raise [PropertyFailure]
+      def report_run_details
+        if @run_details.failed
+          message = []
+
+          message << <<~EOS
+            Property failed after #{@run_details.num_runs} test(s)
+            { seed: #{@run_details.seed} }
+            Counterexample: #{@run_details.counterexample}
+            Shrunk #{@run_details.num_shrinks} time(s)
+            Got #{@run_details.error_instance.class}: #{@run_details.error_message}
+          EOS
+
+          if @run_details.verbose
+            message << "  \n#{@run_details.error_instance.backtrace_locations.join("\n    ")}"
+            message << "\nEncountered failures were:"
+            message << @run_details.failures.map { |f| "- #{f.val}" }
+          end
+
+          raise PropertyFailure, message.join("\n") if message.size > 0
+        end
+      end
+    end
+  end
+end

data/lib/pbt/reporter/run_execution.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "pbt/reporter/run_details"
+
+module Pbt
+  module Reporter
+    # Represents the result of a single run of a property test.
+    class RunExecution
+      # @param verbose [Boolean] Whether to print verbose output.
+      def initialize(verbose)
+        @verbose = verbose
+        @path_to_failure = []
+        @failure = nil
+        @failures = []
+        @num_successes = 0
+      end
+
+      # Record a failure in the run.
+      #
+      # @param c [Pbt::Check::Case]
+      def record_failure(c)
+        @path_to_failure << c.index
+        @failures << c
+
+        # value and failure can be updated through shrinking
+        @value = c.val
+        @failure = c.exception
+      end
+
+      # Record a successful run.
+      #
+      # @return [void]
+      def record_success
+        @num_successes += 1
+      end
+
+      # Whether the test was successful.
+      #
+      # @return [Boolean]
+      def success?
+        !@failure
+      end
+
+      # Convert execution to run details.
+      #
+      # @param config [Hash] Configuration parameters used for the run.
+      # @return [RunDetails] Details of the run.
+      def to_run_details(config)
+        if success?
+          RunDetails.new(
+            failed: false,
+            num_runs: @num_successes,
+            num_shrinks: 0,
+            seed: config[:seed],
+            counterexample: nil,
+            counterexample_path: nil,
+            error_message: nil,
+            error_instance: nil,
+            failures: @failures,
+            verbose: @verbose,
+            run_configuration: config
+          )
+        else
+          RunDetails.new(
+            failed: true,
+            num_runs: @path_to_failure[0] + 1,
+            num_shrinks: @path_to_failure.size - 1,
+            seed: config[:seed],
+            counterexample: @value,
+            counterexample_path: @path_to_failure.join(":"),
+            error_message: @failure.message,
+            error_instance: @failure,
+            failures: @failures,
+            verbose: @verbose,
+            run_configuration: config
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pbt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pbt
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/pbt.rb

@@ -1,13 +1,73 @@
 # frozen_string_literal: true
 
 require_relative "pbt/version"
-require_relative "pbt/generator"
-require_relative "pbt/runner"
+require_relative "pbt/arbitrary/arbitrary_methods"
+require_relative "pbt/check/runner_methods"
+require_relative "pbt/check/property"
+require_relative "pbt/check/configuration"
 
 module Pbt
-  class CaseFailure < StandardError; end
+  # Represents a property-based test failure.
+  class PropertyFailure < StandardError; end
 
-  def self.forall(generator, &block)
-    Runner.new(generator, &block).run
+  # Represents an invalid configuration.
+  class InvalidConfiguration < StandardError; end
+
+  extend Arbitrary::ArbitraryMethods
+  extend Check::RunnerMethods
+  extend Check::ConfigurationMethods
+
+  # Create a property-based test with arbitraries. To run the test, pass the returned value to `Pbt.assert` method.
+  # Be aware that using both positional and keyword arguments is not supported.
+  #
+  # @example Basic usage
+  #   Pbt.property(Pbt.integer) do |n|
+  #     # your test code here
+  #   end
+  #
+  # @example Use multiple arbitraries
+  #   Pbt.property(Pbt.string, Pbt.symbol) do |str, sym|
+  #     # your test code here
+  #   end
+  #
+  # @example Use hash arbitraries
+  #   Pbt.property(x: Pbt.integer, y: Pbt.integer) do |x, y|
+  #     # your test code here
+  #   end
+  #
+  # @param args [Array<Arbitrary>] Arbitraries to generate values. You can pass one or more arbitraries.
+  # @param kwargs [Hash<Symbol,Arbitrary>] Arbitraries to generate values. You can pass arbitraries with keyword arguments.
+  # @param predicate [Proc] Test code that receives generated values and runs the test.
+  # @return [Property]
+  def self.property(*args, **kwargs, &predicate)
+    arb = to_arbitrary(args, kwargs)
+    Check::Property.new(arb, &predicate)
+  end
+
+  class << self
+    private
+
+    # Convert arguments to suitable arbitrary.
+    # If multiple arguments are given, wrap them by tuple arbitrary.
+    # If keyword arguments are given, wrap them by fixed hash arbitrary.
+    # Else, return the single arbitrary.
+    #
+    # @param args [Array<Arbitrary>]
+    # @param kwargs [Hash<Symbol,Arbitrary>]
+    # @return [Arbitrary]
+    # @raise [ArgumentError] When both positional and keyword arguments are given
+    def to_arbitrary(args, kwargs)
+      if args == [] && kwargs != {}
+        fixed_hash(kwargs)
+      elsif args != [] && kwargs == {}
+        # wrap by tuple arbitrary so that property class doesn't have to take care of an array
+        (args.size == 1) ? args.first : tuple(*args)
+      else
+        raise ArgumentError, <<~MSG
+          It's not supported to use both positional and keyword arguments at the same time.
+          cf. https://www.ruby-lang.org/en/news/2019/12/12/separation-of-positional-and-keyword-arguments-in-ruby-3-0/
+        MSG
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pbt
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - ohbarye
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-27 00:00:00.000000000 Z
+date: 2024-04-13 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -25,8 +25,27 @@ files:
 - README.md
 - Rakefile
 - lib/pbt.rb
-- lib/pbt/generator.rb
-- lib/pbt/runner.rb
+- lib/pbt/arbitrary/arbitrary.rb
+- lib/pbt/arbitrary/arbitrary_methods.rb
+- lib/pbt/arbitrary/array_arbitrary.rb
+- lib/pbt/arbitrary/choose_arbitrary.rb
+- lib/pbt/arbitrary/constant.rb
+- lib/pbt/arbitrary/constant_arbitrary.rb
+- lib/pbt/arbitrary/filter_arbitrary.rb
+- lib/pbt/arbitrary/fixed_hash_arbitrary.rb
+- lib/pbt/arbitrary/integer_arbitrary.rb
+- lib/pbt/arbitrary/map_arbitrary.rb
+- lib/pbt/arbitrary/one_of_arbitrary.rb
+- lib/pbt/arbitrary/tuple_arbitrary.rb
+- lib/pbt/check/case.rb
+- lib/pbt/check/configuration.rb
+- lib/pbt/check/property.rb
+- lib/pbt/check/runner_iterator.rb
+- lib/pbt/check/runner_methods.rb
+- lib/pbt/check/tosser.rb
+- lib/pbt/reporter/run_details.rb
+- lib/pbt/reporter/run_details_reporter.rb
+- lib/pbt/reporter/run_execution.rb
 - lib/pbt/version.rb
 - sig/pbt.rbs
 homepage: https://github.com/ohbarye/pbt

data/lib/pbt/generator.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Pbt
-  module Generator
-    def self.integer(low = nil, high = nil)
-      rng = Random.new
-      size = 100
-      if low && high
-        -> { rng.rand(low..high) }
-      else
-        -> { rng.rand(-size..size) }
-      end
-    end
-  end
-end