forall 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: eaeea253a28a5ebeda2e96db04da2c8a8d611062dd553ec2ed1a3dba85bdfae1
+  data.tar.gz: 25bb8fbec548c8a6e0c5d907a48170554129e4d46554ed5115a9c9cedd3ce377
+SHA512:
+  metadata.gz: 9031e48f3c8c7f03e36dd331b73223574b3496ac91380d4ef368f311c625f3603c543965cfc6d67f0164d745bdf6b8c659fe8bb30e006443656b8a382cf7ce66
+  data.tar.gz: 94c69967b093cb6b0bcc56d998cca7e926c283a1d7c0fa56fd4f6270db874dfbd2ea463b670fa53aef583513b65c8db42cbf27a4a89715f51bca467c2609f6f6

data/lib/forall.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+class Forall
+  autoload :Input,    "forall/input"
+  autoload :Random,   "forall/random"
+  autoload :Counter,  "forall/counter"
+  autoload :Matchers, "forall/matchers"
+
+  # The property was true of all tested inputs
+  Ok = Struct.new(:seed, :counter)
+
+  # The property was not true of at least one tested input
+  No = Struct.new(:seed, :counter, :counterexample)
+
+  # Couldn't find enough suitable inputs to test
+  Vacuous = Struct.new(:seed, :counter)
+
+  # An error occurred while checking the property
+  Fail = Struct.new(:seed, :counter, :counterexample, :error)
+
+  # TODO: Is there a way to provide a default implementation of `shrink`? Will
+  # it interefere with a user-given implementation?
+
+  Options = Struct.new(
+    :max_ok,      # Stop looking for counterexamples after this many inputs pass
+    :max_skip,    # Give up if more than this many inputs are skipped
+    :max_shrink)  # Number of similar inputs to evaluate when searching for simpler counterexamples
+
+  class << self
+    def check(input, random, options = nil, &prop)
+      options            ||= Options.new
+      options.max_shrink ||= 100
+
+      if input.exhaustive?
+        options.max_ok   ||= input.size * 0.90
+        options.max_skip ||= input.size * 0.10
+      else
+        options.max_ok   ||= 100
+        options.max_skip ||= options.max_ok * 0.10
+      end
+
+      if prop.arity == 1
+        _prop = prop
+        prop  = lambda{|x,_| _prop.call(x) }
+      end
+
+      raise ArgumentError, "property must take one or two arguments" \
+        unless prop.arity == 2
+
+      counter = Counter.new
+
+      input.each(random) do |example|
+        return Ok.new(random.seed, counter)      if counter.ok   >= options.max_ok
+        return Vacuous.new(random.seed, counter) if counter.skip >= options.max_skip
+
+        catch(:skip) do
+          if prop.call(example, counter)
+            counter.ok += 1
+          else
+            return no(random, counter, input.shrink, example, options, prop)
+          end
+        end
+      rescue Exception => error
+        counter.fail += 1
+        return fail(random, counter, input.shrink, example, options, prop, error)
+      end
+
+      # Didn't meet ok_max because input was exhausted
+      Ok.new(random.seed, counter)
+    end
+
+  private
+
+    def weight(xs, min, max)
+      n = xs.length.to_f
+      r = max - min
+      xs.map.with_index{|x, k| [x, max-r*k/n] }
+    end
+
+    def _weight(xs, min, max)
+      n = xs.length.to_f
+      r = max - min
+      xs.map.with_index{|x, k| C.new(x, max-r*k/n, 0) }
+    end
+
+    C = Struct.new(:value, :fitness, :heuristic, :note)
+
+    # Search for the simplest counterexample
+    def no(random, counter, shrink, shrunk, options, prop)
+      return No.new(random.seed, counter, shrunk) if shrink.nil?
+
+      # The problem of finding the smallest counterexample can be described in
+      # terms of local search. The search space has a graph structure (think of
+      # a tree with duplicate nodes) and candidate solutions are examples drawn
+      # from the domain over which the property holds. The criterion to be
+      # maximized is the simplicity of a counterexample -- simple is not meant
+      # in any particular formal sense. The neighborhood relation is described
+      # by the user-provided function `shrink`, which should return candidate
+      # solutions that are only incrementally simpler than its argument.
+      #
+      # For ease of use, the user does not need to provide a function to
+      # calculate the criterion (simplicity) of a candidate solution. Instead,
+      # it is inferred by the number of edges in its path from the root and by
+      # the relative order in which it was returned among other candidate
+      # solutions (the first being simplest). There is also no requirement for
+      # a user-supplied heuristic to rank partial solutions, as it would often
+      # be difficult to estimate an optimal solution let alone calculate some
+      # quantifiable difference between it and any other partial solution.
+      #
+      # The solution space is finite (eventually `shrink` must return an empty
+      # list), but because we have a self-imposed computational budget, its not
+      # feasible to exhaustively search for the deepest leaf in the tree. We can
+      # conjure a heuristic based on a candidate solution's ratio of ancestors
+      # that are examples or counterexamples. If one candidate solution was
+      # generated among many which did not disprove the property, and another
+      # candidate solution was generated among many counterexamples, we will
+      # assume the first candidate is less likely to produce more
+      # counterexamples than the second.
+      #
+      #                           shrunk
+      #                             |
+      #                       [✓]  [✕]  [✓]
+      #                       1.0  0.8  0.6
+      #                      /            \
+      #                [✕] [✓] [✕]      [✓] [✓]
+      #                2.0 1.8 1.6      1.6 1.4
+      #               /     /     \     /     \
+      #
+      # The computational budget is in terms of how many candidate solutions we
+      # will test to determine if they are counterexamples. As a result, the
+      # heuristic value of a candidate solution would seem to require testing
+      # each of its ancestors and their siblings. For now that is what we'll do,
+      # but in the future there may be a way to make do with partial information
+      # and reserve the budget for exploring the tree more deeply.
+
+      fitness  = 0
+      queue    = _weight(shrink.call(shrunk), 0, 1)
+      _counter = counter.shrunk
+
+      until queue.empty? or _counter.total >= options.max_shrink
+        c = queue.shift
+
+        catch(:skip) do
+          if prop.call(c.value, _counter)
+            _counter.ok += 1
+          else
+            if c.fitness > fitness
+              fitness = c.fitness
+              shrunk  = c.value
+            end
+
+            _counter.no += 1
+            queue.concat(_weight(shrink.call(c.value), c.fitness+0.5, c.fitness+1.0))
+            queue.sort_by!{|x| -x.fitness }
+          end
+        rescue => e
+          _counter.fail += 1
+          raise e
+        end
+      end
+
+      No.new(random.seed, counter, shrunk)
+    end
+
+    # Search for a simpler example that causes the same exception
+    def fail(random, counter, shrink, shrunk, options, prop, error)
+      return Fail.new(random.seed, counter, shrunk, error) if shrink.nil?
+
+      fitness  = 0
+      queue    = _weight(shrink.call(shrunk), 0, 1)
+      _counter = counter.shrunk
+
+      until queue.empty? or _counter.total >= options.max_shrink
+        c = queue.shift
+
+        catch(:skip) do
+          if prop.call(c.value, _counter)
+            _counter.ok += 1
+          else
+            _counter.no += 1
+          end
+        rescue => e
+          # TODO: Do we care if this exception is different from `error`?
+
+          if c.fitness > fitness
+            fitness = c.fitness
+            shrunk  = c.value
+          end
+
+          _counter.fail += 1
+          queue.concat(_weight(shrink.call(c.value), c.fitness+0.5, c.fitness+1.0))
+          queue.sort_by!{|x| -x.fitness }
+        end
+      end
+
+      Fail.new(random.seed, counter, shrunk, error)
+    end
+  end
+
+end

data/lib/forall/counter.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Forall
+  class Counter
+    attr_accessor :ok, :no, :skip, :fail, :steps
+
+    attr_reader :shrunk, :labels
+
+    def initialize(top = true)
+      @ok      = 0
+      @no      = 0
+      @skip    = 0
+      @fail    = 0
+      @steps   = 0
+      @shrunk  = Counter.new(false) if top
+      @labels  = Hash.new{|h,k| h[k] = 0 }
+      @private = nil
+    end
+
+    def total
+      @ok + @no + @skip + @fail
+    end
+
+    def test
+      @ok + @no + @fail
+    end
+
+    def skip!
+      @skip += 1
+      throw :skip, true
+    end
+
+    def label!(*names)
+      names.each do |x|
+        @labels[x] += 1
+      end
+    end
+  end
+end

data/lib/forall/input.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+class Forall
+  class Input
+    class << self
+      # @param value [Proc | Enumerable]
+      def build(value = nil, &block)
+        if Input === value
+          value
+        elsif Enumerable === value
+          # This includes Range
+          All.new(value)
+        elsif Proc === value
+          Some.new(value)
+        elsif block_given?
+          Some.new(block)
+        else
+          raise TypeError, "argument must be a Proc or Enumerable"
+        end
+      end
+
+      # @param value [Enumerable]
+      def exhaustive(value = nil, &block)
+        if Enumerable === value
+          All.new(value)
+        else
+          raise TypeError, "argument must be Enumerable"
+        end
+      end
+
+      # @param value [Proc | Array | Enumerator | ... | Enumerable]
+      def sampled(value = nil, &block)
+        if block_given?
+          Some.new(block)
+        elsif Proc === value
+          Some.new(value)
+        elsif Range === value or value.respond_to?(:sample)
+          Some.new(lambda{|rnd| rnd.sample(value) })
+        elsif value.respond_to?(:to_a)
+          array = value.to_a
+          Some.new(lambda{|rnd| rnd.sample(array) })
+        else
+          raise TypeError, "argument must be a Proc or respond_to?(:sample) or respond_to?(:to_a)"
+        end
+      end
+    end
+
+    def shrink(value = nil, &block)
+      if block_given?
+        @shrink = block
+        self
+      else
+        @shrink
+      end
+    end
+
+    # Exhaustive list of possible input values
+    class All < Input
+      def initialize(items)
+        @items  = items
+        @shrink = nil
+      end
+
+      def exhaustive?
+        true
+      end
+
+      def sample(random, count: nil)
+        random.sample(@items, count: count)
+      end
+
+      def each(random, *args)
+        @items.each{|input| yield input }
+      end
+
+      def size
+        @items.size
+      end
+    end
+
+    # Randomized sample of possible input values
+    class Some < Input
+      def initialize(block)
+        @block  = block
+        @shrink = nil
+      end
+
+      def exhaustive?
+        false
+      end
+
+      def sample(random, count: nil)
+        if count.nil?
+          @block.call(random)
+        else
+          count.times.map { @block.call(random) }
+        end
+      end
+
+      def each(random, *args)
+        while true
+          yield @block.call(random, *args)
+        end
+      end
+    end
+  end
+end

data/lib/forall/matchers.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+class Forall
+  module Matchers
+    # @example:
+    #   describe "foo" do
+    #     forall([1,2,3]).check                 {|x,c| c.skip if x.even?; (x*2+1).even? }
+    #     forall(...).check(seed: 999)          {|x,c| c.skip if x.even?; (x*2+1).even? }
+    #     forall(...).check(success_limit: 50)  {|x,c| c.skip if x.even?; (x*2+1).even? }
+    #     forall(...).check(discard_limit: 0.10){|x,c| c.skip if x.even?; (x*2+1).even? }
+    #     forall(...).check(shrink_limit: 10)   {|x,c| c.skip if x.even?; (x*2+1).even? }
+    #
+    #     forall(lambda{|rnd, x| rnd.integer(x)}).
+    #       check(0..9){|x,_| x.between?(0,9)}
+    #   end
+    def forall(input)
+      ForallMatcher.new(input)
+    end
+
+    def sampled(input = nil, &block)
+      Input.sampled(input || block)
+    end
+
+    def exhaustive(input)
+      Input.exhaustive(input)
+    end
+
+    class ForallMatcher
+      def initialize(input)
+        @input = Input.build(input)
+      end
+
+      def check(property = nil, seed: nil, &block)
+        property ||= block
+        random = Forall::Random.new(seed: seed)
+        result = Forall.check(@input, random, nil, &property)
+
+        if defined?(RSpec::Expectations)
+          case result
+          when Forall::Ok
+          when Forall::Vacuous
+            message = "gave up (after %d test%s and %d discarded)\nSeed: %d" %
+              [result.counter.test,
+               result.counter.test == 1 ? "" : "s",
+               result.counter.skip,
+               result.seed]
+
+            error   = ::RSpec::Expectations::ExpectationNotMetError.new(message)
+            source  = property.binding.source_location.join(":")
+            source << " in `block in ...'"
+            error.set_backtrace(source)
+            raise ::RSpec::Expectations::ExpectationNotMetError, message
+
+          else
+            if Forall::No === result or RSpec::Expectations::ExpectationNotMetError === result.error
+              message = "falsified (after %d test%s%s):\nInput: %s\nSeed:  %d" %
+                [result.counter.ok,
+                 result.counter.ok == 1 ? "" : "s",
+                 (result.counter.shrunk.steps == 1 ? "and 1 shrink" :
+                  result.counter.shrunk.steps == 0 ? "" : "and #{result.counter.shrinks} shrinks"),
+                 result.counterexample.inspect,
+                 result.seed]
+
+              error   = ::RSpec::Expectations::ExpectationNotMetError.new(message)
+              source  = property.binding.source_location.join(":")
+              source << " in `block in ...'"
+              error.set_backtrace(source)
+              raise error
+            else
+              message = "exception %s (after %d test%s%s):\nInput: %s\nSeed: %d" %
+                [result.error,
+                 result.counter.ok,
+                 result.counter.ok == 1 ? "" : "s",
+                 (result.counter.shrunk.steps == 1 ? "and 1 shrink" :
+                  result.counter.shrunk.steps == 0 ? "" : "and #{result.counter.shrinks} shrinks"),
+                 result.counterexample.inspect,
+                 result.seed]
+
+              error   = ::RSpec::Expectations::ExpectationNotMetError.new(message)
+              source  = property.binding.source_location.join(":")
+              source << " in `block in ...'"
+              error.set_backtrace(source)
+              raise error
+            end
+          end
+        else
+          result
+        end
+      end
+
+      # Assigns a classification label to a random input
+      # def label(name)
+      #   @labels[name] += 1
+      # end
+
+      # # Declares at least `pct` of inputs should have the given label, or a test
+      # # will fail.
+      # def cover(name, pct=0.01)
+      #   @cover[name] = pct
+      # end
+
+      # # Skip over this input and choose a new random input
+      # def skip
+      # end
+    end
+  end
+end

data/lib/forall/random.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+class Forall
+  class Random
+    def initialize(seed: nil)
+      @prng = ::Random.new(seed || ::Random.new_seed)
+    end
+
+    # @return [Integer]
+    def seed
+      @prng.seed
+    end
+
+    # @return [TrueClass | FalseClass]
+    def boolean
+      @prng.rand >= 0.5
+    end
+
+    # Returns a randomly chosen integer within the given bounds
+    #
+    # @param range [Range<Integer>]
+    # @return [Integer]
+    def integer(range = nil)
+      min = range&.min || -2**64-1
+      max = range&.max ||  2**64+1
+      @prng.rand(min..max)
+    end
+
+    # Returns a randomly chosen floating point number
+    #
+    # @paoram range [Range<Float>]
+    # @return [Float]
+    def float(range = nil)
+      min = range&.min || Float::MIN
+      max = range&.max || Float::MAX
+      @prng.rand(min..max)
+    end
+
+    # Returns a randomly chosen Date within the given bounds
+    #
+    # @param range [Range<Date>]
+    # @return [Date]
+    def date(range = nil)
+      min = (range&.min || Date.civil(0000, 1, 1)).to_time
+      max = (range&.max || Date.civil(9999,12,31)).to_time + 86399
+      Time.at(float(min.to_f .. max.to_f)).to_date
+    end
+
+    # Returns a randomly chosen Time within the given bounds
+    #
+    # @param range [Range<Time>]
+    # @return [Time]
+    def time(range = nil, utc: nil)
+      min = range&.min || Time.utc(0000,1,1,0,0,0)
+      max = range&.max || Time.utc(9999,12,31,23,59,59)
+      rnd = float(min.to_f .. max.to_f)
+
+      if utc or (utc.nil? and (min.utc? or max.utc?))
+        Time.at(rnd).utc
+      else
+        Time.at(rnd)
+      end
+    end
+
+    # Returns a randomly chosen DateTime within the given bounds
+    #
+    # @param range [Range<DateTime>]
+    # @return [DateTime]
+    def datetime(range = nil)
+      min = range&.min&.to_time
+      max = range&.max&.to_time
+      time(min..max).to_datetime
+    end
+
+    # Returns a randomly chosen range within the given bounds
+    #
+    # @param range [Range<Object>]
+    # @param width [Integer]
+    # @return [Range]
+    def range(range, width: nil)
+      min = range.min
+      max = range.max
+
+      if width.nil?
+        case min or max
+        when Float
+          a = float(range)
+          b = float(range)
+        when Integer
+          a = integer(range)
+          b = integer(range)
+        when Time
+          a = time(range)
+          b = time(range)
+        when Date
+          a = date(range)
+          b = date(range)
+        when DateTime
+          a = datetime(range)
+          b = datetime(range)
+        else
+          a, b = choose(range, count: 2)
+        end
+
+        if a < b
+          min = a
+          max = b
+        else
+          min = b
+          max = a
+        end
+      else
+        # Randomly choose a width within given bounds
+        width = choose(width) if Enumerable === width
+
+        case min or max
+        when Float
+          min = float(min: min, max: max-width)
+          max = min+width-1
+        when Integer
+          min = integer(min: min, max: max-width)
+          max = min+width-1
+        else
+          all = (min..max).to_a
+          max = all.size-1
+
+          # Randomly choose element indices
+          min = integer(min: 0, max: max-width)
+          max = min+width-1
+
+          min = all[min]
+          max = all[max]
+        end
+      end
+
+      min..max
+    end
+
+    # Returns a uniformly random chosen element(s) from the given Enumerable
+    #
+    # @param items [Input::Some | Input::All | Range | Array]
+    # @return [Object]
+    def sample(items, count: nil)
+      case items
+      when Input
+        items.sample(self, count: count)
+      when Range
+        method =
+          if                           Integer  === (items.min || items.max) then :integer
+          elsif                        Float    === (items.min || items.max) then :float
+          elsif                        Time     === (items.min || items.max) then :time
+          elsif defined?(Date)     and Date     === (items.min || items.max) then :date
+          elsif defined?(DateTime) and DateTime === (items.min || items.max) then :datetime
+          else
+            # NOTE: This is memory inefficient
+            items = items.to_a
+
+            if count.nil?
+              return items.sample(random: @prng)
+            else
+              return count.times.map{|_| items.sample(random: @prng) }
+            end
+          end
+
+        if count.nil?
+          send(method, items)
+        else
+          count.times.map{|_| send(method, items) }
+        end
+      else
+        unless items.respond_to?(:sample)
+          # NOTE: This works across many types but is memory inefficient
+          items = items.to_a
+        end
+
+        if count.nil?
+          items.sample(random: @prng)
+        else
+          # Sample *with* replacement
+          count.times.map{|_| items.sample(random: @prng) }
+        end
+      end
+    end
+
+    alias_method :choose, :sample
+
+    # Returns a uniformly random chosen element(s) from the given Enumerable
+    #
+    # @param items [Array<Object>]
+    # @param freqs [Array<Numeric>]
+    # @param count [Numeric]
+    # @return [Object]
+    def weighted(items, freqs, count: nil)
+      unless items.size == freqs.size
+        raise ArgumentError, "items and frequencies must have same size"
+      end
+
+      # This runs in O(n) time where n is the number of possible items. This is
+      # not dependent on `count`, the number of requested items.
+      if count.nil?
+        sum = freqs[0].to_f
+        res = items[0]
+
+        (1..items.size - 1).each do |i|
+          sum += freqs[i]
+          p = freqs[i] / sum
+          j = @prng.rand
+          res = items[i] if j <= p
+        end
+      else
+        sum = freqs[0, count].sum.to_f
+        res = items[0, count]
+
+        (count..items.size).each do |i|
+          sum += freqs[i]
+          p = count * freqs[i] / sum
+          j = @prng.rand
+          res[@prng.rand(count)] = items[i] if j <= p
+        end
+      end
+
+      res
+    end
+
+    # @param items [Array<A>]
+    # @return [Array<A>]
+    def shuffle(items)
+      items.shuffle(random: @prng)
+    end
+
+    # Generates a random permutation of the given size
+    #
+    # @param size [Intege]
+    # @return [Array<Integer>]
+    def permutation(size: nil)
+      (0..(size || integer(0..64))-1).to_a.shuffle!(random: @prng)
+    end
+
+    # Generates an Array by repeatedly calling a block that returns a random
+    # value.
+    #
+    # @example
+    #   rnd.array         {|n| n }          #=> [0,1,2,3,...]
+    #   rnd.array(10..50) { integer(0..9) } #=> [8,2,1,1,...]
+    #
+    # @return Array
+    def array(size: nil)
+      size ||= integer(0..64)
+      size = choose(size) if Range === size
+      size.times.map{|n| yield n }
+    end
+
+    # Generates a Hash by repeatedly calling a block that returns a random [key,
+    # val] pair.
+    #
+    # @yieldparam [Integer]
+    # @yieldreturn [Array<K, V>]
+    # @return [Hash<K, V>]
+    def hash(size: nil)
+      size ||= integer(0..64)
+      size = choose(size) if Range === size
+      hash = {}
+
+      until hash.size >= size
+        k, v = yield(hash.size)
+        hash[k] = v
+      end
+
+      hash
+    end
+
+    def set(size: nil)
+      size ||= integer(0..64)
+      size = choose(size) if Range === size
+      set  = Set.new
+
+      until set.size == size
+        set << yield(set.size)
+      end
+
+      set
+    end
+  end
+end

data/lib/forall/shrink.rb

@@ -0,0 +1,49 @@
+class Forall
+  class Shrink
+    def boolean(x)
+      x ? [false] : []
+    end
+
+    def integer(x, range = 0..2**64-1)
+    end
+
+    def float(x, range = 0..Float::MAX)
+    end
+
+    def string
+      # TODO
+    end
+
+    def date
+      # TODO
+    end
+
+    def time
+      # TODO
+    end
+
+    def datetime
+      # TODO
+    end
+
+    def range(x, range = nil, width: nil)
+    end
+
+    def sample(items, count: nil)
+    end
+
+    alias_method :choose, :sample
+
+    def permutation(x, size: nil)
+    end
+
+    def array(xs, size: nil)
+    end
+
+    def hash(x, size: nil)
+    end
+
+    def set(x, size: nil)
+    end
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: forall
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Kyle Putnam
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-05-04 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/forall.rb
+- lib/forall/counter.rb
+- lib/forall/input.rb
+- lib/forall/matchers.rb
+- lib/forall/random.rb
+- lib/forall/shrink.rb
+homepage: https://github.com/kputnam/forall
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Ruby generative property test library (ala QuickCheck)
+test_files: []