predicator 1.2.1 → 2.0.0

data/lib/predicator/evaluator/json_functions.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "json"
+require "strscan"
+require_relative "json_scan"
+
+module Predicator
+  module Evaluator
+    # The library's JSON functions, over Ruby's standard json library (a
+    # default gem, not a dependency).
+    #
+    # `JSON.stringify` writes a value as JSON with every map's keys in sorted
+    # order, the order the conformance corpus fixes (a map has no order of
+    # its own). A Float is written as the reference writes one ({FloatText}).
+    # The values JSON has no form for are written as the reference writes
+    # them: a date and an instant as a JSON string of their ISO 8601 text, a
+    # duration as an object of its eight units, and undefined as the string
+    # "undefined". A value with no JSON form at all (a map key that is not a
+    # String, a String that is not valid in its encoding, an object outside
+    # the value domain) is written as the value's inspect, as the reference
+    # falls back to its own inspect, so the function does not fail on one.
+    #
+    # `JSON.parse` reads a JSON text into lists, maps with String keys and
+    # scalars; a key an object repeats keeps its first value, as the
+    # reference reads one ({JsonRead}). A text that is not JSON
+    # is refused with the reference's sentence naming the first fault and its
+    # offset in bytes ({JsonScan}): `Invalid JSON: unexpected byte 0x6F at
+    # position 1`, or `Invalid JSON: unexpected end of input at position 4`.
+    # Where the reference reports a sequence it cannot read (an unpaired
+    # surrogate escape, a number too large for a Float), the sentence is
+    # `Invalid JSON: unexpected sequence at position N`, N the offset where
+    # the sequence starts.
+    #
+    # Both refuse a list or a map nested more than {DEPTH_LIMIT} levels deep
+    # with the reason depth_limit_exceeded, the limit and the reason the
+    # tagged codec keeps; the outermost list or map is level one.
+    module JsonFunctions
+      # The deepest level of lists and maps either function reads or writes.
+      DEPTH_LIMIT = 100
+
+      # A value with no JSON form, somewhere inside the value written.
+      class NoForm < StandardError; end
+
+      # The JSON text of value, a list or a map at level depth.
+      def self.text(value, depth)
+        case value
+        when nil, true, false, Integer, String then string(value)
+        when Float then FloatText.spell(value)
+        when Array then deep(depth) && "[#{value.map { |member| text(member, depth + 1) }.join(",")}]"
+        when Hash then map(value, depth)
+        else tagged(value, depth)
+        end
+      end
+
+      # A scalar's JSON text; a String that is not valid in its encoding has
+      # none.
+      def self.string(value)
+        JSON.generate(value)
+      rescue JSON::GeneratorError, EncodingError
+        raise NoForm
+      end
+
+      def self.map(value, depth)
+        deep(depth)
+        raise NoForm unless value.each_key.all?(String)
+
+        members = value.keys.sort.map { |key| "#{string(key)}:#{text(value[key], depth + 1)}" }
+        "{#{members.join(",")}}"
+      end
+
+      def self.tagged(value, depth)
+        case value
+        when Undefined then string("undefined")
+        when Predicator::Date, Predicator::DateTime then string(value.to_s)
+        when Duration then map(Access.units(value), depth)
+        else raise NoForm
+        end
+      end
+
+      def self.deep(depth)
+        return true if depth <= DEPTH_LIMIT
+
+        raise Calls::Refusal, "depth_limit_exceeded"
+      end
+
+      private_class_method :string, :map, :tagged, :deep
+
+      private
+
+      def json_stringify(value)
+        JsonFunctions.text(value, 1)
+      rescue JsonFunctions::NoForm
+        value.inspect
+      end
+
+      def json_parse(text)
+        raise Calls::Refusal, "JSON.parse expects a string argument" unless text.is_a?(String)
+
+        fault = JsonScan.fault(text, DEPTH_LIMIT)
+        raise Calls::Refusal, fault if fault
+
+        JsonRead.new(text).value
+      rescue JSON::ParserError
+        # The scan accepts only what the reference reads, so Ruby's parser
+        # does not refuse a scalar of it; if it ever does, its own text stays
+        # here.
+        raise Calls::Refusal, "Invalid JSON: the text could not be read"
+      end
+    end
+    private_constant :JsonFunctions
+
+    # Reads a JSON text {JsonScan} has accepted, its bytes read as UTF-8,
+    # into lists, maps and scalars. Each scalar's own text is read by Ruby's
+    # json library; the lists and maps are built here, because that library
+    # keeps the last value of a key an object repeats and the reference
+    # keeps the first. The scan has bounded the nesting, so the reader
+    # recurses.
+    class JsonRead
+      # The bytes a JSON text may hold between tokens.
+      SPACE = /[ \t\n\r]*/
+      # A scalar's text: a string, a number or a literal.
+      SCALAR = /"(?:[^"\\]|\\.)*"|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|true|false|null/n
+
+      def initialize(text)
+        @scanner = StringScanner.new(text.b)
+      end
+
+      # The value at the scan, and past it.
+      def value
+        @scanner.skip(SPACE)
+        if @scanner.skip("[") then list
+        elsif @scanner.skip("{") then map
+        else JSON.parse(@scanner.scan(SCALAR).force_encoding(Encoding::UTF_8))
+        end
+      end
+
+      private
+
+      def list
+        items = []
+        items << value until closed?("]")
+        items
+      end
+
+      def map
+        members = {}
+        until closed?("}")
+          key = value
+          @scanner.skip(SPACE)
+          @scanner.skip(":")
+          member = value
+          members[key] = member unless members.key?(key)
+        end
+        members
+      end
+
+      # Whether the container closes here, past a comma between members.
+      def closed?(closer)
+        @scanner.skip(SPACE)
+        @scanner.skip(",")
+        @scanner.skip(SPACE)
+        !@scanner.skip(closer).nil?
+      end
+    end
+    private_constant :JsonRead
+  end
+end

data/lib/predicator/evaluator/json_scan.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+module Predicator
+  module Evaluator
+    # What every part of {JsonScan} shares: the byte at its offset, `@at`,
+    # over its bytes, `@bytes`, and the faults it throws as `:fault` with the
+    # sentence the text is refused with. The reference's reader reports
+    # three kinds: an unexpected byte at its offset, the end of the input,
+    # and a sequence it cannot read (its own report of that last kind is a
+    # term of its host language, so a sequence here is reported in a
+    # sentence of the same form as the other two).
+    module JsonFaults
+      private
+
+      # The byte at the scan, or nil at the end.
+      def here = @bytes.getbyte(@at)
+
+      # The byte at the scan; the end of the text is a fault here.
+      def present = here || ending(@at)
+
+      def expect(byte)
+        unexpected unless present == byte
+        @at += 1
+      end
+
+      def unexpected
+        throw :fault, format("Invalid JSON: unexpected byte 0x%<byte>02X at position %<at>d", byte: present, at: @at)
+      end
+
+      def ending(at) = throw(:fault, "Invalid JSON: unexpected end of input at position #{at}")
+      def sequence(at) = throw(:fault, "Invalid JSON: unexpected sequence at position #{at}")
+    end
+    private_constant :JsonFaults
+
+    # A JSON string's bytes, as the reference's reader reads them: a control
+    # byte must be escaped; the bytes from 0x80 must be well-formed UTF-8,
+    # refused at the first byte where the sequence breaks; and a `\u` escape
+    # must have its four hexadecimal digits, a high surrogate must be
+    # followed by a `\u` escape of a low surrogate, and a low surrogate must
+    # not stand alone.
+    #
+    # Where the reference reads an escape's bytes ahead, it reports a text
+    # too short to hold them (six bytes from the backslash, twelve for a
+    # surrogate pair) as the end of the whole input, not of the string.
+    module JsonStrings
+      include JsonFaults
+
+      # The bytes a backslash in a string may precede, but for `u`.
+      ESCAPES = '"\\/bfnrt'.bytes.freeze
+      # For each UTF-8 lead byte, the range each byte after it must fall in.
+      CONTINUATIONS = [
+        [0xC2..0xDF, [0x80..0xBF]], [0xE0..0xE0, [0xA0..0xBF, 0x80..0xBF]],
+        [0xE1..0xEC, [0x80..0xBF] * 2], [0xED..0xED, [0x80..0x9F, 0x80..0xBF]], [0xEE..0xEF, [0x80..0xBF] * 2],
+        [0xF0..0xF0, [0x90..0xBF] + ([0x80..0xBF] * 2)], [0xF1..0xF3, [0x80..0xBF] * 3],
+        [0xF4..0xF4, [0x80..0x8F] + ([0x80..0xBF] * 2)]
+      ].freeze
+      # The high and the low surrogates.
+      HIGH = 0xD800..0xDBFF
+      LOW = 0xDC00..0xDFFF
+
+      private
+
+      # A string, from its opening quote past its closing one.
+      def string
+        @at += 1
+        content(present) until here == 0x22
+        @at += 1
+        :after
+      end
+
+      # One byte of a string's content, and what it opens.
+      def content(byte)
+        case byte
+        when 0x5C then escape
+        when 0x00..0x1F then unexpected
+        when 0x80..0xFF then utf8(byte)
+        else @at += 1
+        end
+      end
+
+      def utf8(lead)
+        ranges = CONTINUATIONS.find { |leads, _| leads.cover?(lead) }&.last || unexpected
+        @at += 1
+        ranges.each do |range|
+          unexpected unless range.cover?(present)
+          @at += 1
+        end
+      end
+
+      # An escape, from its backslash.
+      def escape
+        start = @at
+        @at += 1
+        return unicode(start) if present == 0x75
+
+        unexpected unless ESCAPES.include?(present)
+        @at += 1
+      end
+
+      def unicode(start)
+        code = code_at(start)
+        @at = start + 6
+        return unless HIGH.cover?(code) || LOW.cover?(code)
+
+        sequence(start) if LOW.cover?(code)
+        low_half(start)
+      end
+
+      # The low surrogate a high one at start must be followed by.
+      def low_half(start)
+        ending(@bytes.bytesize) if @bytes.bytesize - start < 12
+        unexpected unless @bytes.byteslice(start + 6, 2) == "\\u"
+        sequence(start) unless LOW.cover?(digits(start + 6))
+        @at = start + 12
+      end
+
+      # The code a `\u` escape at start names.
+      def code_at(start)
+        ending(@bytes.bytesize) if @bytes.bytesize - start < 6
+        digits(start) || sequence(start)
+      end
+
+      def digits(start)
+        hex = @bytes.byteslice(start + 2, 4)
+        hex.to_i(16) if hex.match?(/\A\h{4}\z/)
+      end
+    end
+    private_constant :JsonStrings
+
+    # The scalars of a JSON text besides its strings: numbers and the three
+    # literals. A number too large for a Float is a sequence the reference
+    # cannot read, reported at the number's first byte.
+    module JsonScalars
+      include JsonStrings
+
+      # The literal each first byte begins.
+      LITERALS = { 0x74 => "true", 0x66 => "false", 0x6E => "null" }.transform_values(&:bytes).freeze
+
+      private
+
+      def number
+        start = @at
+        @at += 1 if here == 0x2D
+        present == 0x30 ? @at += 1 : digits_run
+        fraction_and_exponent
+        token = @bytes.byteslice(start, @at - start)
+        sequence(start) if token.match?(/[.eE]/) && !JSON.parse(token).finite?
+        :after
+      end
+
+      def fraction_and_exponent
+        if here == 0x2E
+          @at += 1
+          digits_run
+        end
+        return unless [0x65, 0x45].include?(here)
+
+        @at += 1
+        @at += 1 if [0x2B, 0x2D].include?(here)
+        digits_run
+      end
+
+      def digits_run
+        unexpected unless present.between?(0x30, 0x39)
+        @at += 1 while here&.between?(0x30, 0x39)
+      end
+
+      def literal(bytes)
+        bytes.each { |byte| expect(byte) }
+        :after
+      end
+    end
+    private_constant :JsonScalars
+
+    # Finds the first fault in a JSON text, by byte offset, as the
+    # reference's JSON reader reports it. JSON.parse is asked to read only a
+    # text this has accepted, because the sentence a malformed text is
+    # refused with names the offending byte and its offset, and Ruby's
+    # parser's own messages are its own.
+    #
+    # Content after the value is refused at the first byte after the value,
+    # a space included, as the reference refuses it. The scan walks the
+    # text's bytes once, keeping the open lists and maps on a stack of its
+    # own rather than recursing, so a deeply nested text cannot exhaust
+    # Ruby's stack. It holds no state between scans.
+    class JsonScan
+      include JsonScalars
+
+      # The bytes a JSON text may hold between tokens.
+      SPACE = [0x20, 0x09, 0x0A, 0x0D].freeze
+      # The byte that closes each kind of container.
+      CLOSERS = { array: 0x5D, object: 0x7D }.freeze
+
+      # Why text is not a JSON text: "Invalid JSON: " and the fault, or
+      # "depth_limit_exceeded" when it nests past limit levels; nil when it
+      # is one.
+      #
+      # @param text [String] the text, its bytes read as UTF-8
+      # @param limit [Integer] the deepest level of lists and maps accepted
+      # @return [String, nil]
+      def self.fault(text, limit) = new(text, limit).fault
+
+      def initialize(text, limit)
+        @bytes = text.b
+        @limit = limit
+        @at = 0
+        @open = []
+      end
+
+      def fault
+        catch(:fault) do
+          space
+          walk
+          trailing
+        end
+      end
+
+      private
+
+      # Only space may follow the value; anything else is refused at the
+      # first byte after the value.
+      def trailing
+        after = @at
+        space
+        return if @at == @bytes.bytesize
+
+        @at = after
+        unexpected
+      end
+
+      # Scans one value and, while containers are open, the rest of each.
+      def walk
+        state = value
+        state = state == :first ? first_member : next_member until state == :done
+      end
+
+      def first_member
+        space
+        return close if closer?
+
+        member_start
+        value
+      end
+
+      def next_member
+        return :done if @open.empty?
+
+        space
+        return close if closer?
+
+        expect(0x2C)
+        space
+        member_start
+        value
+      end
+
+      def closer? = here == CLOSERS.fetch(@open.last)
+
+      def close
+        @at += 1
+        @open.pop
+        :after
+      end
+
+      # An object's member opens with its key and a colon.
+      def member_start
+        return unless @open.last == :object
+
+        unexpected unless present == 0x22
+        string
+        space
+        expect(0x3A)
+        space
+      end
+
+      # Scans a scalar (answering :after) or enters a container (:first).
+      def value
+        byte = here
+        case byte
+        when 0x5B then enter(:array)
+        when 0x7B then enter(:object)
+        when 0x22 then string
+        when 0x2D, 0x30..0x39 then number
+        when *LITERALS.keys then literal(LITERALS.fetch(byte))
+        else unexpected
+        end
+      end
+
+      def enter(kind)
+        throw :fault, "depth_limit_exceeded" if @open.size >= @limit
+
+        @open.push(kind)
+        @at += 1
+        :first
+      end
+
+      def space
+        @at += 1 while SPACE.include?(here)
+      end
+    end
+    private_constant :JsonScan
+  end
+end

data/lib/predicator/evaluator/library.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "math_functions"
+require_relative "json_functions"
+
+module Predicator
+  module Evaluator
+    # The package's function library: the functions the reference ships as
+    # its builtins (its string, list, date, math and JSON functions), by the
+    # names a `call` reaches them under, with the counts of arguments each
+    # takes. {Calls} looks a name up here after the caller's own functions.
+    #
+    # Each function is a private method of {Machine} that takes the call's
+    # arguments and answers its value, or raises {Calls::Refusal} with the
+    # sentence the reference refuses those arguments with. This module holds
+    # the string, list and date functions; {MathFunctions} and
+    # {JsonFunctions} hold the rest.
+    #
+    # A string's length and a substring's offsets count grapheme clusters,
+    # and `index_of` answers a byte offset into the string's UTF-8, as the
+    # reference's own string functions do. `trim` removes the characters
+    # with the Unicode White_Space property from both ends.
+    module Library
+      include MathFunctions
+      include JsonFunctions
+
+      # One library function: the counts of arguments it takes and the
+      # Machine method that runs it.
+      Entry = Data.define(:counts, :handler)
+
+      # The library, by name.
+      FUNCTIONS = {
+        "len" => [1..1, :string_len], "upper" => [1..1, :string_upper], "lower" => [1..1, :string_lower],
+        "trim" => [1..1, :string_trim], "starts_with" => [2..2, :string_starts_with?],
+        "ends_with" => [2..2, :string_ends_with?], "substring" => [2..3, :string_substring],
+        "index_of" => [2..2, :string_index_of], "concat" => [2..2, :list_concat],
+        "Date.year" => [1..1, :date_year], "Date.month" => [1..1, :date_month], "Date.day" => [1..1, :date_day],
+        "Date.now" => [0..0, :date_now],
+        "Math.pow" => [2..2, :math_pow], "Math.sqrt" => [1..1, :math_sqrt], "Math.abs" => [1..1, :math_abs],
+        "Math.floor" => [1..1, :math_floor], "Math.ceil" => [1..1, :math_ceil], "Math.round" => [1..1, :math_round],
+        "Math.min" => [2..2, :math_min], "Math.max" => [2..2, :math_max], "Math.random" => [0..0, :math_random],
+        "JSON.stringify" => [1..1, :json_stringify], "JSON.parse" => [1..1, :json_parse]
+      }.transform_values { |counts, handler| Entry.new(counts:, handler:) }.freeze
+
+      # The ends `trim` removes.
+      LEADING_SPACE = /\A\p{White_Space}+/
+      TRAILING_SPACE = /\p{White_Space}+\z/
+      # What substring says of arguments of the wrong types, with two
+      # arguments (true) and with three.
+      SUBSTRING_TYPES = {
+        true => "substring() expects a string and an integer start index",
+        false => "substring() expects a string, an integer start index, and an integer length"
+      }.freeze
+      private_constant :LEADING_SPACE, :TRAILING_SPACE, :SUBSTRING_TYPES
+
+      private
+
+      def string_len(value) = string_arg("len", value).grapheme_clusters.size
+      def string_upper(value) = string_arg("upper", value).upcase
+      def string_lower(value) = string_arg("lower", value).downcase
+      def string_trim(value) = string_arg("trim", value).sub(LEADING_SPACE, "").sub(TRAILING_SPACE, "")
+
+      def string_arg(name, value)
+        return value if value.is_a?(String)
+
+        raise Calls::Refusal, "#{name}() expects a string argument"
+      end
+
+      def string_starts_with?(value, prefix) = two_strings("starts_with", value, prefix).start_with?(prefix)
+      def string_ends_with?(value, suffix) = two_strings("ends_with", value, suffix).end_with?(suffix)
+      def string_index_of(value, sub) = two_strings("index_of", value, sub).byteindex(sub) || -1
+
+      def two_strings(name, value, other)
+        return value if value.is_a?(String) && other.is_a?(String)
+
+        raise Calls::Refusal, "#{name}() expects two string arguments"
+      end
+
+      # The grapheme clusters from start, all of them or length of them. A
+      # start or a length past the end answers what the string has. The
+      # call's shape is its count of arguments, so a third argument of null
+      # is a length that is not an Integer, not a missing one.
+      def string_substring(value, start, *length)
+        two = length.empty?
+        substring_types(value, start, length, two)
+        substring_signs(start, length.first, two)
+        clusters = value.grapheme_clusters
+        (clusters[start, two ? clusters.size : length.first] || []).join
+      end
+
+      def substring_types(value, start, length, two)
+        return if value.is_a?(String) && start.is_a?(Integer) && length.all?(Integer)
+
+        raise Calls::Refusal, SUBSTRING_TYPES.fetch(two)
+      end
+
+      def substring_signs(start, length, two)
+        return unless start.negative? || (!two && length.negative?)
+
+        raise Calls::Refusal, "substring() expects a non-negative start index#{" and length" unless two}"
+      end
+
+      def list_concat(left, right)
+        return left + right if left.is_a?(Array) && right.is_a?(Array)
+
+        raise Calls::Refusal, "concat() expects two list arguments"
+      end
+
+      def date_year(value) = civil_date("Date.year", value).year
+      def date_month(value) = civil_date("Date.month", value).month
+      def date_day(value) = civil_date("Date.day", value).day
+
+      # The date a date names, or the date in UTC an instant falls on.
+      def civil_date(name, value)
+        return value if value.is_a?(Predicator::Date)
+        return Civil.date_of(value) if value.is_a?(Predicator::DateTime)
+
+        raise Calls::Refusal, "#{name}() expects a date or datetime argument"
+      end
+
+      # The current time of this evaluation: the clock the caller passed,
+      # read at most once per evaluation, as relative_date reads it.
+      def date_now = now
+    end
+    private_constant :Library
+  end
+end

data/lib/predicator/evaluator/math_functions.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Predicator
+  module Evaluator
+    # The library's math functions, as the reference's own: each takes
+    # numbers (an Integer or a Float) and refuses anything else with the
+    # reference's sentence for it.
+    #
+    # Integers stay exact where the reference keeps them exact: `Math.pow`
+    # of two Integers with a non-negative exponent, `Math.sqrt` of an
+    # Integer with an exact Integer root, `Math.abs`, `Math.min` and
+    # `Math.max` of Integers. `Math.floor`, `Math.ceil` and `Math.round`
+    # answer an Integer, rounding half away from zero. Where the reference's
+    # float arithmetic has no real, finite answer (`Math.pow(-8, 0.5)`, an
+    # overflow), the call is refused rather than answering a complex number
+    # or an infinity, which the value domain does not hold.
+    module MathFunctions
+      private
+
+      def math_pow(base, exponent)
+        two_numbers("Math.pow", base, exponent)
+        return base**exponent if base.is_a?(Integer) && exponent.is_a?(Integer) && !exponent.negative?
+
+        power = base.to_f**exponent.to_f
+        return power if power.is_a?(Float) && power.finite?
+
+        raise Calls::Refusal, "Math.pow has no finite real answer for #{base} and #{exponent}"
+      end
+
+      def math_sqrt(value)
+        raise Calls::Refusal, "Math.sqrt expects a numeric argument" unless number?(value)
+        raise Calls::Refusal, "Math.sqrt expects a non-negative number" if value.negative?
+
+        if value.is_a?(Integer)
+          root = Integer.sqrt(value)
+          return root if root * root == value
+        end
+        Math.sqrt(value)
+      end
+
+      def math_abs(value) = numeric("Math.abs", value).abs
+      def math_floor(value) = numeric("Math.floor", value).to_f.floor
+      def math_ceil(value) = numeric("Math.ceil", value).to_f.ceil
+      def math_round(value) = numeric("Math.round", value).round
+
+      # The smaller of the two, the first when they are equal.
+      def math_min(left, right) = two_numbers("Math.min", left, right) && [left, right].min
+
+      # The larger of the two, the first when they are equal.
+      def math_max(left, right) = two_numbers("Math.max", left, right) && [left, right].max
+
+      # A Float from 0.0 up to 1.0, from a generator of its own, so that no
+      # state outside this call is read or moved.
+      def math_random = Random.new.rand
+
+      def number?(value) = value.is_a?(Integer) || value.is_a?(Float)
+
+      def numeric(name, value)
+        return value if number?(value)
+
+        raise Calls::Refusal, "#{name} expects a numeric argument"
+      end
+
+      def two_numbers(name, left, right)
+        return true if number?(left) && number?(right)
+
+        raise Calls::Refusal, "#{name} expects two numeric arguments"
+      end
+    end
+    private_constant :MathFunctions
+  end
+end