janeway-jsonpath 1.0.0 → 1.2.0

data/lib/janeway/iregexp.rb

@@ -0,0 +1,402 @@
+# frozen_string_literal: true
+
+require_relative 'error'
+
+module Janeway
+  # Validates an I-Regexp and translates it to an equivalent Ruby Regexp.
+  #
+  # The pattern is parsed with the grammar from RFC 9485 section 3, and the Ruby source is emitted
+  # while parsing, so a pattern that is not an I-Regexp never reaches Regexp.new, and characters
+  # that Ruby treats as syntax but I-Regexp treats as ordinary characters are escaped.
+  #
+  # The limits below are this library's own choices. RFC 9485 section 8 allows an implementation to
+  # limit range quantifiers in composability and range, but gives no values.
+  # @see https://www.rfc-editor.org/rfc/rfc9485.html
+  module IRegexp
+    # Longest pattern accepted, in characters.
+    MAX_LENGTH = 10_000
+
+    # Largest number accepted in a range quantifier such as a{2,4}.
+    MAX_REPEAT = 1_000
+
+    # Deepest nesting of groups accepted. This also bounds the recursion depth of the parser.
+    MAX_GROUP_DEPTH = 100
+
+    # Seconds a single match may take, where the Ruby version supports a per-regexp timeout (3.2+).
+    MATCH_TIMEOUT = 1
+
+    # Regexp.new accepts the timeout: keyword from Ruby 3.2, which is also when Regexp.timeout appeared.
+    # Probe with respond_to? and not by trying the keyword: on an older Ruby a trailing hash would be
+    # read as the positional options argument, and a pattern would silently become case-insensitive.
+    TIMEOUT_SUPPORTED = Regexp.respond_to?(:timeout)
+
+    # Error raised when a match exceeds MATCH_TIMEOUT. Regexp::TimeoutError only exists on Ruby 3.2+,
+    # elsewhere this is a class that is never raised for that reason.
+    TIMEOUT_ERROR = defined?(Regexp::TimeoutError) ? Regexp::TimeoutError : Class.new(StandardError)
+
+    # Unicode general category names, from the IsCategory production of RFC 9485 section 3.
+    CATEGORY = /\A(?:L[lmotu]?|M[cen]?|N[dlo]?|P[c-fios]?|Z[lps]?|S[ckmo]?|C[cfno]?)\z/
+
+    # Base class for the errors IRegexp.compile raises. They describe a problem in a pattern, not in a
+    # JSONPath query, so they carry the index of the problem in the pattern and have no query or location.
+    class Error < Janeway::Error
+      # @return [Integer, nil] index of the problem in the pattern, nil when there is no single place
+      attr_reader :offset
+
+      # @param msg [String]
+      # @param offset [Integer, nil]
+      def initialize(msg, offset: nil)
+        super(msg)
+        @offset = offset
+      end
+    end
+
+    # Raised for a pattern that is not an I-Regexp.
+    class Invalid < Error; end
+
+    # Raised for an I-Regexp that exceeds one of the limits above.
+    class LimitExceeded < Error
+      # @return [Integer] number of pattern characters the problem covers
+      attr_reader :length
+
+      # @param msg [String]
+      # @param offset [Integer, nil]
+      # @param length [Integer]
+      def initialize(msg, offset: nil, length: 1)
+        super(msg, offset: offset)
+        @length = length
+      end
+    end
+
+    # Can this string be an I-Regexp at all, as far as its encoding goes?
+    #
+    # RFC 9485 section 1: a pattern is a sequence of Unicode scalar values. Accept UTF-8, and US-ASCII
+    # which is a subset of it. Any other encoding is refused and not transcoded, and that includes a
+    # binary (ASCII-8BIT) string even when its bytes are all ASCII.
+    # @param string [String]
+    # @return [Boolean]
+    def self.valid_encoding?(string)
+      [Encoding::UTF_8, Encoding::US_ASCII].include?(string.encoding) && string.valid_encoding?
+    end
+
+    # Validate an I-Regexp and translate it to a Ruby Regexp.
+    #
+    # @param iregex [String]
+    # @param anchor [Boolean] add anchors to match the string start and end
+    # @return [Regexp]
+    # @raise [Invalid] the pattern is not an I-Regexp
+    # @raise [LimitExceeded] the pattern is an I-Regexp, but is over one of the limits
+    def self.compile(iregex, anchor: true)
+      body = Parser.new(iregex).parse
+      # RFC 9485 section 5.4: enclose the regexp in \A(?: and )\z.
+      body = "\\A(?:#{body})\\z" if anchor
+      # RFC 9485 section 3: the pattern is valid I-Regexp, so a Ruby warning about it (such as a redundant
+      # nested repeat operator in (a*)*) is not the caller's business. A pattern from the input document
+      # must not be able to write to stderr. $VERBOSE is process-wide, so warnings from other threads are
+      # also suppressed during this one compile.
+      old_verbose = $VERBOSE
+      begin
+        $VERBOSE = nil
+        TIMEOUT_SUPPORTED ? Regexp.new(body, timeout: MATCH_TIMEOUT) : Regexp.new(body)
+      rescue RegexpError => e
+        # Deliberate deviation from RFC 9485 section 3: Ruby refuses some patterns that are valid
+        # I-Regexp, such as a{3,2}, and these are treated as not an I-Regexp.
+        raise Invalid, e.message
+      ensure
+        $VERBOSE = old_verbose
+      end
+    end
+
+    # Recursive-descent parser for the ABNF of RFC 9485 section 3, which builds the Ruby regexp source.
+    # Each method is named for the production it implements.
+    class Parser
+      # Characters allowed after a backslash. RFC 9485 section 3, SingleCharEsc.
+      SINGLE_CHAR_ESC = '()*+-.?[]^nrt{|}\\'
+
+      # catEsc and complEsc start with one of these after the backslash, RFC 9485 section 3
+      CATEGORY_ESCAPES = %w[p P].freeze
+
+      # @param iregex [String]
+      def initialize(iregex)
+        @original = iregex
+        @src = iregex.valid_encoding? ? iregex.chars : [] # array: indexing a multibyte String is O(n)
+        @i = 0
+        @out = +''
+        @depth = 0
+        @has_range = false # does the group being parsed contain a range quantifier?
+        @limit = nil # first limit violation, raised only if the pattern is otherwise valid
+      end
+
+      # @return [String] Ruby regexp source, without anchors
+      # @raise [Invalid, LimitExceeded]
+      def parse
+        raise Invalid, 'pattern is not valid UTF-8' unless IRegexp.valid_encoding?(@original) # RFC 9485 section 1
+
+        i_regexp
+        raise Invalid.new("unexpected #{@src[@i].inspect}", offset: @i) if @i < @src.length
+
+        limit("pattern is longer than #{MAX_LENGTH} characters", nil) if @src.length > MAX_LENGTH
+        raise @limit if @limit
+
+        @out
+      end
+
+      private
+
+      def peek
+        @src[@i]
+      end
+
+      # Grammar errors take precedence over limit errors, so a limit violation is remembered and
+      # only raised once the whole pattern is known to be an I-Regexp. The one exception is the group
+      # nesting limit, which is raised as soon as it is hit (see #group): reading on past it is what
+      # would overflow the stack.
+      def limit(msg, offset, length = 1)
+        @limit ||= LimitExceeded.new(msg, offset: offset, length: length)
+      end
+
+      # i-regexp = branch *( "|" branch )
+      # RFC 9485 section 3, the top production.
+      def i_regexp
+        branch
+        while peek == '|'
+          @i += 1
+          @out << '|'
+          branch
+        end
+      end
+
+      # branch = *piece
+      def branch
+        piece while !peek.nil? && peek != '|' && peek != ')'
+      end
+
+      # piece = atom [ quantifier ]
+      def piece
+        contains_range = atom
+        quantifier(contains_range)
+      end
+
+      # atom = NormalChar / charClass / ( "(" i-regexp ")" )
+      # @return [Boolean] true if the atom is a group that contains a range quantifier
+      def atom
+        case peek
+        when '(' then return group
+        when '[' then char_class
+        when '.'
+          # RFC 9485 section 5.4: replace an unescaped dot outside a character class with [^\n\r]
+          @i += 1
+          @out << '[^\n\r]'
+        when '\\' then escape
+        else normal_char(in_class: false)
+        end
+        false
+      end
+
+      # atom = ... / ( "(" i-regexp ")" ), RFC 9485 section 3.
+      # The nesting limit is this library's own choice, which RFC 9485 section 8 permits: it bounds recursion.
+      # Unlike the other limits it is raised at once and not remembered (see #limit), because parsing on
+      # past it is what would overflow the stack. So a too-deeply nested pattern that is also invalid is
+      # reported as LimitExceeded, not Invalid.
+      # @return [Boolean] true if the group contains a range quantifier
+      def group
+        open = @i
+        @i += 1
+        @depth += 1
+        if @depth > MAX_GROUP_DEPTH
+          raise LimitExceeded.new("groups are nested deeper than #{MAX_GROUP_DEPTH}", offset: open)
+        end
+
+        @out << '(?:'
+        outer_has_range = @has_range
+        @has_range = false
+        i_regexp
+        raise Invalid.new('unterminated group', offset: @i) unless peek == ')'
+
+        @i += 1
+        @depth -= 1
+        @out << ')'
+        inner_has_range = @has_range
+        @has_range = outer_has_range || inner_has_range
+        inner_has_range
+      end
+
+      # quantifier = ( "*" / "+" / "?" ) / range-quantifier
+      def quantifier(contains_range)
+        case peek
+        when '*', '+', '?'
+          @out << peek
+          @i += 1
+        when '{' then range_quantifier(contains_range)
+        end
+      end
+
+      # range-quantifier = "{" QuantExact [ "," [ QuantExact ] ] "}"
+      # QuantExact = 1*%x30-39
+      #
+      # RFC 9485 section 8 allows limiting range quantifiers in range and in composability: here a
+      # number above MAX_REPEAT, and a range quantifier applied to a group that contains one.
+      def range_quantifier(contains_range)
+        start = @i
+        counts, finish = scan_range_quantifier(start)
+        raise Invalid.new('malformed range quantifier', offset: start) unless counts
+
+        counts.each do |digits, offset|
+          next if digits.empty? || digits.to_i <= MAX_REPEAT
+
+          limit("repeat count #{digits} exceeds #{MAX_REPEAT}", offset, digits.length)
+        end
+        limit('range quantifier applied to a group that contains a range quantifier', start) if contains_range
+        @has_range = true
+        @out << @src[start...finish].join
+        @i = finish
+      end
+
+      # Match \{(\d+)(?:,(\d*))?\} at start by scanning the character array, which a Regexp cannot
+      # do in O(1) per call on a multibyte string. The digit runs can be long, so no fixed-size slice.
+      # @return [Array, nil] [[[digits, offset], ...], index after the closing brace], nil if malformed
+      def scan_range_quantifier(start)
+        return nil unless @src[start] == '{'
+
+        first_start = start + 1
+        first_end = digits_end(first_start)
+        return nil if first_end == first_start # QuantExact needs at least one digit
+
+        counts = [[@src[first_start...first_end].join, first_start]]
+        pos = first_end
+        if @src[pos] == ','
+          second_start = pos + 1
+          pos = digits_end(second_start)
+          counts << [@src[second_start...pos].join, second_start]
+        end
+        @src[pos] == '}' ? [counts, pos + 1] : nil
+      end
+
+      # @return [Integer] index of the first character at or after pos that is not an ASCII digit
+      def digits_end(pos)
+        pos += 1 while @src[pos]&.match?(/\A[0-9]\z/)
+        pos
+      end
+
+      # charClassExpr = "[" [ "^" ] ( "-" / CCE1 ) *CCE1 [ "-" ] "]"
+      def char_class
+        open = @i
+        @i += 1
+        @out << '['
+        negated = peek == '^'
+        if negated
+          @i += 1
+          @out << '^'
+        end
+        # RFC 9485 section 3: charClassExpr is not allowed to match [^]
+        raise Invalid.new('empty negated character class [^]', offset: open) if negated && peek == ']'
+        # RFC 9485 section 3: charClassExpr has at least one item, so [] is not allowed either
+        raise Invalid.new('empty character class []', offset: open) if peek == ']'
+
+        first = true
+        until peek == ']'
+          raise Invalid.new('unterminated character class', offset: @i) if peek.nil?
+
+          class_item(first)
+          first = false
+        end
+        @i += 1
+        @out << ']'
+      end
+
+      # CCE1 = ( CCchar [ "-" CCchar ] ) / charClassEsc, RFC 9485 section 3
+      def class_item(first)
+        if peek == '-'
+          # a "-" is only a plain character at the start or the end of the class
+          raise Invalid.new("misplaced '-' in character class", offset: @i) unless first || @src[@i + 1] == ']'
+
+          @i += 1
+          @out << '\-'
+        elsif peek == '\\' && CATEGORY_ESCAPES.include?(@src[@i + 1])
+          # charClassEsc is not a CCchar, so it takes no range: a following "-" is rejected by the next item
+          escape
+        else
+          cc_char
+          class_range_end if peek == '-' && @src[@i + 1] != ']'
+        end
+      end
+
+      # "-" CCchar, the end of a range. RFC 9485 section 3, CCE1.
+      def class_range_end
+        @i += 1
+        @out << '-'
+        raise Invalid.new('unterminated range in character class', offset: @i) if peek.nil? || peek == ']'
+
+        cc_char
+      end
+
+      # CCchar = ( %x00-2C / %x2E-5A / %x5E-D7FF / %xE000-10FFFF ) / SingleCharEsc
+      # RFC 9485 section 3: a SingleCharEsc is a CCchar, so it can start or end a range.
+      def cc_char
+        if peek == '\\' then escape(single_char_only: true)
+        else normal_char(in_class: true)
+        end
+      end
+
+      # NormalChar (outside a class) and CCchar (inside a class). RFC 9485 section 3.
+      # In I-Regexp "^" and "$" are ordinary characters, not anchors: the RFC 9485 author confirmed
+      # this, see https://github.com/jsonpath-standard/jsonpath-compliance-test-suite (open issue).
+      def normal_char(in_class:)
+        char = peek
+        raise Invalid.new('unexpected end of pattern', offset: @i) if char.nil?
+        unless allowed_char?(char.ord, in_class)
+          raise Invalid.new("character #{char.inspect} must be escaped", offset: @i)
+        end
+
+        @i += 1
+        # Regexp.escape does not escape "&", but Ruby reads "&&" in a class as set intersection
+        @out << (char == '&' ? '\&' : Regexp.escape(char))
+      end
+
+      # @param code [Integer] code point
+      def allowed_char?(code, in_class)
+        return false if (0xD800..0xDFFF).cover?(code) # surrogates
+
+        if in_class
+          # CCchar: %x00-2C / %x2E-5A / %x5E-D7FF / %xE000-10FFFF
+          code <= 0x2C || (0x2E..0x5A).cover?(code) || code >= 0x5E
+        else
+          # NormalChar: %x00-27 / "," / "-" / %x2F-3E / %x40-5A / %x5E-7A / %x7E-D7FF / %xE000-10FFFF
+          code <= 0x27 || [0x2C, 0x2D].include?(code) || (0x2F..0x3E).cover?(code) ||
+            (0x40..0x5A).cover?(code) || (0x5E..0x7A).cover?(code) || code >= 0x7E
+        end
+      end
+
+      # SingleCharEsc / catEsc / complEsc, RFC 9485 section 3
+      # @param single_char_only [Boolean] refuse catEsc and complEsc, where a CCchar is required
+      def escape(single_char_only: false)
+        start = @i
+        nxt = @src[@i + 1]
+        raise Invalid.new('trailing backslash', offset: start) if nxt.nil?
+
+        if CATEGORY_ESCAPES.include?(nxt)
+          if single_char_only
+            raise Invalid.new("\\#{nxt} is not allowed here, only a single-character escape", offset: start)
+          end
+
+          category_escape(start)
+        elsif SINGLE_CHAR_ESC.include?(nxt)
+          @i += 2
+          @out << { 'n' => '\n', 'r' => '\r', 't' => '\t' }.fetch(nxt) { "\\#{nxt}" }
+        else
+          raise Invalid.new("invalid escape \\#{nxt}", offset: start)
+        end
+      end
+
+      # catEsc = %s"\p{" charProp "}" ; complEsc = %s"\P{" charProp "}", RFC 9485 section 3
+      def category_escape(start)
+        # at most \p{Xx}, 6 characters
+        m = /\A\\[pP]\{([A-Za-z]{1,2})\}/.match(@src[start, 6].join)
+        raise Invalid.new('malformed category escape', offset: start) unless m && CATEGORY.match?(m[1])
+
+        @i = start + m[0].length
+        @out << (@src[start + 1] == 'P' ? "\\p{^#{m[1]}}" : m[0])
+      end
+    end
+  end
+end

data/lib/janeway/lexer.rb

@@ -38,9 +38,23 @@ module Janeway
     TWO_CHAR_LEX_FIRST = TWO_CHAR_LEX.map { |lexeme| lexeme[0] }.freeze
     ONE_OR_TWO_CHAR_LEX = ONE_CHAR_LEX & TWO_CHAR_LEX.map { |str| str[0] }.freeze
 
+    # O(1)-lookup companions to the arrays above. Hash#include? and the reverse
+    # lookup avoid the linear scan that .include? / .key would do per token.
+    LEXEME_TO_TYPE = OPERATORS.invert.freeze
+    ONE_CHAR_LEX_SET = ONE_CHAR_LEX.each_with_object({}) { |c, h| h[c] = true }.freeze
+    TWO_CHAR_LEX_SET = TWO_CHAR_LEX.each_with_object({}) { |c, h| h[c] = true }.freeze
+    TWO_CHAR_LEX_FIRST_SET = TWO_CHAR_LEX_FIRST.each_with_object({}) { |c, h| h[c] = true }.freeze
+    ONE_OR_TWO_CHAR_LEX_SET = ONE_OR_TWO_CHAR_LEX.each_with_object({}) { |c, h| h[c] = true }.freeze
+
     WHITESPACE = " \t\n\r"
     KEYWORD = %w[true false null].freeze
+    KEYWORD_SET = KEYWORD.each_with_object({}) { |k, h| h[k] = true }.freeze
     FUNCTIONS = %w[length count match search value].freeze
+    FUNCTIONS_SET = FUNCTIONS.each_with_object({}) { |k, h| h[k] = true }.freeze
+
+    # Hoisted from lex_delimited_string. Given a delimiter, the value is the "other" delimiter.
+    OTHER_DELIMITER = { "'" => '"', '"' => "'" }.freeze
+    QUOTE_LEX_SET = { "'" => true, '"' => true }.freeze
 
     # faster to check membership in a string than an array of char (benchmarked ruby 3.1.2)
     ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
@@ -89,18 +103,18 @@ module Janeway
       return if WHITESPACE.include?(c)
 
       token =
-        if ONE_OR_TWO_CHAR_LEX.include?(c)
+        if ONE_OR_TWO_CHAR_LEX_SET.include?(c)
           token_from_one_or_two_char_lex(c)
-        elsif ONE_CHAR_LEX.include?(c)
+        elsif ONE_CHAR_LEX_SET.include?(c)
           token_from_one_char_lex(c)
-        elsif TWO_CHAR_LEX_FIRST.include?(c)
+        elsif TWO_CHAR_LEX_FIRST_SET.include?(c)
           token_from_two_char_lex(c)
-        elsif %w[" '].include?(c)
+        elsif QUOTE_LEX_SET.include?(c)
           lex_delimited_string(c)
         elsif digit?(c)
           lex_number
         elsif name_first_char?(c)
-          lex_member_name_shorthand(ignore_keywords: tokens.last&.type == :dot)
+          lex_member_name_shorthand(ignore_keywords: previous_token_type == :dot)
         end
       raise err("Unknown character: #{c.inspect}") unless token
 
@@ -127,20 +141,20 @@ module Janeway
         raise err("Operator #{lexeme.inspect} must not be followed by whitespace")
       end
 
-      Token.new(OPERATORS.key(lexeme), lexeme, nil, current_location)
+      Token.new(LEXEME_TO_TYPE[lexeme], lexeme, nil, current_location)
     end
 
     # Consumes an operator that could be either 1 or 2 chars in length
     # @return [Token]
     def token_from_one_or_two_char_lex(lexeme)
-      next_two_chars = [lexeme, lookahead].join
-      if TWO_CHAR_LEX.include?(next_two_chars)
+      next_two_chars = lexeme + lookahead
+      if TWO_CHAR_LEX_SET.include?(next_two_chars)
         consume
         if next_two_chars == '..' && WHITESPACE.include?(lookahead)
           raise err("Operator #{next_two_chars.inspect} must not be followed by whitespace")
         end
 
-        Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
+        Token.new(LEXEME_TO_TYPE[next_two_chars], next_two_chars, nil, current_location)
       else
         token_from_one_char_lex(lexeme)
       end
@@ -149,11 +163,11 @@ module Janeway
     # Consumes a 2 char operator
     # @return [Token]
     def token_from_two_char_lex(lexeme)
-      next_two_chars = [lexeme, lookahead].join
-      raise err("Unknown operator \"#{lexeme}\"") unless TWO_CHAR_LEX.include?(next_two_chars)
+      next_two_chars = lexeme + lookahead
+      raise err("Unknown operator \"#{lexeme}\"") unless TWO_CHAR_LEX_SET.include?(next_two_chars)
 
       consume
-      Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
+      Token.new(LEXEME_TO_TYPE[next_two_chars], next_two_chars, nil, current_location)
     end
 
     def consume
@@ -166,17 +180,31 @@ module Janeway
       consume while digit?(lookahead)
     end
 
+    # @return [String] source substring for the lexeme currently being built
+    def current_lexeme
+      source[lexeme_start_p..(next_p - 1)]
+    end
+
+    # Type of the most-recently-emitted token, or nil if none.
+    # Named lookback for the two context-sensitive lexing decisions
+    # (identifier-in-brackets rejection; keyword ignore after `.`).
+    # @return [Symbol, nil]
+    def previous_token_type
+      @tokens.last&.type
+    end
+
     # @param delimiter [String] eg. ' or "
     # @return [Token] string token
     def lex_delimited_string(delimiter)
-      allowed_delimiters = %w[' "]
       # the "other" delimiter char, which is not currently being treated as a delimiter
-      non_delimiter = allowed_delimiters.reject { |char| char == delimiter }.first
+      non_delimiter = OTHER_DELIMITER[delimiter]
 
       literal_chars = []
+      offsets = [] # index within the lexeme of the start of each literal char, see Token#offsets
       while lookahead != delimiter && source_uncompleted?
         # Transform escaped representation to literal chars
         next_char = lookahead
+        offsets << (next_p - lexeme_start_p)
         literal_chars <<
           if next_char == '\\'
             if lookahead(2) == delimiter
@@ -190,7 +218,7 @@ module Janeway
             end
           elsif unescaped?(next_char)
             consume
-          elsif allowed_delimiters.include?(next_char) && next_char != delimiter
+          elsif QUOTE_LEX_SET.include?(next_char) && next_char != delimiter
             consume
           else
             raise err("invalid character #{next_char.inspect}")
@@ -204,9 +232,9 @@ module Janeway
       literal = literal_chars.join
 
       # lexeme value includes delimiters and literal escape characters
-      lexeme = source[lexeme_start_p..(next_p - 1)]
+      lexeme = current_lexeme
 
-      Token.new(:string, lexeme, literal, current_location)
+      Token.new(:string, lexeme, literal, current_location, offsets)
     end
 
     # Read escape char literals, and transform them into the described character
@@ -285,10 +313,11 @@ module Janeway
     # @param low_surrogate_hex [String] string of hex digits, eg. "DE09"
     # @return [String] UTF-8 string containing a single multi-byte unicode character, eg. "😉"
     def convert_surrogate_pair_to_codepoint(high_surrogate_hex, low_surrogate_hex)
-      [high_surrogate_hex, low_surrogate_hex].each do |hex_str|
-        raise ArgumentError, "expect 4 hex digits, got #{hex_string.inspect}" unless hex_str.size == 4
-      end
-
+      # Callers guarantee 4-char inputs (consume_four_hex_digits followed by
+      # high_surrogate?/low_surrogate? which pre-check size). The old defensive
+      # guard here referenced a non-existent `hex_string` local — dead code
+      # that would NameError if ever reached.
+      #
       # Calculate the code point from the surrogate pair values
       # algorithm from https://russellcottrell.com/greek/utilities/SurrogatePairCalculator.htm
       high = high_surrogate_hex.hex
@@ -317,20 +346,19 @@ module Janeway
     #
     # @return [String]
     def consume_four_hex_digits
-      hex_digits = []
+      hex_digits = String.new(capacity: 4)
       4.times do
-        hex_digits << consume
-        case hex_digits.last.ord
-        when 0x30..0x39 then next # '0'..'1'
-        when 0x40..0x46 then next # 'A'..'F'
+        c = consume
+        hex_digits << c
+        case c.ord
+        when 0x30..0x39 then next # '0'..'9'
+        when 0x41..0x46 then next # 'A'..'F'
         when 0x61..0x66 then next # 'a'..'f'
         else
-          raise err("Invalid unicode escape sequence: \\u#{hex_digits.join}")
+          raise err("Invalid unicode escape sequence: \\u#{hex_digits}")
         end
       end
-      raise err("Incomplete unicode escape sequence: \\u#{hex_digits.join}") if hex_digits.size < 4
-
-      hex_digits.join
+      hex_digits
     end
 
     # Consume a numeric string. May be an integer, fractional, or exponent.
@@ -353,13 +381,13 @@ module Janeway
           consume # "+" / "-"
         end
         unless digit?(lookahead)
-          lexeme = source[lexeme_start_p..(next_p - 1)]
+          lexeme = current_lexeme
           raise err("Exponent 'e' must be followed by number: #{lexeme.inspect}")
         end
         consume_digits
       end
 
-      lexeme = source[lexeme_start_p..(next_p - 1)]
+      lexeme = current_lexeme
       if lexeme.start_with?('0') && lexeme.size > 1
         raise err("Number may not start with leading zero: #{lexeme.inspect}")
       end
@@ -375,36 +403,6 @@ module Janeway
 
     # Consume an alphanumeric string.
     # If `ignore_keywords`, the result is always an :identifier token.
-    # Otherwise, keywords and function names will be recognized and tokenized as those types.
-    #
-    # @param ignore_keywords [Boolean]
-    def lex_identifier(ignore_keywords: false)
-      consume while alpha_numeric?(lookahead)
-
-      identifier = source[lexeme_start_p..(next_p - 1)]
-      type =
-        if KEYWORD.include?(identifier) && !ignore_keywords
-          identifier.to_sym
-        elsif FUNCTIONS.include?(identifier) && !ignore_keywords
-          :function
-        else
-          :identifier
-        end
-
-      Token.new(type, identifier, identifier, current_location)
-    end
-
-    # Parse an identifier string which is not within delimiters.
-    # The standard set of unicode code points are allowed.
-    # No character escapes are allowed.
-    # Keywords and function names are ignored in this context.
-    # @return [Token]
-    def lex_unescaped_identifier
-      consume while unescaped?(lookahead)
-      identifier = source[lexeme_start_p..(next_p - 1)]
-      Token.new(:identifier, identifier, identifier, current_location)
-    end
-
     # Return true if string matches the definition of "unescaped" from RFC9535:
     # unescaped     = %x20-21 /        ; see RFC 8259
     #                    ; omit 0x22 "
@@ -428,19 +426,6 @@ module Janeway
       end
     end
 
-    def escapable?(char)
-      case char.ord
-      when 0x62 then true # backspace
-      when 0x66 then true # form feed
-      when 0x6E then true # line feed
-      when 0x72 then true # carriage return
-      when 0x74 then true # horizontal tab
-      when 0x2F then true # slash
-      when 0x5C then true # backslash
-      else false
-      end
-    end
-
     # True if character is suitable as the first character in a name selector
     # using shorthand notation (ie. no bracket notation.)
     #
@@ -481,14 +466,14 @@ module Janeway
     def lex_member_name_shorthand(ignore_keywords: false)
       # Abort if name is preceded by child_start.  Catches non-delimited identifiers in brackets,
       # eg.  $["key"] is allowed, but $[key] is not
-      raise err('Identifier within brackets must be surrounded by quotes') if @tokens.last&.type == :child_start
+      raise err('Identifier within brackets must be surrounded by quotes') if previous_token_type == :child_start
 
       consume while name_char?(lookahead)
-      identifier = source[lexeme_start_p..(next_p - 1)]
+      identifier = current_lexeme
       type =
-        if KEYWORD.include?(identifier) && !ignore_keywords
+        if KEYWORD_SET.include?(identifier) && !ignore_keywords
           identifier.to_sym
-        elsif FUNCTIONS.include?(identifier) && !ignore_keywords
+        elsif FUNCTIONS_SET.include?(identifier) && !ignore_keywords
           :function
         else
           :identifier

data/lib/janeway/location.rb

@@ -1,3 +1,5 @@
 # frozen_string_literal: true
 
-Location = Struct.new(:col, :length)
+module Janeway
+  Location = Struct.new(:col, :length)
+end