janeway-jsonpath 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff463dd64e17811dddaa23fb3fb82c9bc5cb728dd5b288574f39010174c4f705
-  data.tar.gz: 27f5f8d6c39435fb4ec97eb8a96c7e204c2f06a1e3e7dc569b606d4a52b5cf58
+  metadata.gz: 11e0e5480e8e6e35fec72b79fb6effff412dae3349ead88ecb8f257e072d68fb
+  data.tar.gz: 047f434419ccacdbc6b1107146c97dd8305a6aec000d83c7cd3c1eb813e0f78e
 SHA512:
-  metadata.gz: 113d48240302143e72de6249d1975d0be52074374e94abd0d2a340e54867031653d6f912e0ccb3ab25e0b9734a3d33119bea1fb100c4283bce95560f9c02ad69
-  data.tar.gz: cfdca8a82c1c38f7b3b646afcca7c54d6d996564f2f191cb9cd407fa5abcac8c258c4d4534ce7cd437739174298fda0c66b3d41602e4058a47b7f5d94e4b1208
+  metadata.gz: a767d7c2f72d25b61b5ae8ef87a08f3c67c9c7bacc387fe9f0bc9b343c85643eedcd7f7be4a445522e84f1eff23f570f83b71d50dca0a67d3b70264c1552489d
+  data.tar.gz: 35d6038891ab976044be4ca499896bd614f4feae8adf5d894a85309ace9e5d2d5245bea5f4be1594d27f473906b4c58c32b7fe3a774c64942d507d2b3f2ea2b6

data/README.md

@@ -216,7 +216,7 @@ This is a jsonpath query string that uniquely points to the matched value.
 The `#delete` method deletes matched values from the input.
 ```ruby
     # delete any bird whose name starts with "s" or is "eagle"
-    Janeway.enum_for('$.birds[? search(@, "^s") || @=="eagle"]', data).delete
+    Janeway.enum_for('$.birds[? match(@, "s.*") || @=="eagle"]', data).delete
     # input bird list is now ['cormorant']
 
     # delete all dog names

data/bin/janeway

@@ -197,8 +197,10 @@ begin
   options = parse_args(ARGV.dup)
   main(options)
 rescue Janeway::Error => e
-  warn "Error: #{e.message}\nQuery: #{e.query}\n"
-  warn "       #{' ' * e.location.col}^" if e.location # point to the character at the location index
+  # The message shows the query, and carets under it when there is a location. The label is on its own
+  # line so that the carets stay under the header's copy of the query.
+  warn 'Error:'
+  warn e.message
   exit(1)
 rescue Interrupt, Errno::EPIPE
   abort("\n")

data/lib/janeway/ast/string_type.rb

@@ -5,6 +5,10 @@ module Janeway
     # Represents a string literal within a filter expression.
     # May be used within a comparison or a jsonpath function call.
     class StringType < Janeway::AST::Expression
+      # The string token this literal was parsed from, nil otherwise.
+      # @return [Token, nil]
+      attr_accessor :token
+
       def to_s
         if @value.include?("'")
           %("#{@value}"')

data/lib/janeway/error.rb

@@ -20,14 +20,38 @@ module Janeway
     # @param query [String] entire query string
     # @param location [Location] location of error
     def initialize(msg, query = nil, location = nil)
+      prefix = 'Jsonpath query '
+      pointer = query.is_a?(String) && location
+      # The pointer is only right if the header's copy of the query is on one line.
+      # scrub swaps each invalid byte for one replacement character, so the columns stay valid
+      # and a query that is not valid UTF-8 cannot make building the error raise.
+      shown = query
+      if pointer
+        shown = shown.scrub unless shown.valid_encoding?
+        shown = shown.tr("\r\n", '  ')
+      end
       full = +''
-      full << "Jsonpath query #{query}" if query
+      full << "#{prefix}#{shown}" if query
       full << " (col #{location.col})" if location
       full << ' - ' unless full.empty?
       full << msg
+      full << pointer_line(shown, location, prefix.length) if pointer
       super(full)
       @query = query
       @location = location
     end
+
+    private
+
+    # Build the line of carets that marks the problem, aligned under the query in the message header.
+    # @param line [String] the query as shown in the header, on one line
+    # @param location [Location]
+    # @param offset [Integer] width of the text that comes before the query in the header
+    # @return [String] starts with a newline
+    def pointer_line(line, location, offset)
+      col = (location.col || 0).clamp(0, line.length)
+      length = (location.length || 1).clamp(1, [line.length - col, 1].max)
+      "\n#{' ' * (offset + col)}#{'^' * length}"
+    end
   end
 end

data/lib/janeway/functions/match.rb

@@ -19,16 +19,10 @@ module Janeway
     # otherwise.
     #
     #
-    # The regexp dialect is called "I-Regexp" and is defined in RFC9485.
-    #
-    # Fortunately a shortcut is availalble, that RFC contains instructions
-    # for converting an I-Regexp to ruby's regexp format.
-    # @see https://www.rfc-editor.org/rfc/rfc9485.html#name-pcre-re2-and-ruby-regexps
-    #
-    # The instructions are:
-    #   * For any unescaped dots (.) outside character classes (first
-    #     alternative of charClass production), replace the dot with [^\n\r].
-    #   * Enclose the regexp in \A(?: and )\z.
+    # The pattern is validated and translated by Janeway::IRegexp (RFC 9485). A pattern that is not an
+    # I-Regexp gives LogicalFalse (RFC 9535 sections 2.4.6 and 2.4.7). A literal pattern that is over
+    # one of the limits in IRegexp (MAX_LENGTH, MAX_REPEAT, MAX_GROUP_DEPTH) raises Janeway::Error when
+    # the query is parsed. In a pattern ^ and $ are ordinary characters, as RFC 9485 has no anchors.
     #
     # tl;dr: How is this different from the search function?
     # "match" must match the entire string, "search" matches a substring.
@@ -52,6 +46,8 @@ module Janeway
       parameters << parse_function_parameter
       raise Error, 'Too many parameters for match() function call' unless current.type == :group_end
 
+      validate_literal_pattern(parameters.last) # after the syntax checks, so they are reported first
+
       # Body (including literal-regex fast path) is supplied by
       # Functions::REGISTRY at FunctionInterpreter construction time.
       AST::Function.new('match', parameters)

data/lib/janeway/functions/search.rb

@@ -25,6 +25,11 @@ module Janeway
     # the result is LogicalTrue if at least one such substring exists and is
     # LogicalFalse otherwise.
     #
+    # The pattern is validated and translated by Janeway::IRegexp (RFC 9485). A pattern that is not an
+    # I-Regexp gives LogicalFalse (RFC 9535 sections 2.4.6 and 2.4.7). A literal pattern that is over
+    # one of the limits in IRegexp (MAX_LENGTH, MAX_REPEAT, MAX_GROUP_DEPTH) raises Janeway::Error when
+    # the query is parsed. In a pattern ^ and $ are ordinary characters, as RFC 9485 has no anchors.
+    #
     # tl;dr: How is this different from the match function?
     # "match" must match the entire string, "search" matches a substring.
     def parse_function_search
@@ -45,7 +50,9 @@ module Janeway
       # Otherwise it is an expression that takes the regexp from the input document,
       # and the iregexp will not be available until interpretation.
       parameters << parse_function_parameter
-      raise Error, 'Too many parameters for match() function call' unless current.type == :group_end
+      raise Error, 'Too many parameters for search() function call' unless current.type == :group_end
+
+      validate_literal_pattern(parameters.last) # after the syntax checks, so they are reported first
 
       # Body (including literal-regex fast path) is supplied by
       # Functions::REGISTRY at FunctionInterpreter construction time.

data/lib/janeway/functions.rb

@@ -2,42 +2,29 @@
 
 require_relative 'ast/function'
 require_relative 'ast/string_type'
+require_relative 'iregexp'
 
 module Janeway
-  # Mixin to provide JSONPath function handlers for Parser, plus the shared
-  # I-Regexp translation helper and the built-in function registry.
+  # Mixin to provide JSONPath function handlers for Parser, plus the built-in function registry.
   module Functions
-    # Convert IRegexp format to ruby regexp equivalent, following the instructions in rfc9485.
-    # @see https://www.rfc-editor.org/rfc/rfc9485.html#name-pcre-re2-and-ruby-regexps
-    # @param iregex [String]
-    # @param anchor [Boolean] add anchors to match the string start and end
-    # @return [Regexp]
-    def translate_iregex_to_ruby_regex(iregex, anchor: true)
-      # * For any unescaped dots (.) outside character classes (first
-      #   alternative of charClass production), replace the dot with [^\n\r].
-      chars = iregex.chars
-      in_char_class = false
-      indexes = []
-      chars.each_with_index do |char, i|
-        case char
-        when '[' then in_char_class = true
-        when ']'
-          in_char_class = false unless chars[i - 1] == '\\' # escaped ] does not close char class
-        when '.'
-          next if in_char_class || chars[i - 1] == '\\' # escaped dot
-
-          indexes << i # replace this dot
-        end
-      end
-      indexes.reverse_each do |i|
-        chars[i] = '[^\n\r]'
-      end
+    # Reject a literal pattern that exceeds an I-Regexp limit, at the point where the query is parsed.
+    #
+    # A literal that is not an I-Regexp is not an error: match() and search() return LogicalFalse
+    # for it, see build_regex_body. A literal over a limit is a valid I-Regexp that this library
+    # refuses to run, so the user is told, with the position in the query.
+    # @see https://www.rfc-editor.org/rfc/rfc9485.html#name-security-considerations
+    #   RFC 9485 section 8: an implementation may limit range quantifiers in range and composability
+    # @param parameter [AST::Expression] the second parameter of match() or search(), just parsed
+    # @raise [Janeway::Error]
+    def validate_literal_pattern(parameter)
+      return unless parameter.is_a?(AST::StringType)
 
-      # * Enclose the regexp in \A(?: and )\z.
-      regex_str = anchor ? format('\A(?:%s)\z', chars.join) : chars.join
-      Regexp.new(regex_str)
+      IRegexp.compile(parameter.value)
+    rescue IRegexp::Invalid
+      nil # RFC 9535 section 2.4.6: the function returns LogicalFalse, see build_regex_body
+    rescue IRegexp::LimitExceeded => e
+      raise Error.new("I-Regexp #{e.message}", @jsonpath, parameter.token&.location_of(e.offset, e.length))
     end
-    module_function :translate_iregex_to_ruby_regex
 
     # All jsonpath function parameters are one of these accepted types.
     # Parse the function parameter and return the result.
@@ -58,32 +45,72 @@ module Janeway
       result
     end
 
-    # Build a match/search body. When the pattern is a StringType literal we
-    # compile the regex once at parse time (see perf pass, commit fd8b92a).
-    # Otherwise fall back to per-call compilation.
+    # Largest number of distinct document-supplied patterns that one function call remembers.
+    DYNAMIC_PATTERN_CACHE_SIZE = 100
+
+    # Build a match/search body. When the pattern is a StringType literal, the parser has already
+    # validated it (Functions#validate_literal_pattern, which discards the result); the interpreter
+    # compiles it once more, with the anchoring this function needs, when it is built.
+    # Otherwise the pattern comes from the input document and is compiled when it is first used.
+    #
+    # A pattern that is not an I-Regexp, or that exceeds a limit, gives LogicalFalse.
+    # @see https://www.rfc-editor.org/rfc/rfc9535.html#name-match-function-extension
+    #   RFC 9535 section 2.4.6: "If the first argument is not a string or the second argument is
+    #   not a string conforming to [RFC9485], the result is LogicalFalse." Section 2.4.7 says the
+    #   same for search().
+    # A literal that exceeds a limit has already been rejected by the parser, see
+    # Functions#validate_literal_pattern.
     def self.build_regex_body(parameters, anchor:)
-      literal_regexp =
-        if parameters[1].is_a?(AST::StringType)
-          begin
-            translate_iregex_to_ruby_regex(parameters[1].value, anchor: anchor)
-          rescue RegexpError
-            nil # fall back to per-call compilation, which will raise at eval time
-          end
-        end
+      literal = parameters[1].is_a?(AST::StringType) ? parameters[1].value : nil
+      # The cache holds compiled regexps keyed by pattern text, so reusing it for other subjects is
+      # correct. It lives as long as the interpreter tree, which Query#interpreter_tree_for caches
+      # on an unfrozen Query.
+      cache = {} # pattern string => Regexp, or nil when the pattern is unusable
+      cache[literal] = compile_or_nil(literal, anchor) if literal # a literal is compiled once, now
+      lambda do |str, pattern|
+        pattern = literal if literal
+        next false unless str.is_a?(String) && pattern.is_a?(String)
+        # A pattern in the wrong encoding is false and never reaches the cache: Hash lookup compares
+        # strings by content, so a binary 'a' would otherwise share the entry of a UTF-8 'a'.
+        next false unless IRegexp.valid_encoding?(pattern)
+        # RFC 9485 section 8 lets an implementation limit patterns; the limit is this library's own
+        # choice. Checked before parsing so that a huge pattern costs almost nothing.
+        next false if pattern.length > IRegexp::MAX_LENGTH
 
-      if literal_regexp
-        ->(str, _pattern) { str.is_a?(String) && literal_regexp.match?(str) }
-      else
-        lambda do |str, pattern|
-          if str.is_a?(String) && pattern.is_a?(String)
-            translate_iregex_to_ruby_regex(pattern, anchor: anchor).match?(str)
-          else
-            false
-          end
+        unless cache.key?(pattern)
+          cache.shift if cache.size >= DYNAMIC_PATTERN_CACHE_SIZE
+          cache[pattern] = compile_or_nil(pattern, anchor)
         end
+
+        !cache[pattern].nil? && match_safely(cache[pattern], str)
       end
     end
 
+    # RFC 9535 section 2.4.6: a pattern that does not conform to RFC 9485 gives LogicalFalse, so it is nil here.
+    # A pattern over a limit (RFC 9485 section 8) is treated the same way.
+    # @return [Regexp, nil] nil when the pattern is not usable, see build_regex_body
+    def self.compile_or_nil(pattern, anchor)
+      IRegexp.compile(pattern, anchor: anchor)
+    rescue IRegexp::Invalid, IRegexp::LimitExceeded
+      nil
+    end
+
+    # Match, and treat a match that Ruby cannot or will not complete as no match.
+    #
+    # Deliberate deviation from RFC 9535 sections 2.4.6 and 2.4.7, which say the result is LogicalTrue or
+    # LogicalFalse and say nothing about a match that cannot complete: a timeout, or a subject that is
+    # not valid UTF-8, gives false. RFC 9485 section 8 (security considerations) is why a timeout exists.
+    # @param regexp [Regexp]
+    # @param str [String]
+    # @return [Boolean]
+    def self.match_safely(regexp, str)
+      regexp.match?(str)
+    rescue IRegexp::TIMEOUT_ERROR, ArgumentError, Encoding::CompatibilityError
+      # TIMEOUT_ERROR: the match took longer than IRegexp::MATCH_TIMEOUT.
+      # ArgumentError, Encoding::CompatibilityError: str is not valid UTF-8.
+      false
+    end
+
     # Metadata + body builder for every built-in JSONPath function.
     #
     #   arity          — number of parameters

data/lib/janeway/iregexp.rb

@@ -0,0 +1,402 @@
+# frozen_string_literal: true
+
+require_relative 'error'
+
+module Janeway
+  # Validates an I-Regexp and translates it to an equivalent Ruby Regexp.
+  #
+  # The pattern is parsed with the grammar from RFC 9485 section 3, and the Ruby source is emitted
+  # while parsing, so a pattern that is not an I-Regexp never reaches Regexp.new, and characters
+  # that Ruby treats as syntax but I-Regexp treats as ordinary characters are escaped.
+  #
+  # The limits below are this library's own choices. RFC 9485 section 8 allows an implementation to
+  # limit range quantifiers in composability and range, but gives no values.
+  # @see https://www.rfc-editor.org/rfc/rfc9485.html
+  module IRegexp
+    # Longest pattern accepted, in characters.
+    MAX_LENGTH = 10_000
+
+    # Largest number accepted in a range quantifier such as a{2,4}.
+    MAX_REPEAT = 1_000
+
+    # Deepest nesting of groups accepted. This also bounds the recursion depth of the parser.
+    MAX_GROUP_DEPTH = 100
+
+    # Seconds a single match may take, where the Ruby version supports a per-regexp timeout (3.2+).
+    MATCH_TIMEOUT = 1
+
+    # Regexp.new accepts the timeout: keyword from Ruby 3.2, which is also when Regexp.timeout appeared.
+    # Probe with respond_to? and not by trying the keyword: on an older Ruby a trailing hash would be
+    # read as the positional options argument, and a pattern would silently become case-insensitive.
+    TIMEOUT_SUPPORTED = Regexp.respond_to?(:timeout)
+
+    # Error raised when a match exceeds MATCH_TIMEOUT. Regexp::TimeoutError only exists on Ruby 3.2+,
+    # elsewhere this is a class that is never raised for that reason.
+    TIMEOUT_ERROR = defined?(Regexp::TimeoutError) ? Regexp::TimeoutError : Class.new(StandardError)
+
+    # Unicode general category names, from the IsCategory production of RFC 9485 section 3.
+    CATEGORY = /\A(?:L[lmotu]?|M[cen]?|N[dlo]?|P[c-fios]?|Z[lps]?|S[ckmo]?|C[cfno]?)\z/
+
+    # Base class for the errors IRegexp.compile raises. They describe a problem in a pattern, not in a
+    # JSONPath query, so they carry the index of the problem in the pattern and have no query or location.
+    class Error < Janeway::Error
+      # @return [Integer, nil] index of the problem in the pattern, nil when there is no single place
+      attr_reader :offset
+
+      # @param msg [String]
+      # @param offset [Integer, nil]
+      def initialize(msg, offset: nil)
+        super(msg)
+        @offset = offset
+      end
+    end
+
+    # Raised for a pattern that is not an I-Regexp.
+    class Invalid < Error; end
+
+    # Raised for an I-Regexp that exceeds one of the limits above.
+    class LimitExceeded < Error
+      # @return [Integer] number of pattern characters the problem covers
+      attr_reader :length
+
+      # @param msg [String]
+      # @param offset [Integer, nil]
+      # @param length [Integer]
+      def initialize(msg, offset: nil, length: 1)
+        super(msg, offset: offset)
+        @length = length
+      end
+    end
+
+    # Can this string be an I-Regexp at all, as far as its encoding goes?
+    #
+    # RFC 9485 section 1: a pattern is a sequence of Unicode scalar values. Accept UTF-8, and US-ASCII
+    # which is a subset of it. Any other encoding is refused and not transcoded, and that includes a
+    # binary (ASCII-8BIT) string even when its bytes are all ASCII.
+    # @param string [String]
+    # @return [Boolean]
+    def self.valid_encoding?(string)
+      [Encoding::UTF_8, Encoding::US_ASCII].include?(string.encoding) && string.valid_encoding?
+    end
+
+    # Validate an I-Regexp and translate it to a Ruby Regexp.
+    #
+    # @param iregex [String]
+    # @param anchor [Boolean] add anchors to match the string start and end
+    # @return [Regexp]
+    # @raise [Invalid] the pattern is not an I-Regexp
+    # @raise [LimitExceeded] the pattern is an I-Regexp, but is over one of the limits
+    def self.compile(iregex, anchor: true)
+      body = Parser.new(iregex).parse
+      # RFC 9485 section 5.4: enclose the regexp in \A(?: and )\z.
+      body = "\\A(?:#{body})\\z" if anchor
+      # RFC 9485 section 3: the pattern is valid I-Regexp, so a Ruby warning about it (such as a redundant
+      # nested repeat operator in (a*)*) is not the caller's business. A pattern from the input document
+      # must not be able to write to stderr. $VERBOSE is process-wide, so warnings from other threads are
+      # also suppressed during this one compile.
+      old_verbose = $VERBOSE
+      begin
+        $VERBOSE = nil
+        TIMEOUT_SUPPORTED ? Regexp.new(body, timeout: MATCH_TIMEOUT) : Regexp.new(body)
+      rescue RegexpError => e
+        # Deliberate deviation from RFC 9485 section 3: Ruby refuses some patterns that are valid
+        # I-Regexp, such as a{3,2}, and these are treated as not an I-Regexp.
+        raise Invalid, e.message
+      ensure
+        $VERBOSE = old_verbose
+      end
+    end
+
+    # Recursive-descent parser for the ABNF of RFC 9485 section 3, which builds the Ruby regexp source.
+    # Each method is named for the production it implements.
+    class Parser
+      # Characters allowed after a backslash. RFC 9485 section 3, SingleCharEsc.
+      SINGLE_CHAR_ESC = '()*+-.?[]^nrt{|}\\'
+
+      # catEsc and complEsc start with one of these after the backslash, RFC 9485 section 3
+      CATEGORY_ESCAPES = %w[p P].freeze
+
+      # @param iregex [String]
+      def initialize(iregex)
+        @original = iregex
+        @src = iregex.valid_encoding? ? iregex.chars : [] # array: indexing a multibyte String is O(n)
+        @i = 0
+        @out = +''
+        @depth = 0
+        @has_range = false # does the group being parsed contain a range quantifier?
+        @limit = nil # first limit violation, raised only if the pattern is otherwise valid
+      end
+
+      # @return [String] Ruby regexp source, without anchors
+      # @raise [Invalid, LimitExceeded]
+      def parse
+        raise Invalid, 'pattern is not valid UTF-8' unless IRegexp.valid_encoding?(@original) # RFC 9485 section 1
+
+        i_regexp
+        raise Invalid.new("unexpected #{@src[@i].inspect}", offset: @i) if @i < @src.length
+
+        limit("pattern is longer than #{MAX_LENGTH} characters", nil) if @src.length > MAX_LENGTH
+        raise @limit if @limit
+
+        @out
+      end
+
+      private
+
+      def peek
+        @src[@i]
+      end
+
+      # Grammar errors take precedence over limit errors, so a limit violation is remembered and
+      # only raised once the whole pattern is known to be an I-Regexp. The one exception is the group
+      # nesting limit, which is raised as soon as it is hit (see #group): reading on past it is what
+      # would overflow the stack.
+      def limit(msg, offset, length = 1)
+        @limit ||= LimitExceeded.new(msg, offset: offset, length: length)
+      end
+
+      # i-regexp = branch *( "|" branch )
+      # RFC 9485 section 3, the top production.
+      def i_regexp
+        branch
+        while peek == '|'
+          @i += 1
+          @out << '|'
+          branch
+        end
+      end
+
+      # branch = *piece
+      def branch
+        piece while !peek.nil? && peek != '|' && peek != ')'
+      end
+
+      # piece = atom [ quantifier ]
+      def piece
+        contains_range = atom
+        quantifier(contains_range)
+      end
+
+      # atom = NormalChar / charClass / ( "(" i-regexp ")" )
+      # @return [Boolean] true if the atom is a group that contains a range quantifier
+      def atom
+        case peek
+        when '(' then return group
+        when '[' then char_class
+        when '.'
+          # RFC 9485 section 5.4: replace an unescaped dot outside a character class with [^\n\r]
+          @i += 1
+          @out << '[^\n\r]'
+        when '\\' then escape
+        else normal_char(in_class: false)
+        end
+        false
+      end
+
+      # atom = ... / ( "(" i-regexp ")" ), RFC 9485 section 3.
+      # The nesting limit is this library's own choice, which RFC 9485 section 8 permits: it bounds recursion.
+      # Unlike the other limits it is raised at once and not remembered (see #limit), because parsing on
+      # past it is what would overflow the stack. So a too-deeply nested pattern that is also invalid is
+      # reported as LimitExceeded, not Invalid.
+      # @return [Boolean] true if the group contains a range quantifier
+      def group
+        open = @i
+        @i += 1
+        @depth += 1
+        if @depth > MAX_GROUP_DEPTH
+          raise LimitExceeded.new("groups are nested deeper than #{MAX_GROUP_DEPTH}", offset: open)
+        end
+
+        @out << '(?:'
+        outer_has_range = @has_range
+        @has_range = false
+        i_regexp
+        raise Invalid.new('unterminated group', offset: @i) unless peek == ')'
+
+        @i += 1
+        @depth -= 1
+        @out << ')'
+        inner_has_range = @has_range
+        @has_range = outer_has_range || inner_has_range
+        inner_has_range
+      end
+
+      # quantifier = ( "*" / "+" / "?" ) / range-quantifier
+      def quantifier(contains_range)
+        case peek
+        when '*', '+', '?'
+          @out << peek
+          @i += 1
+        when '{' then range_quantifier(contains_range)
+        end
+      end
+
+      # range-quantifier = "{" QuantExact [ "," [ QuantExact ] ] "}"
+      # QuantExact = 1*%x30-39
+      #
+      # RFC 9485 section 8 allows limiting range quantifiers in range and in composability: here a
+      # number above MAX_REPEAT, and a range quantifier applied to a group that contains one.
+      def range_quantifier(contains_range)
+        start = @i
+        counts, finish = scan_range_quantifier(start)
+        raise Invalid.new('malformed range quantifier', offset: start) unless counts
+
+        counts.each do |digits, offset|
+          next if digits.empty? || digits.to_i <= MAX_REPEAT
+
+          limit("repeat count #{digits} exceeds #{MAX_REPEAT}", offset, digits.length)
+        end
+        limit('range quantifier applied to a group that contains a range quantifier', start) if contains_range
+        @has_range = true
+        @out << @src[start...finish].join
+        @i = finish
+      end
+
+      # Match \{(\d+)(?:,(\d*))?\} at start by scanning the character array, which a Regexp cannot
+      # do in O(1) per call on a multibyte string. The digit runs can be long, so no fixed-size slice.
+      # @return [Array, nil] [[[digits, offset], ...], index after the closing brace], nil if malformed
+      def scan_range_quantifier(start)
+        return nil unless @src[start] == '{'
+
+        first_start = start + 1
+        first_end = digits_end(first_start)
+        return nil if first_end == first_start # QuantExact needs at least one digit
+
+        counts = [[@src[first_start...first_end].join, first_start]]
+        pos = first_end
+        if @src[pos] == ','
+          second_start = pos + 1
+          pos = digits_end(second_start)
+          counts << [@src[second_start...pos].join, second_start]
+        end
+        @src[pos] == '}' ? [counts, pos + 1] : nil
+      end
+
+      # @return [Integer] index of the first character at or after pos that is not an ASCII digit
+      def digits_end(pos)
+        pos += 1 while @src[pos]&.match?(/\A[0-9]\z/)
+        pos
+      end
+
+      # charClassExpr = "[" [ "^" ] ( "-" / CCE1 ) *CCE1 [ "-" ] "]"
+      def char_class
+        open = @i
+        @i += 1
+        @out << '['
+        negated = peek == '^'
+        if negated
+          @i += 1
+          @out << '^'
+        end
+        # RFC 9485 section 3: charClassExpr is not allowed to match [^]
+        raise Invalid.new('empty negated character class [^]', offset: open) if negated && peek == ']'
+        # RFC 9485 section 3: charClassExpr has at least one item, so [] is not allowed either
+        raise Invalid.new('empty character class []', offset: open) if peek == ']'
+
+        first = true
+        until peek == ']'
+          raise Invalid.new('unterminated character class', offset: @i) if peek.nil?
+
+          class_item(first)
+          first = false
+        end
+        @i += 1
+        @out << ']'
+      end
+
+      # CCE1 = ( CCchar [ "-" CCchar ] ) / charClassEsc, RFC 9485 section 3
+      def class_item(first)
+        if peek == '-'
+          # a "-" is only a plain character at the start or the end of the class
+          raise Invalid.new("misplaced '-' in character class", offset: @i) unless first || @src[@i + 1] == ']'
+
+          @i += 1
+          @out << '\-'
+        elsif peek == '\\' && CATEGORY_ESCAPES.include?(@src[@i + 1])
+          # charClassEsc is not a CCchar, so it takes no range: a following "-" is rejected by the next item
+          escape
+        else
+          cc_char
+          class_range_end if peek == '-' && @src[@i + 1] != ']'
+        end
+      end
+
+      # "-" CCchar, the end of a range. RFC 9485 section 3, CCE1.
+      def class_range_end
+        @i += 1
+        @out << '-'
+        raise Invalid.new('unterminated range in character class', offset: @i) if peek.nil? || peek == ']'
+
+        cc_char
+      end
+
+      # CCchar = ( %x00-2C / %x2E-5A / %x5E-D7FF / %xE000-10FFFF ) / SingleCharEsc
+      # RFC 9485 section 3: a SingleCharEsc is a CCchar, so it can start or end a range.
+      def cc_char
+        if peek == '\\' then escape(single_char_only: true)
+        else normal_char(in_class: true)
+        end
+      end
+
+      # NormalChar (outside a class) and CCchar (inside a class). RFC 9485 section 3.
+      # In I-Regexp "^" and "$" are ordinary characters, not anchors: the RFC 9485 author confirmed
+      # this, see https://github.com/jsonpath-standard/jsonpath-compliance-test-suite (open issue).
+      def normal_char(in_class:)
+        char = peek
+        raise Invalid.new('unexpected end of pattern', offset: @i) if char.nil?
+        unless allowed_char?(char.ord, in_class)
+          raise Invalid.new("character #{char.inspect} must be escaped", offset: @i)
+        end
+
+        @i += 1
+        # Regexp.escape does not escape "&", but Ruby reads "&&" in a class as set intersection
+        @out << (char == '&' ? '\&' : Regexp.escape(char))
+      end
+
+      # @param code [Integer] code point
+      def allowed_char?(code, in_class)
+        return false if (0xD800..0xDFFF).cover?(code) # surrogates
+
+        if in_class
+          # CCchar: %x00-2C / %x2E-5A / %x5E-D7FF / %xE000-10FFFF
+          code <= 0x2C || (0x2E..0x5A).cover?(code) || code >= 0x5E
+        else
+          # NormalChar: %x00-27 / "," / "-" / %x2F-3E / %x40-5A / %x5E-7A / %x7E-D7FF / %xE000-10FFFF
+          code <= 0x27 || [0x2C, 0x2D].include?(code) || (0x2F..0x3E).cover?(code) ||
+            (0x40..0x5A).cover?(code) || (0x5E..0x7A).cover?(code) || code >= 0x7E
+        end
+      end
+
+      # SingleCharEsc / catEsc / complEsc, RFC 9485 section 3
+      # @param single_char_only [Boolean] refuse catEsc and complEsc, where a CCchar is required
+      def escape(single_char_only: false)
+        start = @i
+        nxt = @src[@i + 1]
+        raise Invalid.new('trailing backslash', offset: start) if nxt.nil?
+
+        if CATEGORY_ESCAPES.include?(nxt)
+          if single_char_only
+            raise Invalid.new("\\#{nxt} is not allowed here, only a single-character escape", offset: start)
+          end
+
+          category_escape(start)
+        elsif SINGLE_CHAR_ESC.include?(nxt)
+          @i += 2
+          @out << { 'n' => '\n', 'r' => '\r', 't' => '\t' }.fetch(nxt) { "\\#{nxt}" }
+        else
+          raise Invalid.new("invalid escape \\#{nxt}", offset: start)
+        end
+      end
+
+      # catEsc = %s"\p{" charProp "}" ; complEsc = %s"\P{" charProp "}", RFC 9485 section 3
+      def category_escape(start)
+        # at most \p{Xx}, 6 characters
+        m = /\A\\[pP]\{([A-Za-z]{1,2})\}/.match(@src[start, 6].join)
+        raise Invalid.new('malformed category escape', offset: start) unless m && CATEGORY.match?(m[1])
+
+        @i = start + m[0].length
+        @out << (@src[start + 1] == 'P' ? "\\p{^#{m[1]}}" : m[0])
+      end
+    end
+  end
+end

data/lib/janeway/lexer.rb

@@ -200,9 +200,11 @@ module Janeway
       non_delimiter = OTHER_DELIMITER[delimiter]
 
       literal_chars = []
+      offsets = [] # index within the lexeme of the start of each literal char, see Token#offsets
       while lookahead != delimiter && source_uncompleted?
         # Transform escaped representation to literal chars
         next_char = lookahead
+        offsets << (next_p - lexeme_start_p)
         literal_chars <<
           if next_char == '\\'
             if lookahead(2) == delimiter
@@ -232,7 +234,7 @@ module Janeway
       # lexeme value includes delimiters and literal escape characters
       lexeme = current_lexeme
 
-      Token.new(:string, lexeme, literal, current_location)
+      Token.new(:string, lexeme, literal, current_location, offsets)
     end
 
     # Read escape char literals, and transform them into the described character

data/lib/janeway/parser.rb

@@ -184,7 +184,7 @@ module Janeway
     end
 
     def parse_string
-      AST::StringType.new(current.literal)
+      AST::StringType.new(current.literal).tap { |node| node.token = current }
     end
 
     def parse_number

data/lib/janeway/token.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require_relative 'location'
 
 module Janeway
   # Tokens are produced by the lexer, they represent jsonpath query elements in a low-level, non-hierarchical way.
@@ -9,13 +10,50 @@ module Janeway
 
     attr_reader :type, :lexeme, :literal, :location
 
+    # For a string token, the index within the lexeme of each character of the decoded literal,
+    # so lexeme[offsets[i]] is where literal[i] came from. It differs from i when the string
+    # contains escape sequences or the lexeme starts with a quote. Nil for other token types.
+    # @return [Array<Integer>, nil]
+    attr_reader :offsets
+
     def_delegators :@location, :line, :col, :length
 
-    def initialize(type, lexeme, literal, location)
+    def initialize(type, lexeme, literal, location, offsets = nil)
       @type = type
       @lexeme = lexeme
       @literal = literal
       @location = location
+      @offsets = offsets
+    end
+
+    # Find where a character of a string token's decoded value came from in the query.
+    #
+    # Relies on two invariants: offsets[i] is the index within the lexeme of the start of literal[i],
+    # and location.col is the column of the lexeme in the query.
+    # @param index [Integer] index of a character in the decoded value
+    # @return [Integer, nil] column in the query of that character (or of the backslash that starts its
+    #   escape); nil when the token has no offsets, or no character at that index
+    def column_of(index)
+      return nil if @offsets.nil? || !index.between?(0, @offsets.length - 1)
+
+      offset = @offsets[index]
+      offset && (@location.col + offset)
+    end
+
+    # Find where a run of characters of a string token's decoded value came from in the query.
+    # @param offset [Integer, nil] index in the decoded value of the first character
+    # @param length [Integer, nil] number of decoded characters, treated as 1 if nil or not positive
+    # @return [Location] from the first character to the start of the last one; the location of the whole
+    #   token when offset is nil or the range cannot be found in the token
+    def location_of(offset, length = 1)
+      return @location unless offset
+
+      length = 1 unless length&.positive?
+      first = column_of(offset)
+      last = column_of(offset + length - 1)
+      return @location unless first && last
+
+      Location.new(first, last - first + 1)
     end
 
     def to_s

data/lib/janeway/version.rb

@@ -2,5 +2,5 @@
 
 module Janeway
   # Version for janeway-jsonpath gem
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: janeway-jsonpath
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Fraser Hanson
@@ -84,6 +84,7 @@ files:
 - lib/janeway/interpreters/wildcard_selector_delete_if.rb
 - lib/janeway/interpreters/wildcard_selector_interpreter.rb
 - lib/janeway/interpreters/yielder.rb
+- lib/janeway/iregexp.rb
 - lib/janeway/lexer.rb
 - lib/janeway/location.rb
 - lib/janeway/normalized_path.rb
@@ -109,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.8
+rubygems_version: 3.6.9
 specification_version: 4
 summary: jsonpath parser which implements the finalized IETF standard of Goessner
   JSONPath