janeway-jsonpath 0.1.0 → 0.2.0

data/lib/janeway/parser.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 require_relative 'functions'
+require_relative 'lexer'
 
 module Janeway
   # Transform a list of tokens into an Abstract Syntax Tree
   class Parser
     class Error < Janeway::Error; end
 
-    attr_accessor :tokens, :ast
+    attr_accessor :tokens
 
     include Functions
 
@@ -33,27 +34,36 @@ module Janeway
     # @param query [String] jsonpath query to be lexed and parsed
     #
     # @return [AST]
-    def self.parse(query)
-      raise ArgumentError, "expect string, got #{query.inspect}" unless query.is_a?(String)
+    def self.parse(jsonpath)
+      raise ArgumentError, "expect jsonpath string, got #{jsonpath.inspect}" unless jsonpath.is_a?(String)
 
-      tokens = Janeway::Lexer.lex(query)
-      new(tokens).parse
+      tokens = Janeway::Lexer.lex(jsonpath)
+      new(tokens, jsonpath).parse
     end
 
-    def initialize(tokens)
+    # @param token [Array<Token>]
+    # @param jsonpath [String] original jsonpath query string
+    def initialize(tokens, jsonpath)
       @tokens = tokens
-      @ast = AST::Query.new
       @next_p = 0
+      @jsonpath = jsonpath
     end
 
+    # Parse the token list and create an Abstract Syntax Tree
+    # @return [AST::Query]
     def parse
-
       consume
-      @ast.root = parse_expr_recursively
+      raise err('JsonPath queries must start with root identifier "$"') unless current.type == :root
+
+      root_node = parse_expr_recursively
       consume
-      raise "unparsed tokens" unless current.type == :eof
+      unless current.type == :eof
+        remaining = tokens[@next_p..].map(&:lexeme).join
+        raise err("Unrecognized expressions after query: #{remaining}")
+      end
 
-      @ast
+      # Freeze so this can be used in ractors
+      AST::Query.new(root_node, @jsonpath).freeze
     end
 
     private
@@ -123,10 +133,12 @@ module Janeway
 
     def unexpected_token_error(expected = nil)
       if expected
-        raise Error, "Unexpected token #{current.lexeme.inspect} (expected #{expected.inspect}) (next is #{next_token.inspect})"
-      else
-        raise Error, "Unexpected token #{current.lexeme.inspect} (next is #{next_token.inspect})"
+        raise err(
+          "Unexpected token #{current.lexeme.inspect} " \
+          "(expected #{expected.inspect}, got #{next_token.lexeme.inspect} )"
+        )
       end
+      raise err("Unexpected token #{current.lexeme.inspect} (next is #{next_token.inspect})")
     end
 
     def check_syntax_compliance(ast_node)
@@ -156,7 +168,7 @@ module Janeway
       elsif current.type == :null # null
         :parse_null
       else
-        raise "Don't know how to parse #{current}"
+        raise err("Don't know how to parse #{current}")
       end
     end
 
@@ -184,15 +196,16 @@ module Janeway
     # Consume minus operator and apply it to the (expected) number token following it.
     # Don't consume the number token.
     def parse_minus_operator
-      raise "Expect token '-', got #{current.lexeme.inspect}" unless current.type == :minus
+      raise err("Expect token '-', got #{current.lexeme.inspect}") unless current.type == :minus
 
-      # RFC: negative 0 is allowed within a filter selector comparison, but is NOT allowed within an index selector or array slice selector.
+      # RFC: negative 0 is allowed within a filter selector comparison, but is NOT allowed
+      #      within an index selector or array slice selector.
       # Detect that condition here
-      if next_token.type == :number && next_token.literal == 0
-        if [previous.type, lookahead(2).type].any? { _1 == :array_slice_separator}
-          raise Error, 'Negative zero is not allowed in an array slice selector'
+      if next_token.type == :number && next_token.literal.zero?
+        if [previous.type, lookahead(2).type].any? { _1 == :array_slice_separator }
+          raise err('Negative zero is not allowed in an array slice selector')
         elsif %i[union child_start].include?(previous.type)
-          raise Error, 'Negative zero is not allowed in an index selector'
+          raise err('Negative zero is not allowed in an index selector')
         end
       end
 
@@ -200,6 +213,10 @@ module Janeway
       # Parse number and apply - sign to its literal value
       consume
       parse_number
+      unless current.literal.is_a?(Numeric)
+        raise err("Minus operator \"-\" must be followed by number, got #{current.lexeme.inspect}")
+      end
+
       current.literal *= -1
       current
     end
@@ -209,6 +226,7 @@ module Janeway
       AST::Null.new
     end
 
+    # @return [AST::Boolean]
     def parse_boolean
       AST::Boolean.new(current.literal == 'true')
     end
@@ -236,33 +254,24 @@ module Janeway
     def parse_descendant_segment
       consume # '..'
 
-      # DescendantSegment must be followed by a selector S which it applies to all descendants.
-      #
-      # Normally the parser makes the selector after S be a child of S.
-      # However that is not the desired behavior for DescendantSelector.
-      # Consider '$.a..b[1]'. The first element must be taken from the set of all 'b' keys.
-      # If the ChildSegment was a child of the `b` NameSelector, then it would be taking
-      # index 1 from every 'b' found rather than from the set of all 'b's.
-      #
-      # To get around this, the Parser must embed a Selector object that
-      # doesn't include the following selector as a child. Then the following
-      # selector must be made a child of the DescendantSegment.
       selector =
         case next_token.type
-        when :wildcard then parse_wildcard_selector(and_child: false)
-        when :child_start then parse_child_segment(and_child: false)
-        when :string, :identifier then parse_name_selector(and_child: false)
+        when :wildcard then parse_wildcard_selector
+        when :child_start then parse_child_segment
+        when :string, :identifier then parse_name_selector
         else
-          raise "Invalid query: descendant segment must have selector, got ..#{next_token.type}"
+          msg = 'Descendant segment ".." must be followed by selector'
+          msg += ", got ..#{next_token.type}" unless next_token.type == :eof
+          raise err(msg)
         end
 
-      AST::DescendantSegment.new(selector).tap do |ds|
+      AST::DescendantSegment.new.tap do |ds|
         # If there is another selector after this one, make it a child
-        ds.child = parse_next_selector
+        ds.next = selector
       end
     end
 
-    # Dot notation reprsents a name selector, and is an alternative to bracket notation.
+    # Dot notation represents a name selector, and is an alternative to bracket notation.
     # These examples are equivalent:
     #   $.store
     #   $[store]
@@ -272,16 +281,26 @@ module Janeway
     #   * member name (with only certain chars useable. For example, names containing dots are not allowed here.)
     def parse_dot_notation
       consume # "."
-      raise "#parse_dot_notation expects to consume :dot, got #{current}" unless current.type == :dot
-
+      unless current.type == :dot
+        # Parse error, determine most useful error message
+        msg =
+          if current.type == :number
+            "Decimal point must be preceded by number, got \".#{current.lexeme}\""
+          else
+            'Dot "." begins a name selector, and must be followed by an ' \
+              "object member name, #{next_token.lexeme.inspect} is invalid here"
+          end
+        raise err(msg)
+      end
 
       case next_token.type
-      # FIXME: implement a different name lexer which is limited to only the chars allowed under dot notation
-      # @see https://www.rfc-editor.org/rfc/rfc9535.html#section-2.5.1.1
       when :identifier then parse_name_selector
       when :wildcard then parse_wildcard_selector
       else
-        raise "cannot parse #{current.type}"
+        raise err(
+          'Dot "." begins a name selector, and must be followed by an ' \
+          "object member name, #{next_token.lexeme.inspect} is invalid here"
+        )
       end
     end
 
@@ -300,7 +319,6 @@ module Janeway
     end
 
     def parse_root
-
       # detect optional following selector
       selector =
         case next_token.type
@@ -314,7 +332,6 @@ module Janeway
 
     # Parse the current node operator "@", and optionally a selector which is applied to it
     def parse_current_node
-
       # detect optional following selector
       selector =
         case next_token.type
@@ -347,17 +364,16 @@ module Janeway
     # This is not just a speed optimization. Serial selectors that feed into
     # each other have different behaviour than serial child segments.
     #
-    # @param and_child [Boolean] make following token a child of this selector list
     # @return [AST::ChildSegment]
-    def parse_child_segment(and_child: true)
+    def parse_child_segment
       consume
-      raise "Expect token [, got #{current.lexeme.inspect}" unless current.type == :child_start
+      raise err("Expect token [, got #{current.lexeme.inspect}") unless current.type == :child_start
 
       consume # "["
 
       child_segment = AST::ChildSegment.new
       loop do
-        selector = parse_selector
+        selector = parse_current_selector
         child_segment << selector if selector # nil selector means empty brackets
 
         break unless current.type == :union # no more selectors in these parentheses
@@ -366,33 +382,26 @@ module Janeway
         consume # ","
 
         # not allowed to have comma with nothing after it
-        if current.type == :child_end
-          raise Error.new("Comma must be followed by another expression in filter selector")
-        end
+        raise err('Comma must be followed by another expression in filter selector') if current.type == :child_end
       end
 
-      # Do not consume the final ']', the top-level parsing loop will eat that
-      unless current.type == :child_end
-        # developer error, check the parsing function
-        raise "expect current token to be ], got #{current.type.inspect}"
-      end
+      # Expect ']' after the selector definitions
+      raise err("Unexpected character #{current.lexeme.inspect} within brackets") unless current.type == :child_end
 
       # if the child_segment contains just one selector, then return the selector instead.
       # This way a series of selectors feed results to each other without
       # combining results in a node list.
-      node =
+      expr =
         case child_segment.size
-        when 0 then raise Error.new('Empty child segment')
+        when 0 then raise err('Empty child segment')
         when 1 then child_segment.first
         else child_segment
         end
 
-      if and_child
-        # Parse any subsequent expression which consumes this child segment
-        node.child = parse_next_selector
-      end
+      # Parse any subsequent expression which consumes this child segment
+      expr.next = parse_next_selector
 
-      node
+      expr
     end
 
     # Parse a selector and return it.
@@ -401,6 +410,9 @@ module Janeway
       case next_token.type
       when :child_start then parse_child_segment
       when :dot then parse_dot_notation
+      when :descendants then parse_descendant_segment
+      when :wildcard then parse_wildcard_selector
+      when :eof, :child_end then nil
       end
     end
 
@@ -415,7 +427,7 @@ module Janeway
     end
 
     # Parse a selector which is inside brackets
-    def parse_selector
+    def parse_current_selector
       case current.type
       when :array_slice_separator then parse_array_slice_selector
       when :filter then parse_filter_selector
@@ -423,7 +435,7 @@ module Janeway
       when :minus
         # apply the - sign to the following number and retry
         parse_minus_operator
-        parse_selector
+        parse_current_selector
       when :number
         if lookahead.type == :array_slice_separator
           parse_array_slice_selector
@@ -434,16 +446,17 @@ module Janeway
         AST::NameSelector.new(current_literal_and_consume)
       when :child_end then nil # empty brackets, do nothing.
       else
-        raise "Unhandled selector: #{current}"
+        raise err("Expect selector, got #{current.lexeme.inspect}")
       end
     end
 
     # Parse wildcard selector and any following selector
-    # @param and_child [Boolean] make following token a child of this selector
-    def parse_wildcard_selector(and_child: true)
+    def parse_wildcard_selector
       selector = AST::WildcardSelector.new
       consume
-      selector.child = parse_next_selector if and_child
+      return selector if %i[child_end union].include?(current.type)
+
+      selector.next = parse_next_selector
       selector
     end
 
@@ -461,9 +474,7 @@ module Janeway
     # @return [AST::ArraySliceSelector]
     def parse_array_slice_selector
       start, end_, step = Array.new(3) { parse_array_slice_component }.map { _1&.literal }
-
-
-      raise "After array slice, expect ], got #{current.lexeme}" unless current.type == :child_end # ]
+      raise err("After array slice selector, expect ], got #{current.lexeme}") unless current.type == :child_end # ]
 
       AST::ArraySliceSelector.new(start, end_, step)
     end
@@ -480,7 +491,7 @@ module Janeway
           parse_minus_operator
           parse_array_slice_component
         when :number then current
-        else raise "Unexpected token in array slice selector: #{current}"
+        else raise err("Unexpected token in array slice selector: #{current.lexeme.inspect}")
         end
       consume if current.type == :number
       consume if current.type == :array_slice_separator
@@ -491,25 +502,21 @@ module Janeway
     # The name selector may have been in dot notation or parentheses, that part is already parsed.
     # Next token is just the name.
     #
-    # @param and_child [Boolean] make following token a child of this selector
     # @return [AST::NameSelector]
-    def parse_name_selector(and_child: true)
+    def parse_name_selector
       consume
       selector = AST::NameSelector.new(current.lexeme)
-      if and_child
-        # If there is a following expression, parse that too
-        case next_token.type
-        when :dot then selector.child = parse_dot_notation
-        when :child_start then selector.child = parse_child_segment
-        when :descendants then selector.child = parse_descendant_segment
-        end
+      # If there is a following expression, parse that too
+      case next_token.type
+      when :dot then selector.next = parse_dot_notation
+      when :child_start then selector.next = parse_child_segment
+      when :descendants then selector.next = parse_descendant_segment
       end
       selector
     end
 
     # Feed tokens to the FilterSelector until hitting a terminator
     def parse_filter_selector
-
       selector = AST::FilterSelector.new
       terminator_types = %I[child_end union eof]
       while next_token && !terminator_types.include?(next_token.type)
@@ -526,9 +533,8 @@ module Janeway
       end
 
       # Check for literal, they are not allowed to be a complete condition in a filter selector
-      if selector.value.literal?
-        raise Error, "Literal #{selector.value} must be used within a comparison"
-      end
+      # This includes jsonpath functions that return a numeric value.
+      raise err("Literal value #{selector.value} must be used within a comparison") if selector.value.literal?
 
       consume
 
@@ -543,7 +549,7 @@ module Janeway
         parse_minus_operator
         parse_number
       else
-        raise "unknown unary operator: #{current.inspect}"
+        raise err("Unknown unary operator: #{current.lexeme.inspect}")
       end
     end
 
@@ -567,29 +573,26 @@ module Janeway
     # Parse a JSONPath function call
     def parse_function
       parsing_function = "parse_function_#{current.literal}"
-      result = send(parsing_function)
-      result
+      send(parsing_function)
     end
 
     # Parse an expression
     def parse_expr
       parsing_function = determine_parsing_function
-      raise Error, "Unrecognized token: #{current.lexeme.inspect}" unless parsing_function
+      raise err("Unrecognized token: #{current.lexeme.inspect}") unless parsing_function
 
       send(parsing_function)
     end
 
     def parse_expr_recursively(precedence = LOWEST_PRECEDENCE)
       parsing_function = determine_parsing_function
-      raise Error, "Unrecognized token: #{current.lexeme.inspect}" unless parsing_function
+      raise err("Unrecognized token: #{current.lexeme.inspect}") unless parsing_function
 
-      tk = current
+      current
       expr = send(parsing_function)
       return unless expr # When expr is nil, it means we have reached a \n or a eof.
 
       # Note that here we are checking the NEXT token.
-      if next_not_terminator? && precedence < next_precedence
-      end
       while next_not_terminator? && precedence < next_precedence
         infix_parsing_function = determine_infix_function(next_token)
 
@@ -602,6 +605,14 @@ module Janeway
       expr
     end
 
+    # Return a Parser::Error with the specified message, include the query.
+    #
+    # @param msg [String] error message
+    # @return [Parser::Error]
+    def err(msg)
+      Error.new(msg, @jsonpath)
+    end
+
     alias parse_true parse_boolean
     alias parse_false parse_boolean
   end

data/lib/janeway/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Janeway
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/janeway.rb

@@ -6,6 +6,9 @@ require 'English'
 module Janeway
   # Abstract Syntax Tree
   module AST
+    # These are the limits of what javascript's Number type can represent
+    INTEGER_MIN = -9_007_199_254_740_991
+    INTEGER_MAX = 9_007_199_254_740_991
   end
 
   # Apply a JsonPath query to the input, and return the result.
@@ -14,8 +17,8 @@ module Janeway
   # @param input [Object] ruby object to be searched
   # @return [Array] all matched objects
   def self.find_all(query, input)
-    query = compile(query)
-    Janeway::Interpreter.new(input).interpret(query)
+    ast = compile(query)
+    Janeway::Interpreter.new(ast).interpret(input)
   end
 
   # Compile a JsonPath query into an Abstract Syntax Tree.
@@ -44,8 +47,5 @@ def require_libs(dir)
   end
 end
 
-# These are dependencies of the other AST source files, and must come first
-require_relative 'janeway/ast/expression'
-
 require_libs('janeway/ast')
 require_libs('janeway')

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: janeway-jsonpath
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Fraser Hanson
 bindir: bin
 cert_chain: []
-date: 2025-01-09 00:00:00.000000000 Z
+date: 2025-01-13 00:00:00.000000000 Z
 dependencies: []
 description: |+
   JSONPath is a query language for selecting and extracting values from a JSON text.