janeway-jsonpath 0.1.0 → 0.2.0

data/lib/janeway/functions/value.rb

@@ -33,7 +33,9 @@ module Janeway
 
       # Read parameter
       parameters = [parse_function_parameter]
-      raise "expect group_end token, found #{current}" unless current.type == :group_end
+      unless current.type == :group_end
+        raise Error, 'Too many parameters for value() function call'
+      end
 
       AST::Function.new('value', parameters) do |nodes|
         if nodes.is_a?(Array) && nodes.size == 1

data/lib/janeway/functions.rb

@@ -17,12 +17,12 @@ module Janeway
       in_char_class = false
       indexes = []
       chars.each_with_index do |char, i|
-        # FIXME: does not handle escaped '[', ']', or '.'
         case char
         when '[' then in_char_class = true
-        when ']' then in_char_class = false
+        when ']'
+          in_char_class = false unless chars[i - 1] == '\\' # escaped ] does not close char class
         when '.'
-          next if in_char_class || chars[i-1] == '\\' # escaped dot
+          next if in_char_class || chars[i - 1] == '\\' # escaped dot
 
           indexes << i # replace this dot
         end
@@ -45,6 +45,7 @@ module Janeway
         when :string then parse_string
         when :current_node then parse_current_node
         when :root then parse_root
+        when :group_end then raise Error, 'Function call is missing parameter'
         else
           # Invalid, no function uses this.
           # Instead of crashing here, accept it and let the function return an empty result.

data/lib/janeway/interpreter.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require_relative 'parser'
+
 module Janeway
-  # Tree-walk interpreter to apply the operations from the abstract syntax tree to the input value
+  # Tree-walk interpreter to apply the operations from the abstract syntax tree to the input
   class Interpreter
-    attr_reader :query, :output, :env, :call_stack
+    attr_reader :jsonpath, :output, :env, :call_stack
 
     class Error < Janeway::Error; end
 
@@ -23,29 +25,29 @@ module Janeway
     # @param input [Hash, Array]
     # @param query [String]
     def self.interpret(input, query)
-      raise ArgumentError, "expect query string, got #{query.inspect}" unless query.is_a?(String)
-
       tokens = Lexer.lex(query)
-      ast = Parser.new(tokens).parse
-      new(input).interpret(ast)
+      ast = Parser.new(tokens, query).parse
+      new(ast).interpret(input)
     end
 
-    # @param input [Array,Hash] tree of data which the jsonpath query is addressing
-    def initialize(input)
-      @input = input
+    # @param query [AST::Query] abstract syntax tree of the jsonpath query
+    def initialize(query)
+      raise ArgumentError, "expect AST::Query, got #{query.inspect}" unless query.is_a?(AST::Query)
+
+      @query = query
+      @jsonpath = query.jsonpath
+      @input = nil
     end
 
-    # @param ast [AST::Query] abstract syntax tree
+    # @param input [Array, Hash] object to be searched
     # @return [Object]
-    def interpret(ast)
-      @query = ast
-      raise "expect AST, got #{ast.inspect}" unless ast.is_a?(AST::Query)
-
+    def interpret(input)
+      @input = input
       unless @input.is_a?(Hash) || @input.is_a?(Array)
         return [] # can't query on any other types
       end
 
-      interpret_node(ast.root, nil)
+      interpret_node(@query.root, nil)
     end
 
     private
@@ -62,26 +64,10 @@ module Janeway
       when AST::Selector then interpret_selector(node.value, @input)
       when nil then [@input]
       else
-        raise "don't know how to interpret #{node.value.class}"
+        raise err("don't know how to interpret #{node.value.class}")
       end
     end
 
-    # Prepare a single node from an input node list to be sent to a selector.
-    # (Selectors require a node list as input)
-    # Helper method for interpret_child_segment.
-    #
-    # @param node [Object]
-    def as_node_list(node)
-      # FIXME: method still used?  Can this be delted?
-      result =
-        case node
-        when Array then node
-        when Hash then node
-        else [node]
-        end
-      result
-    end
-
     # Interpret a list of 1 or more selectors, seperated by the union operator.
     #
     # @param child_segment [AST::ChildSegment]
@@ -105,9 +91,8 @@ module Janeway
         end
       end
 
-
       # Send result to the next node in the AST, if any
-      child = child_segment.child
+      child = child_segment.next
       unless child
         return child_segment.size == 1 ? [results] : results
       end
@@ -129,12 +114,11 @@ module Janeway
 
       # early exit, no point continuing the chain with no results
 
-      return [result] unless selector.child
+      return [result] unless selector.next
 
       # Interpret child using output of this name selector, and return result
-      child = selector.child
-      results = send(:"interpret_#{child.type}", child, result)
-      results
+      child = selector.next
+      send(:"interpret_#{child.type}", child, result)
     end
 
     # Filter the input by returning the array element with the given index.
@@ -152,11 +136,10 @@ module Janeway
       result = input.fetch(selector.value) # raises IndexError if no such index
 
       # Interpret child using output of this name selector, and return result
-      child = selector.child
+      child = selector.next
       return [result] unless child
 
-      results = send(:"interpret_#{child.type}", child, result)
-      results
+      send(:"interpret_#{child.type}", child, result)
     rescue IndexError
       [] # returns empty array if no such index
     end
@@ -178,15 +161,13 @@ module Janeway
         end
 
       return values if values.empty? # early exit, no need for further processing on empty list
-      return values unless selector.child
+      return values unless selector.next
 
-      # Apply next selector to every value
-      # FIXME: is this correct??  Makes the CTS pass  "basic, wildcard shorthand, then name shorthand"
-      child = selector.child
+      # Apply child selector to each node in the output node list
+      child = selector.next
       results = []
       values.each do |value|
-        result = send(:"interpret_#{child.type}", child, value)
-        results << result.first unless result.empty?
+        results.concat send(:"interpret_#{child.type}", child, value)
       end
       results
     end
@@ -199,23 +180,31 @@ module Janeway
     # @return [Array]
     def interpret_array_slice_selector(selector, input)
       return [] unless input.is_a?(Array)
-      return [] if selector.step && selector.step.zero? # IETF: When step is 0, no elements are selected.
-
-      # Calculate the "real" start and end index based on the array size
-      start_index = selector.start_index(input.size)
-      last_index = selector.end_index(input.size)
+      return [] if selector&.step&.zero? # IETF: When step is 0, no elements are selected.
 
+      # Calculate the upper and lower indices of the target range
+      lower = selector.lower_index(input.size)
+      upper = selector.upper_index(input.size)
 
-      # Collect values from target indices.
-      results = start_index
-        .step(to: last_index, by: selector.step)
-        .map { |i| input[i] }
+      # Collect values from target indices. Omit the value from the final index.
+      results =
+        if selector.step.positive?
+          lower.step(to: upper - 1, by: selector.step).map { input[_1] }
+        else
+          upper.step(to: lower + 1, by: selector.step).map { input[_1] }
+        end
 
       # Interpret child using output of this name selector, and return result
-      child = selector.child
+      child = selector.next
       return results unless child
 
-      send(:"interpret_#{child.type}", child, results)
+      # Apply child selector to each node in the output node list
+      node_list = results
+      results = []
+      node_list.each do |node|
+        results.concat send(:"interpret_#{child.type}", child, node)
+      end
+      results
     end
 
     # Return the set of values from the input for which the filter is true.
@@ -231,12 +220,11 @@ module Janeway
         case input
         when Array then input
         when Hash then input.values
-        else return []
+        else return [] # early exit
         end
 
       results = []
       values.each do |value|
-
         # Run filter and interpret result
         result = interpret_node(selector.value, value)
 
@@ -249,11 +237,16 @@ module Janeway
         end
       end
 
-      return results unless selector.child
+      child = selector.next
+      return results unless child
 
-      # Interpret child using output of this name selector, and return result
-      child = selector.child
-      send(:"interpret_#{child.type}", child, results)
+      # Apply child selector to each node in the output node list
+      node_list = results
+      results = []
+      node_list.each do |node|
+        results.concat send(:"interpret_#{child.type}", child, node)
+      end
+      results
     end
 
     # Combine results from selectors into a single list.
@@ -277,12 +270,7 @@ module Janeway
     # @param input [Object]
     # @return [Array<AST::Expression>] node list
     def interpret_descendant_segment(descendant_segment, input)
-      results = visit(input) { |node| interpret_node(descendant_segment.selector, node) }
-
-      return results unless descendant_segment.child
-
-      child = descendant_segment.child
-      send(:"interpret_#{child.type}", child, results)
+      visit(input) { |node| interpret_node(descendant_segment.next, node) }
     end
 
     # Visit all descendants of `root`.
@@ -329,14 +317,13 @@ module Janeway
     # @param node [AST::Expression]
     # @param input [Object]
     def interpret_node_as_value(node, input)
-
       # nodes must be singular queries or literals
       case node
       when AST::CurrentNode, AST::RootNode
-        raise Error, "Expression #{node} does not produce a singular value for comparison" unless node.singular_query?
+        raise err("Expression #{node} does not produce a singular value for comparison") unless node.singular_query?
       when AST::Number, AST::StringType, AST::Null, AST::Function, AST::Boolean then nil
       else
-        raise "Invalid expression for comparison: #{node}"
+        raise err("Invalid expression for comparison: #{node}")
       end
 
       result = interpret_node(node, input)
@@ -352,7 +339,7 @@ module Janeway
       return result if result.empty?
 
       # Return the only node in the node list
-      raise 'node list contains multiple elements but this is a comparison' unless result.size == 1
+      raise err('node list contains multiple elements but this is a comparison') unless result.size == 1
 
       result.first
     end
@@ -366,7 +353,7 @@ module Janeway
 
       return node_list.first if node_list.size == 1
 
-      raise "don't know how to handle node list with size > 1: #{node_list.inspect}"
+      raise err("don't know how to handle node list with size > 1: #{node_list.inspect}")
     end
 
     # Evaluate a selector and return the result
@@ -379,7 +366,7 @@ module Janeway
       when AST::ArraySliceSelector then interpret_array_slice_selector(selector, input)
       when AST::FilterSelector then interpret_filter_selector(selector, input)
       else
-        raise "Not a selector: #{selector.inspect}"
+        raise err("Not a selector: #{selector.inspect}")
       end
     end
 
@@ -387,7 +374,8 @@ module Janeway
     #
     # The result is an Array containing all results of evaluating the CurrentNode's selector (if any.)
     #
-    # If the selector extracted values from nodes such as strings, numbers or nil/null, these will be included in the array.
+    # If the selector extracted values from nodes such as strings, numbers or nil/null,
+    # these will be included in the array.
     # If the selector did not match any node, the array may be empty.
     # If there was no selector, then the current input node is returned in the array.
     #
@@ -396,21 +384,19 @@ module Janeway
     # @return [Array] Node List containing all results from evaluating this node's selectors.
     def interpret_current_node(current_node, input)
       next_expr = current_node.value
-      result =
-        # All of these return a node list
-        case next_expr
-        when AST::NameSelector then interpret_name_selector(next_expr, input)
-        when AST::WildcardSelector then interpret_wildcard_selector(next_expr, input)
-        when AST::IndexSelector then interpret_index_selector(next_expr, input)
-        when AST::ArraySliceSelector then interpret_array_slice_selector(next_expr, input)
-        when AST::FilterSelector then interpret_filter_selector(next_expr, input)
-        when AST::ChildSegment then interpret_child_segment(next_expr, input)
-        when AST::DescendantSegment then interpret_descendant_segment(next_expr, input)
-        when NilClass then input # FIXME: put it in a node list???
-        else
-          raise "don't know how to interpret #{next_expr}"
-        end
-      result
+      # All of these return a node list
+      case next_expr
+      when AST::NameSelector then interpret_name_selector(next_expr, input)
+      when AST::WildcardSelector then interpret_wildcard_selector(next_expr, input)
+      when AST::IndexSelector then interpret_index_selector(next_expr, input)
+      when AST::ArraySliceSelector then interpret_array_slice_selector(next_expr, input)
+      when AST::FilterSelector then interpret_filter_selector(next_expr, input)
+      when AST::ChildSegment then interpret_child_segment(next_expr, input)
+      when AST::DescendantSegment then interpret_descendant_segment(next_expr, input)
+      when NilClass then input
+      else
+        raise err("don't know how to interpret @#{next_expr}")
+      end
     end
 
     def interpret_identifier(identifier, _input)
@@ -422,7 +408,7 @@ module Janeway
         call_stack.last.env[identifier.name]
       else
         # Undefined variable.
-        raise Janeway::Error::Runtime::UndefinedVariable, identifier.name
+        raise err("Undefined identifier: #{identifier.name}")
       end
     end
 
@@ -446,13 +432,12 @@ module Janeway
         lhs = interpret_node_as_value(binary_op.left, input)
         rhs = interpret_node_as_value(binary_op.right, input)
       else
-        raise "don't know how to handle binary operator #{binary_op.inspect}"
+        raise err("Don't know how to handle binary operator #{binary_op.inspect}")
       end
       send(:"interpret_#{binary_op.operator}", lhs, rhs)
     end
 
     def interpret_equal(lhs, rhs)
-
       # When either side of a comparison results in an empty nodelist or the
       # special result Nothing (see Section 2.4.1):
       # A comparison using the operator == yields true if and only if the other
@@ -541,14 +526,14 @@ module Janeway
       nil
     end
 
-    # FIXME: split implementation out into separate methods for not and minus
-    # because they are so different.
+    # @param op [AST::UnaryOperator]
+    # @param input [Object]
     def interpret_unary_operator(op, input)
+      # The only other unary operator, "-", is consumed in the parsing stage and never in the AST
       node_list = send(:"interpret_#{op.operand.type}", op.operand, input)
       case op.operator
       when :not then interpret_not(node_list)
-      when :minus then 0 - node_list.first # FIXME: sure hope this is a number!
-      else raise "unknown unary operator #{op.inspect}"
+      else raise err("unknown unary operator #{op.inspect}")
       end
     end
 
@@ -557,28 +542,25 @@ module Janeway
     # For a boolean, this inverts the meaning of the input.
     # @return [Boolean]
     def interpret_not(input)
-      result =
-        case input
-        when Array then input.empty?
-        when TrueClass, FalseClass then !input
-        else
-          raise "don't know how to apply not operator to #{input.inspect}"
-        end
-      result
+      case input
+      when Array then input.empty?
+      when TrueClass, FalseClass then !input
+      else
+        raise err("don't know how to apply not operator to #{input.inspect}")
+      end
     end
 
     # @param function [AST::Function]
     # @param input [Hash, Array]
     def interpret_function(function, input)
       params = evaluate_function_parameters(function.parameters, function.name, input)
-      result = function.body.call(*params)
-      result
+      function.body.call(*params)
     end
 
     # Evaluate the expressions in the parameter list to make the parameter values
     # to pass in to a JsonPath function.
     #
-    # The node lists returned by a singulare query must be deconstructed into a single value for
+    # The node lists returned by a singular query must be deconstructed into a single value for
     # parameters of ValueType, this is done here.
     # For explanation:
     # @see https://www.rfc-editor.org/rfc/rfc9535.html#name-well-typedness-of-function-
@@ -602,6 +584,8 @@ module Janeway
           else
             result
           end
+        when AST::Function
+          interpret_function(parameter, input)
         when AST::StringType, AST::Number
           interpret_string_type(parameter, input)
         else
@@ -640,5 +624,13 @@ module Janeway
         input # input is a single node, which happens to be an Array
       end
     end
+
+    # Return an Interpreter::Error with the specified message, include the query.
+    #
+    # @param msg [String] error message
+    # @return [Parser::Error]
+    def err(msg)
+      Error.new(msg, @jsonpath)
+    end
   end
 end

data/lib/janeway/lexer.rb

@@ -73,10 +73,10 @@ module Janeway
 
     def start_tokenization
       if WHITESPACE.include?(@source[0]) || WHITESPACE.include?(@source[-1])
-        raise Error, 'JSONPath query may not start or end with whitespace'
+        raise err('JSONPath query may not start or end with whitespace')
       end
-      tokenize while source_uncompleted?
 
+      tokenize while source_uncompleted?
       tokens << Token.new(:eof, '', nil, after_source_end_location)
     end
 
@@ -99,13 +99,13 @@ module Janeway
         elsif digit?(c)
           lex_number
         elsif name_first_char?(c)
-          lex_member_name_shorthand(ignore_keywords: tokens.last.type == :dot)
+          lex_member_name_shorthand(ignore_keywords: tokens.last&.type == :dot)
         end
 
       if token
         tokens << token
       else
-        raise Error.new("Unknown character: #{c.inspect}", @source, current_location)
+        raise err("Unknown character: #{c.inspect}")
       end
     end
 
@@ -126,7 +126,7 @@ module Janeway
 
     def token_from_one_char_lex(lexeme)
       if %w[. -].include?(lexeme) && WHITESPACE.include?(lookahead)
-        raise Error, "Operator #{lexeme.inspect} must not be followed by whitespace"
+        raise err("Operator #{lexeme.inspect} must not be followed by whitespace")
       end
 
       Token.new(OPERATORS.key(lexeme), lexeme, nil, current_location)
@@ -139,7 +139,7 @@ module Janeway
       if TWO_CHAR_LEX.include?(next_two_chars)
         consume
         if next_two_chars == '..' && WHITESPACE.include?(lookahead)
-          raise Error, "Operator #{next_two_chars.inspect} must not be followed by whitespace"
+          raise err("Operator #{next_two_chars.inspect} must not be followed by whitespace")
         end
         Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
       else
@@ -151,7 +151,9 @@ module Janeway
     # @return [Token]
     def token_from_two_char_lex(lexeme)
       next_two_chars = [lexeme, lookahead].join
-      raise Error.new("Unknown operator \"#{next_two_chars}\"", @source) unless TWO_CHAR_LEX.include?(next_two_chars)
+      unless TWO_CHAR_LEX.include?(next_two_chars)
+        raise err("Unknown operator \"#{lexeme}\"")
+      end
 
       consume
       Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
@@ -183,7 +185,7 @@ module Janeway
               consume # delimiter
             elsif lookahead(2) == non_delimiter
               qtype = delimiter == '"' ? 'double' : 'single'
-              raise Error, "Character #{non_delimiter} must not be escaped within #{qtype} quotes"
+              raise err("Character #{non_delimiter} must not be escaped within #{qtype} quotes")
             else
               consume_escape_sequence # consumes multiple chars
             end
@@ -192,10 +194,10 @@ module Janeway
           elsif %w[' "].include?(next_char) && next_char != delimiter
             consume
           else
-            raise Error.new("invalid character #{next_char.inspect}", current_location)
+            raise err("invalid character #{next_char.inspect}")
           end
       end
-      raise Error.new("Unterminated string error: #{literal_chars.join.inspect}") if source_completed?
+      raise err("Unterminated string error: #{literal_chars.join.inspect}") if source_completed?
 
       consume # closing delimiter
 
@@ -211,7 +213,7 @@ module Janeway
     # Read escape char literals, and transform them into the described character
     # @return [String] single character (possibly multi-byte)
     def consume_escape_sequence
-      raise 'not an escape sequence' unless consume == '\\'
+      raise err('Expect escape sequence') unless consume == '\\'
 
       char = consume
       case char
@@ -224,10 +226,10 @@ module Janeway
       when 'u' then consume_unicode_escape_sequence
       else
         if unescaped?(char)
-          raise Error.new("Character #{char} must not be escaped")
+          raise err("Character #{char} must not be escaped")
         else
           # whatever this is, it is not allowed even when escaped
-          raise Error.new("Invalid character #{char.inspect}", current_location)
+          raise err("Invalid character #{char.inspect}")
         end
       end
     end
@@ -255,7 +257,7 @@ module Janeway
       return hex_str.hex.chr('UTF-8') unless hex_str.upcase.start_with?('D')
 
       # hex string starts with D, but is still non-surrogate
-      return hex_str.hex.chr('UTF-8') if (0..7).cover?(hex_str[1].to_i)
+      return [hex_str.hex].pack('U') if '01234567'.include?(hex_str[1])
 
       # hex value is in the high-surrogate or low-surrogate range.
 
@@ -273,11 +275,11 @@ module Janeway
         return convert_surrogate_pair_to_codepoint(hex_str, hex_str2) if prefix == '\\u' && low_surrogate?(hex_str2)
 
         # Not allowed to have high surrogate that is not followed by low surrogate
-        raise "invalid unicode escape sequence: \\u#{hex_str2.join}"
+        raise err("Invalid unicode escape sequence: \\u#{hex_str2}")
 
       end
       # Not allowed to have low surrogate that is not preceded by high surrogate
-      raise "invalid unicode escape sequence: \\u#{hex_str}"
+      raise err("Invalid unicode escape sequence: \\u#{hex_str}")
     end
 
     # Convert a valid UTF-16 surrogate pair into a UTF-8 string containing a single code point.
@@ -313,7 +315,7 @@ module Janeway
     end
 
     # Consume and return 4 hex digits from the source. Either upper or lower case is accepted.
-    # No judgement is made here on whether the resulting sequence is valid,
+    # No judgment is made here on whether the resulting sequence is valid,
     # as long as it is 4 hex digits.
     #
     # @return [String]
@@ -326,10 +328,10 @@ module Janeway
         when 0x40..0x46 then next # 'A'..'F'
         when 0x61..0x66 then next # 'a'..'f'
         else
-          raise "invalid unicode escape sequence: \\u#{hex_digits.join}"
+          raise err("Invalid unicode escape sequence: \\u#{hex_digits.join}")
         end
       end
-      raise "incomplete unicode escape sequence: \\u#{hex_digits.join}" if hex_digits.size < 4
+      raise err("Incomplete unicode escape sequence: \\u#{hex_digits.join}") if hex_digits.size < 4
 
       hex_digits.join
     end
@@ -355,14 +357,14 @@ module Janeway
         end
         unless digit?(lookahead)
           lexeme = source[lexeme_start_p..(next_p - 1)]
-          raise Error, "Exponent 'e' must be followed by number: #{lexeme.inspect}"
+          raise err("Exponent 'e' must be followed by number: #{lexeme.inspect}")
         end
         consume_digits
       end
 
       lexeme = source[lexeme_start_p..(next_p - 1)]
       if lexeme.start_with?('0') && lexeme.size > 1
-        raise Error, "Number may not start with leading zero: #{lexeme.inspect}"
+        raise err("Number may not start with leading zero: #{lexeme.inspect}")
       end
 
       literal =
@@ -375,7 +377,7 @@ module Janeway
     end
 
     # Consume an alphanumeric string.
-    # If `ignore_keywords`, the result is alway an :identifier token.
+    # If `ignore_keywords`, the result is always an :identifier token.
     # Otherwise, keywords and function names will be recognized and tokenized as those types.
     #
     # @param ignore_keywords [Boolean]
@@ -445,7 +447,7 @@ module Janeway
     # True if character is suitable as the first character in a name selector
     # using shorthand notation (ie. no bracket notation.)
     #
-    # Defined in RFC9535 by ths ABNF grammar:
+    # Defined in RFC9535 by this ABNF grammar:
     # name-first    = ALPHA /
     #                 "_"   /
     #                 %x80-D7FF /
@@ -490,8 +492,9 @@ module Janeway
           :identifier
         end
       if type == :function && WHITESPACE.include?(lookahead)
-        raise Error, "Function name \"#{identifier}\" must not be followed by whitespace"
+        raise err("Function name \"#{identifier}\" must not be followed by whitespace")
       end
+
       Token.new(type, identifier, identifier, current_location)
     end
 
@@ -510,5 +513,13 @@ module Janeway
     def after_source_end_location
       Location.new(next_p, 1)
     end
+
+    # Return a Lexer::Error with the specified message, include the query and location
+    #
+    # @param msg [String] error message
+    # @return [Lexer::Error]
+    def err(msg)
+      Error.new(msg, @source, current_location)
+    end
   end
 end