janeway-jsonpath 0.2.0 → 0.3.0

data/lib/janeway/interpreters/unary_operator_interpreter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Janeway
+  module Interpreters
+    # Interprets the "not" operator within a filter selector logical expression.
+    # The only other unary operator is "minus", which is consumed during parsing and is not part of the AST.
+    class UnaryOperatorInterpreter < Base
+      alias operator node
+
+      def initialize(operator)
+        super
+        raise "Unknown unary operator: #{name}" unless operator.name == :not
+
+        # Expression which must be evaluated, not operator will be applied to result
+        @operand = TreeConstructor.ast_node_to_interpreter(operator.operand)
+      end
+
+      # @return [Boolean]
+      def interpret(input, root)
+        result = @operand.interpret(input, root)
+        case result
+        when Array then result.empty?
+        when TrueClass, FalseClass then !result
+        else
+          raise "don't know how to apply not operator to #{result.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/wildcard_selector_interpreter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Janeway
+  module Interpreters
+    # Interprets a wildcard selector, returns results or forwards them to next selector
+    class WildcardSelectorInterpreter < Base
+      alias selector node
+
+      # Return values from the input.
+      # For array, return the array.
+      # For Hash, return hash values.
+      # For anything else, return empty list.
+      #
+      # @param input [Array, Hash] the results of processing so far
+      # @param root [Array, Hash] the entire input
+      def interpret(input, root)
+        values =
+          case input
+          when Array then input
+          when Hash then input.values
+          else []
+          end
+
+        return values if values.empty? # early exit, no need for further processing on empty list
+        return values unless @next
+
+        # Apply child selector to each node in the output node list
+        results = []
+        values.each do |value|
+          results.concat @next.interpret(value, root)
+        end
+        results
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/yielder.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Janeway
+  module Interpreters
+    # Yields each input value.
+    # It is inserted at the end of the "real" selectors in the AST, to receive and yield the output.
+    # This is a supporting class for the supports the Janeway.each method.
+    #
+    # This will get pushed onto the end of the query AST.
+    # Currently it must act like both an AST node and an interpreter.
+    # This will be simpler if TODO some day may the interpreter subclass methods can be merged into the AST classes
+    class Yielder < Base
+      # The implicit constructor forwards a closure to the base class constructor.
+      # Base class constructor stores it in @node.
+      def initialize(&block)
+        super(Struct.new(:next).new)
+        @block = block
+      end
+
+      # Yield each input value
+      #
+      # @param input [Array, Hash] the results of processing so far
+      # @param _root [Array, Hash] the entire input
+      # @yieldparam [Object] matched value
+      # @return [Object] input
+      def interpret(input, _root)
+        @block.call(input)
+        input.is_a?(Array) ? input : [input]
+      end
+    end
+  end
+end

data/lib/janeway/lexer.rb

@@ -101,12 +101,9 @@ module Janeway
         elsif name_first_char?(c)
           lex_member_name_shorthand(ignore_keywords: tokens.last&.type == :dot)
         end
+      raise err("Unknown character: #{c.inspect}") unless token
 
-      if token
-        tokens << token
-      else
-        raise err("Unknown character: #{c.inspect}")
-      end
+      tokens << token
     end
 
     def digit?(lexeme)
@@ -141,6 +138,7 @@ module Janeway
         if next_two_chars == '..' && WHITESPACE.include?(lookahead)
           raise err("Operator #{next_two_chars.inspect} must not be followed by whitespace")
         end
+
         Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
       else
         token_from_one_char_lex(lexeme)
@@ -151,9 +149,7 @@ module Janeway
     # @return [Token]
     def token_from_two_char_lex(lexeme)
       next_two_chars = [lexeme, lookahead].join
-      unless TWO_CHAR_LEX.include?(next_two_chars)
-        raise err("Unknown operator \"#{lexeme}\"")
-      end
+      raise err("Unknown operator \"#{lexeme}\"") unless TWO_CHAR_LEX.include?(next_two_chars)
 
       consume
       Token.new(OPERATORS.key(next_two_chars), next_two_chars, nil, current_location)
@@ -172,7 +168,9 @@ module Janeway
     # @param delimiter [String] eg. ' or "
     # @return [Token] string token
     def lex_delimited_string(delimiter)
-      non_delimiter = %w[' "].reject { _1 == delimiter }.first
+      allowed_delimiters = %w[' "]
+      # the "other" delimiter char, which is not currently being treated as a delimiter
+      non_delimiter = allowed_delimiters.reject { _1 == delimiter }.first
 
       literal_chars = []
       while lookahead != delimiter && source_uncompleted?
@@ -191,7 +189,7 @@ module Janeway
             end
           elsif unescaped?(next_char)
             consume
-          elsif %w[' "].include?(next_char) && next_char != delimiter
+          elsif allowed_delimiters.include?(next_char) && next_char != delimiter
             consume
           else
             raise err("invalid character #{next_char.inspect}")
@@ -225,12 +223,10 @@ module Janeway
       when '/', '\\', '"', "'" then char
       when 'u' then consume_unicode_escape_sequence
       else
-        if unescaped?(char)
-          raise err("Character #{char} must not be escaped")
-        else
-          # whatever this is, it is not allowed even when escaped
-          raise err("Invalid character #{char.inspect}")
-        end
+        raise err("Character #{char} must not be escaped") if unescaped?(char)
+
+        # whatever this is, it is not allowed even when escaped
+        raise err("Invalid character #{char.inspect}")
       end
     end
 

data/lib/janeway/parser.rb

@@ -1,23 +1,20 @@
 # frozen_string_literal: true
 
+require_relative 'error'
 require_relative 'functions'
 require_relative 'lexer'
 
 module Janeway
   # Transform a list of tokens into an Abstract Syntax Tree
   class Parser
-    class Error < Janeway::Error; end
-
-    attr_accessor :tokens
+    attr_reader :tokens
 
     include Functions
 
     UNARY_OPERATORS = %w[! -].freeze
     BINARY_OPERATORS = %w[== != > < >= <= ,].freeze
     LOGICAL_OPERATORS = %w[&& ||].freeze
-
     LOWEST_PRECEDENCE = 0
-    PREFIX_PRECEDENCE = 7
     OPERATOR_PRECEDENCE = {
       ',' => 0,
       '||' => 1,
@@ -31,9 +28,8 @@ module Janeway
       '(' => 8,
     }.freeze
 
-    # @param query [String] jsonpath query to be lexed and parsed
-    #
-    # @return [AST]
+    # @param jsonpath [String] jsonpath query to be lexed and parsed
+    # @return [AST::Query]
     def self.parse(jsonpath)
       raise ArgumentError, "expect jsonpath string, got #{jsonpath.inspect}" unless jsonpath.is_a?(String)
 
@@ -41,7 +37,7 @@ module Janeway
       new(tokens, jsonpath).parse
     end
 
-    # @param token [Array<Token>]
+    # @param tokens [Array<Token>]
     # @param jsonpath [String] original jsonpath query string
     def initialize(tokens, jsonpath)
       @tokens = tokens
@@ -55,23 +51,18 @@ module Janeway
       consume
       raise err('JsonPath queries must start with root identifier "$"') unless current.type == :root
 
-      root_node = parse_expr_recursively
+      root_node = parse_root
       consume
       unless current.type == :eof
         remaining = tokens[@next_p..].map(&:lexeme).join
         raise err("Unrecognized expressions after query: #{remaining}")
       end
 
-      # Freeze so this can be used in ractors
-      AST::Query.new(root_node, @jsonpath).freeze
+      AST::Query.new(root_node, @jsonpath)
     end
 
     private
 
-    def build_token(type, lexeme = nil)
-      Token.new(type, lexeme, nil, nil)
-    end
-
     def pending_tokens?
       @next_p < tokens.length
     end
@@ -94,12 +85,12 @@ module Janeway
       current.literal.tap { consume }
     end
 
-    def consume_if_next_is(expected)
-      if next_token.type == expected.type
+    def consume_if_next_is(token_type)
+      if next_token.type == token_type
         consume
         true
       else
-        unexpected_token_error(expected)
+        unexpected_token_error(token_type)
         false
       end
     end
@@ -131,20 +122,14 @@ module Janeway
       OPERATOR_PRECEDENCE[next_token.lexeme] || LOWEST_PRECEDENCE
     end
 
-    def unexpected_token_error(expected = nil)
-      if expected
+    def unexpected_token_error(expected_type = nil)
+      if expected_type
         raise err(
           "Unexpected token #{current.lexeme.inspect} " \
-          "(expected #{expected.inspect}, got #{next_token.lexeme.inspect} )"
+          "(expected #{expected_type}, got #{next_token.lexeme} )"
         )
       end
-      raise err("Unexpected token #{current.lexeme.inspect} (next is #{next_token.inspect})")
-    end
-
-    def check_syntax_compliance(ast_node)
-      return if ast_node.expects?(next_token)
-
-      unexpected_token_error
+      raise err("Unexpected token #{current.lexeme.inspect} (next is #{next_token})")
     end
 
     def determine_parsing_function
@@ -179,10 +164,11 @@ module Janeway
       :parse_binary_operator
     end
 
+    # A non-delimited word that is not a keyword within a jsonpath query is not allowed.
+    # eg. $[?@==foo]
+    # @raise
     def parse_identifier
-      ident = AST::Identifier.new(current.lexeme)
-      check_syntax_compliance(ident)
-      ident
+      unexpected_token_error
     end
 
     def parse_string
@@ -194,7 +180,7 @@ module Janeway
     end
 
     # Consume minus operator and apply it to the (expected) number token following it.
-    # Don't consume the number token.
+    # Don't consume the number token. The minus operator does not end up in the AST.
     def parse_minus_operator
       raise err("Expect token '-', got #{current.lexeme.inspect}") unless current.type == :minus
 
@@ -282,7 +268,7 @@ module Janeway
     def parse_dot_notation
       consume # "."
       unless current.type == :dot
-        # Parse error, determine most useful error message
+        # Parse error. Determine the most useful error message for this situation:
         msg =
           if current.type == :number
             "Decimal point must be preceded by number, got \".#{current.lexeme}\""
@@ -308,39 +294,29 @@ module Janeway
       consume
 
       expr = parse_expr_recursively
-      return unless consume_if_next_is(build_token(:group_end, ')'))
+      return unless consume_if_next_is(:group_end)
 
       expr
     end
 
-    # TODO: Temporary impl; reflect more deeply about the appropriate way of parsing a terminator.
     def parse_terminator
       nil
     end
 
+    # Parse the root identifier "$", and any subsequent selector
+    # @return [AST::RootNode]
     def parse_root
-      # detect optional following selector
-      selector =
-        case next_token.type
-        when :dot then parse_dot_notation
-        when :child_start then parse_child_segment
-        when :descendants then parse_descendant_segment
-        end
-
-      AST::RootNode.new(selector)
+      AST::RootNode.new.tap do |root_node|
+        root_node.next = parse_next_selector
+      end
     end
 
-    # Parse the current node operator "@", and optionally a selector which is applied to it
+    # Parse the current node operator "@", and any subsequent selector
+    # @return [AST::CurrentNode]
     def parse_current_node
-      # detect optional following selector
-      selector =
-        case next_token.type
-        when :dot then parse_dot_notation
-        when :child_start then parse_child_segment
-        when :descendants then parse_descendant_segment
-        end
-
-      AST::CurrentNode.new(selector)
+      AST::CurrentNode.new.tap do |current_node|
+        current_node.next = parse_next_selector
+      end
     end
 
     # Parse one or more selectors surrounded by parentheses.
@@ -411,21 +387,10 @@ module Janeway
       when :child_start then parse_child_segment
       when :dot then parse_dot_notation
       when :descendants then parse_descendant_segment
-      when :wildcard then parse_wildcard_selector
       when :eof, :child_end then nil
       end
     end
 
-    # Return true if the given token represents the start of any type of selector,
-    # or a collection of selectors.
-    #
-    # @param token [Token]
-    # @return [Boolean]
-    def selector?(token)
-      type = token.type.to_s
-      type.include?('selector') || %w[dot child_start].include?(type)
-    end
-
     # Parse a selector which is inside brackets
     def parse_current_selector
       case current.type
@@ -451,13 +416,12 @@ module Janeway
     end
 
     # Parse wildcard selector and any following selector
+    # @return [AST::WildcardSelector]
     def parse_wildcard_selector
-      selector = AST::WildcardSelector.new
-      consume
-      return selector if %i[child_end union].include?(current.type)
-
-      selector.next = parse_next_selector
-      selector
+      AST::WildcardSelector.new.tap do |selector|
+        consume # *
+        selector.next = parse_next_selector unless %i[child_end union].include?(current.type)
+      end
     end
 
     # An array slice start:end:step selects a series of elements from
@@ -474,7 +438,10 @@ module Janeway
     # @return [AST::ArraySliceSelector]
     def parse_array_slice_selector
       start, end_, step = Array.new(3) { parse_array_slice_component }.map { _1&.literal }
-      raise err("After array slice selector, expect ], got #{current.lexeme}") unless current.type == :child_end # ]
+
+      unless %i[child_end union].include?(current.type)
+        raise err("Array slice selector must be followed by \",\" or \"]\", got #{current.lexeme}")
+      end
 
       AST::ArraySliceSelector.new(start, end_, step)
     end
@@ -500,19 +467,13 @@ module Janeway
 
     # Parse a name selector.
     # The name selector may have been in dot notation or parentheses, that part is already parsed.
-    # Next token is just the name.
+    # Next token is the name.
     #
     # @return [AST::NameSelector]
     def parse_name_selector
-      consume
-      selector = AST::NameSelector.new(current.lexeme)
-      # If there is a following expression, parse that too
-      case next_token.type
-      when :dot then selector.next = parse_dot_notation
-      when :child_start then selector.next = parse_child_segment
-      when :descendants then selector.next = parse_descendant_segment
+      AST::NameSelector.new(consume.lexeme).tap do |selector|
+        selector.next = parse_next_selector
       end
-      selector
     end
 
     # Feed tokens to the FilterSelector until hitting a terminator
@@ -528,11 +489,11 @@ module Janeway
             parse_expr_recursively
           end
 
-        # may replace existing node with a binary operator that incorporates the original node
+        # may replace existing node with a binary operator that contains the existing node
         selector.value = node
       end
 
-      # Check for literal, they are not allowed to be a complete condition in a filter selector
+      # Check for literals, they are not allowed to be a complete condition in a filter selector.
       # This includes jsonpath functions that return a numeric value.
       raise err("Literal value #{selector.value} must be used within a comparison") if selector.value.literal?
 
@@ -556,7 +517,7 @@ module Janeway
     def parse_not_operator
       AST::UnaryOperator.new(current.type).tap do |op|
         consume
-        op.operand = parse_expr_recursively(PREFIX_PRECEDENCE)
+        op.operand = parse_expr_recursively
       end
     end
 
@@ -584,18 +545,18 @@ module Janeway
       send(parsing_function)
     end
 
+    # Parse an expression which may contain binary operators and varying expression precedence.
+    # This is only needed within a filter expression.
     def parse_expr_recursively(precedence = LOWEST_PRECEDENCE)
       parsing_function = determine_parsing_function
       raise err("Unrecognized token: #{current.lexeme.inspect}") unless parsing_function
 
-      current
       expr = send(parsing_function)
-      return unless expr # When expr is nil, it means we have reached a \n or a eof.
+      return unless expr
 
-      # Note that here we are checking the NEXT token.
+      # Keep parsing until next token is higher precedence, or a terminator
       while next_not_terminator? && precedence < next_precedence
         infix_parsing_function = determine_infix_function(next_token)
-
         return expr if infix_parsing_function.nil?
 
         consume
@@ -610,7 +571,7 @@ module Janeway
     # @param msg [String] error message
     # @return [Parser::Error]
     def err(msg)
-      Error.new(msg, @jsonpath)
+      Janeway::Error.new(msg, @jsonpath)
     end
 
     alias parse_true parse_boolean

data/lib/janeway/token.rb

@@ -3,6 +3,7 @@
 require 'forwardable'
 
 module Janeway
+  # Tokens are produced by the lexer, they represent jsonpath query elements in a low-level, non-hierarchical way.
   class Token
     extend Forwardable
 

data/lib/janeway/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Janeway
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/janeway.rb

@@ -2,7 +2,7 @@
 
 require 'English'
 
-# Janeway jsonpath parsing library
+# Janeway JSONPath parsing library
 module Janeway
   # Abstract Syntax Tree
   module AST
@@ -11,17 +11,17 @@ module Janeway
     INTEGER_MAX = 9_007_199_254_740_991
   end
 
-  # Apply a JsonPath query to the input, and return the result.
+  # Apply a JSONPath query to the input, and return all matched values.
   #
-  # @param query [String] jsonpath query
-  # @param input [Object] ruby object to be searched
+  # @param query [String] JSONPath query
+  # @param input [Hash, Array] ruby object to be searched
   # @return [Array] all matched objects
   def self.find_all(query, input)
     ast = compile(query)
     Janeway::Interpreter.new(ast).interpret(input)
   end
 
-  # Compile a JsonPath query into an Abstract Syntax Tree.
+  # Compile a JSONPath query into an Abstract Syntax Tree.
   #
   # This can be used and re-used later on multiple inputs.
   #
@@ -30,6 +30,26 @@ module Janeway
   def self.compile(query)
     Janeway::Parser.parse(query)
   end
+
+  # Iterate through each value matched by the JSONPath query.
+  #
+  # @param query [String] jsonpath query
+  # @param input [Hash, Array] ruby object to be searched
+  # @yieldparam [Object] matched value
+  # @return [void]
+  def self.each(query, input, &block)
+    raise ArgumentError, "Invalid jsonpath query: #{query.inspect}" unless query.is_a?(String)
+    unless [Hash, Array, String].include?(input.class)
+      raise ArgumentError, "Invalid input, expecting array or hash: #{input.inspect}"
+    end
+    return enum_for(:each, query, input) unless block_given?
+
+    ast = Janeway::Parser.parse(query)
+    interpreter = Janeway::Interpreter.new(ast)
+    yielder = Janeway::Interpreters::Yielder.new(&block)
+    interpreter.push(yielder)
+    interpreter.interpret(input)
+  end
 end
 
 # Require ruby source files in the given dir. Do not recurse to subdirs.

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: janeway-jsonpath
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Fraser Hanson
 bindir: bin
 cert_chain: []
-date: 2025-01-13 00:00:00.000000000 Z
+date: 2025-01-22 00:00:00.000000000 Z
 dependencies: []
 description: |+
   JSONPath is a query language for selecting and extracting values from a JSON text.
@@ -42,7 +42,6 @@ files:
 - lib/janeway/ast/filter_selector.rb
 - lib/janeway/ast/function.rb
 - lib/janeway/ast/helpers.rb
-- lib/janeway/ast/identifier.rb
 - lib/janeway/ast/index_selector.rb
 - lib/janeway/ast/name_selector.rb
 - lib/janeway/ast/null.rb
@@ -61,6 +60,21 @@ files:
 - lib/janeway/functions/search.rb
 - lib/janeway/functions/value.rb
 - lib/janeway/interpreter.rb
+- lib/janeway/interpreters/array_slice_selector_interpreter.rb
+- lib/janeway/interpreters/base.rb
+- lib/janeway/interpreters/binary_operator_interpreter.rb
+- lib/janeway/interpreters/child_segment_interpreter.rb
+- lib/janeway/interpreters/current_node_interpreter.rb
+- lib/janeway/interpreters/descendant_segment_interpreter.rb
+- lib/janeway/interpreters/filter_selector_interpreter.rb
+- lib/janeway/interpreters/function_interpreter.rb
+- lib/janeway/interpreters/index_selector_interpreter.rb
+- lib/janeway/interpreters/name_selector_interpreter.rb
+- lib/janeway/interpreters/root_node_interpreter.rb
+- lib/janeway/interpreters/tree_constructor.rb
+- lib/janeway/interpreters/unary_operator_interpreter.rb
+- lib/janeway/interpreters/wildcard_selector_interpreter.rb
+- lib/janeway/interpreters/yielder.rb
 - lib/janeway/lexer.rb
 - lib/janeway/location.rb
 - lib/janeway/parser.rb

data/lib/janeway/ast/identifier.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Janeway
-  module AST
-    class Identifier < Janeway::AST::Expression
-      alias name value
-
-      # Tokens which may follow an identifier string
-      EXPECTED_NEXT_TOKENS = %I[
-        \n
-        +
-        -
-        *
-        /
-        ==
-        !=
-        >
-        <
-        >=
-        <=
-        &&
-        ||
-      ].freeze
-
-      # @return [String]
-      def to_s
-        @value
-      end
-
-      def expects?(next_token)
-        EXPECTED_NEXT_TOKENS.include?(next_token)
-      end
-    end
-  end
-end