janeway-jsonpath 0.3.0 → 0.4.0

data/lib/janeway/interpreters/function_interpreter.rb

@@ -18,15 +18,17 @@ module Janeway
         value: [:nodes_type],
       }.freeze
 
-      # @param [AST::Function]
+      # @param function [AST::Function]
       def initialize(function)
         super
         @params = function.parameters.map { |param| TreeConstructor.ast_node_to_interpreter(param) }
       end
 
       # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
       # @param root [Array, Hash] the entire input
-      def interpret(input, root)
+      # @param _path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, root, _path)
         params = interpret_function_parameters(@params, input, root)
         function.body.call(*params)
       end
@@ -40,8 +42,8 @@ module Janeway
       # @see https://www.rfc-editor.org/rfc/rfc9535.html#name-well-typedness-of-function-
       #
       # @param parameters [Array] parameters before evaluation
-      # @param func [String] function name (eg. "length", "count")
       # @param input [Object]
+      # @param root [Array, Hash] the entire input
       # @return [Array] parameters after evaluation
       def interpret_function_parameters(parameters, input, root)
         param_types = FUNCTION_PARAMETER_TYPES[function.name.to_sym]
@@ -50,13 +52,13 @@ module Janeway
           type = param_types[i]
           case parameter.node
           when AST::CurrentNode, AST::RootNode
-            result = parameter.interpret(input, root)
+            result = parameter.interpret(input, nil, root, [])
             if type == :value_type && parameter.node.singular_query?
               deconstruct(result)
             else
               result
             end
-          when AST::Function, AST::StringType then parameter.interpret(input, root)
+          when AST::Function, AST::StringType then parameter.interpret(input, nil, root, [])
           when String then parameter.value # from a TreeConstructor::Literal
           else
             # invalid parameter type. Function must accept it and return Nothing result

data/lib/janeway/interpreters/index_selector_deleter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'name_selector_interpreter'
+
+module Janeway
+  module Interpreters
+    # Interprets an index selector, and deletes the matched value.
+    class IndexSelectorDeleter < IndexSelectorInterpreter
+      # Interpret selector on the given input.
+      # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
+      # @param _root [Array, Hash] the entire input
+      # @param _path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, _root, _path)
+        return [] unless input.is_a?(Array)
+
+        index = selector.value
+        result = input.fetch(index) # raises IndexError if no such index
+        input.delete_at(index) # returns nil if deleted value is nil, or if no value was deleted
+        [result]
+      rescue IndexError
+        []
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/index_selector_interpreter.rb

@@ -24,13 +24,14 @@ module Janeway
       #
       # @param input [Array, Hash] the results of processing so far
       # @param root [Array, Hash] the entire input
-      def interpret(input, root)
+      # @param path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, root, path)
         return [] unless input.is_a?(Array)
 
         result = input.fetch(selector.value) # raises IndexError if no such index
         return [result] unless @next
 
-        @next.interpret(result, root)
+        @next.interpret(result, input, root, path + [selector.value])
       rescue IndexError
         []
       end

data/lib/janeway/interpreters/name_selector_deleter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative 'name_selector_interpreter'
+
+module Janeway
+  module Interpreters
+    # Interprets a name selector, and deletes the matched values.
+    class NameSelectorDeleter < NameSelectorInterpreter
+      # Interpret selector on the given input.
+      # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
+      # @param _root [Array, Hash] the entire input
+      # @param _path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, _root, _path)
+        return [] unless input.is_a?(Hash) && input.key?(name)
+
+        [input.delete(name)]
+      end
+
+      # Return hash representation of this interpreter
+      # @return [Hash]
+      def as_json
+        { type: type, value: name, next: @next&.as_json }
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/name_selector_interpreter.rb

@@ -12,18 +12,33 @@ module Janeway
     # and a key that does not exist ([])
     class NameSelectorInterpreter < Base
       alias selector node
+      attr_reader :name
+
+      # @param selector [AST::NameSelector]
+      def initialize(selector)
+        super
+        @name = selector.name
+      end
 
       # Interpret selector on the given input.
       # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
       # @param root [Array, Hash] the entire input
-      def interpret(input, root)
-        return [] unless input.is_a?(Hash) && input.key?(selector.name)
+      # @param path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, root, path)
+        return [] unless input.is_a?(Hash) && input.key?(name)
 
-        result = input[selector.name]
+        result = input[name]
         return [result] unless @next
 
         # Forward result to next selector
-        @next.interpret(result, root)
+        @next.interpret(result, input, root, path + [name])
+      end
+
+      # Return hash representation of this interpreter
+      # @return [Hash]
+      def as_json
+        { type: type, value: name, next: @next&.as_json }
       end
     end
   end

data/lib/janeway/interpreters/root_node_deleter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'wildcard_selector_interpreter'
+
+module Janeway
+  module Interpreters
+    # Interprets the root node for deletion
+    class RootNodeDeleter < RootNodeInterpreter
+      # Delete all values from the root node.
+      #
+      # TODO: unclear, for deletion is there a difference between queries '$' and '$.*'?
+      #
+      # @param _input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
+      # @param root [Array, Hash] the entire input
+      # @param _path [Array<String>] elements of normalized path to the current input
+      # @return [Array] deleted elements
+      def interpret(_input, _parent, root, _path)
+        case root
+        when Array
+          results = root.dup
+          root.clear
+          results
+        when Hash
+          results = root.values
+          root.clear
+          results
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/root_node_interpreter.rb

@@ -12,11 +12,13 @@ module Janeway
       # Start an expression chain using the entire, unfiltered input.
       #
       # @param _input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
       # @param root [Array, Hash] the entire input
-      def interpret(_input, root)
+      # @param _path [Array<String>] elements of normalized path to the current input
+      def interpret(_input, _parent, root, _path = nil)
         return [root] unless @next
 
-        @next.interpret(root, root)
+        @next.interpret(root, nil, root, ['$'])
       end
     end
   end

data/lib/janeway/interpreters/tree_constructor.rb

@@ -4,7 +4,10 @@ require_relative 'base'
 
 module Janeway
   module Interpreters
-    # Constructs a tree of interpreter objects
+    # Constructs a tree of interpreter objects.
+    #
+    # It is used when an iterator method such as #each has been called to
+    # construct the chain of interpreter objects to handle the query.
     module TreeConstructor
       # Fake interpreter which just returns the given value
       Literal = Struct.new(:value) do
@@ -34,6 +37,22 @@ module Janeway
           raise "Unknown AST expression: #{expr.inspect}"
         end
       end
+
+      def self.ast_node_to_deleter(expr)
+        case expr
+        when AST::IndexSelector then IndexSelectorDeleter.new(expr)
+        when AST::ArraySliceSelector then ArraySliceSelectorDeleter.new(expr)
+        when AST::NameSelector then NameSelectorDeleter.new(expr)
+        when AST::FilterSelector then FilterSelectorDeleter.new(expr)
+        when AST::WildcardSelector then WildcardSelectorDeleter.new(expr)
+        when AST::ChildSegment then ChildSegmentDeleter.new(expr)
+        when AST::RootNode then RootNodeDeleter.new(expr)
+
+        when nil then nil # caller has no @next node
+        else
+          raise "Unknown AST expression: #{expr.inspect}"
+        end
+      end
     end
   end
 end

data/lib/janeway/interpreters/unary_operator_interpreter.rb

@@ -18,8 +18,8 @@ module Janeway
       end
 
       # @return [Boolean]
-      def interpret(input, root)
-        result = @operand.interpret(input, root)
+      def interpret(input, parent, root, _path)
+        result = @operand.interpret(input, parent, root, [])
         case result
         when Array then result.empty?
         when TrueClass, FalseClass then !result

data/lib/janeway/interpreters/wildcard_selector_deleter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'wildcard_selector_interpreter'
+
+module Janeway
+  module Interpreters
+    # Interprets a wildcard selector, and deletes the results.
+    class WildcardSelectorDeleter < WildcardSelectorInterpreter
+      # Delete all elements from the input
+      #
+      # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
+      # @param _root [Array, Hash] the entire input
+      # @param _path [Array<String>] elements of normalized path to the current input
+      # @return [Array] deleted elements
+      def interpret(input, _parent, _root, _path)
+        case input
+        when Array
+          results = input.dup
+          input.clear
+          results
+        when Hash
+          results = input.values
+          input.clear
+          results
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/janeway/interpreters/wildcard_selector_interpreter.rb

@@ -14,22 +14,37 @@ module Janeway
       # For anything else, return empty list.
       #
       # @param input [Array, Hash] the results of processing so far
+      # @param _parent [Array, Hash] parent of the input object
       # @param root [Array, Hash] the entire input
-      def interpret(input, root)
-        values =
-          case input
-          when Array then input
-          when Hash then input.values
-          else []
-          end
+      # @param path [Array<String>] elements of normalized path to the current input
+      def interpret(input, _parent, root, path)
+        case input
+        when Array then interpret_array(input, root, path)
+        when Hash then interpret_hash(input, root, path)
+        else []
+        end
+      end
+
+      def interpret_hash(input, root, path)
+        return [] if input.empty? # early exit, no need for further processing on empty list
+        return input.values unless @next
+
+        # Apply child selector to each node in the output node list
+        results = []
+        input.each do |key, value|
+          results.concat @next.interpret(value, input, root, path + [key])
+        end
+        results
+      end
 
-        return values if values.empty? # early exit, no need for further processing on empty list
-        return values unless @next
+      def interpret_array(input, root, path)
+        return input if input.empty? # early exit, no need for further processing on empty list
+        return input unless @next
 
         # Apply child selector to each node in the output node list
         results = []
-        values.each do |value|
-          results.concat @next.interpret(value, root)
+        input.each_with_index do |value, i|
+          results.concat @next.interpret(value, input, root, path + [i])
         end
         results
       end

data/lib/janeway/interpreters/yielder.rb

@@ -1,34 +1,72 @@
 # frozen_string_literal: true
 
 require_relative 'base'
+require_relative '../normalized_path'
 
 module Janeway
   module Interpreters
     # Yields each input value.
-    # It is inserted at the end of the "real" selectors in the AST, to receive and yield the output.
-    # This is a supporting class for the supports the Janeway.each method.
     #
-    # This will get pushed onto the end of the query AST.
-    # Currently it must act like both an AST node and an interpreter.
-    # This will be simpler if TODO some day may the interpreter subclass methods can be merged into the AST classes
-    class Yielder < Base
-      # The implicit constructor forwards a closure to the base class constructor.
-      # Base class constructor stores it in @node.
+    # It is inserted at the end of the "real" selectors in the AST, to receive and yield the output.
+    # This is a supporting class for the Janeway.each method.
+    class Yielder
       def initialize(&block)
-        super(Struct.new(:next).new)
         @block = block
+
+        # Decide how many parameters to yield to this block.
+        # block.arity is -1 when no block was given, and an enumerator is being returned from #each
+        @yield_to_block =
+          if block.arity.negative?
+            # Yield values only to an enumerator.
+            proc { |value, _parent, _path| @block.call(value) }
+          elsif block.arity > 3
+            # Only do the work of constructing the normalized path when it is actually used
+            proc { |value, parent, path| @block.call(value, parent, path.last, normalized_path(path)) }
+          else
+            # block arity is 1, 2 or 3. Send all 3.
+            proc { |value, parent, path| @block.call(value, parent, path.last) }
+          end
       end
 
       # Yield each input value
       #
       # @param input [Array, Hash] the results of processing so far
+      # @param parent [Array, Hash] parent of the input object
       # @param _root [Array, Hash] the entire input
+      # @param path [Array<String, Integer>] components of normalized path to the current input
       # @yieldparam [Object] matched value
-      # @return [Object] input
-      def interpret(input, _root)
-        @block.call(input)
+      # @return [Object] input as node list
+      def interpret(input, parent, _root, path)
+        @yield_to_block.call(input, parent, path)
         input.is_a?(Array) ? input : [input]
       end
+
+      # Convert the list of path elements into a normalized query string.
+      #
+      # This form uses a subset of jsonpath that unambiguously points to a value
+      # using only name and index selectors.
+      # @see https://www.rfc-editor.org/rfc/rfc9535.html#name-normalized-paths
+      #
+      # Name selectors must use bracket notation, not shorthand.
+      #
+      # @param components [Array<String, Integer>]
+      # @return [String]
+      def normalized_path(components)
+        # First component is the root identifer, the remaining components are
+        # all index selectors or name selectors.
+        # Handle the root identifier separately, because .normalize does not handle those.
+        '$' + components[1..].map { NormalizedPath.normalize(_1) }.join
+      end
+
+      # Dummy method from Interpreters::Base, allow child segment interpreter to disable the
+      # non-exist 'next' link.
+      # @return [void]
+      def next=(*); end
+
+      # @return [Hash]
+      def as_json
+        { type: self.class.to_s.split('::').last }
+      end
     end
   end
 end

data/lib/janeway/lexer.rb

@@ -170,7 +170,7 @@ module Janeway
     def lex_delimited_string(delimiter)
       allowed_delimiters = %w[' "]
       # the "other" delimiter char, which is not currently being treated as a delimiter
-      non_delimiter = allowed_delimiters.reject { _1 == delimiter }.first
+      non_delimiter = allowed_delimiters.reject { |char| char == delimiter }.first
 
       literal_chars = []
       while lookahead != delimiter && source_uncompleted?

data/lib/janeway/normalized_path.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Janeway
+  # Converts index and name selector values to normalized path components.
+  #
+  # This implements the normalized path description in RFC 9535:
+  # @see https://www.rfc-editor.org/rfc/rfc9535.html#name-normalized-paths
+  #
+  # This does a lot of escaping, and much of it is the inverse of Lexer code that does un-escaping.
+  module NormalizedPath
+    # Characters that do not need escaping, defined by hexadecimal range
+    NORMAL_UNESCAPED_RANGES = [(0x20..0x26), (0x28..0x5B), (0x5D..0xD7FF), (0xE000..0x10FFFF)].freeze
+
+    def self.normalize(value)
+      case value
+      when String then normalize_name(value)
+      when Integer then normalize_index(value)
+      else
+        raise "Cannot normalize #{value.inspect}"
+      end
+    end
+
+    # @param index [Integer] index selector value
+    # @return [String] eg. "[1]"
+    def self.normalize_index(index)
+      "[#{index}]"
+    end
+
+    # @param name [Integer] name selector value
+    # @return [String] eg. "['']"
+    def self.normalize_name(name)
+      "['#{escape(name)}']"
+    end
+
+    def self.escape(str)
+      # Common case, all chars are normal.
+      return str if str.chars.all? { |char| NORMAL_UNESCAPED_RANGES.any? { |range| range.include?(char.ord) } }
+
+      # Some escaping must be done
+      str.chars.map { |char| escape_char(char) }.join
+    end
+
+    # Escape or hex-encode the given character
+    # @param char [String] single character, possibly multi-byte
+    # @return [String]
+    def self.escape_char(char)
+      # Character ranges defined by https://www.rfc-editor.org/rfc/rfc9535.html#section-2.7-8
+      case char.ord
+      when 0x20..0x26, 0x28..0x5B, 0x5D..0xD7FF, 0xE000..0x10FFFF # normal-unescaped range
+        char # unescaped
+      when 0x62, 0x66, 0x6E, 0x72, 0x74, 0x27, 0x5C # normal-escapable range
+        # backspace, form feed, line feed, carriage return, horizontal tab, apostrophe, backslash
+        "\\#{char}" # escaped
+      else # normal-hexchar range
+        hex_encode_char(char)
+      end
+    end
+
+    # Hex-encode the given character
+    # @param char [String] single character, possibly multi-byte
+    # @return [String]
+    def self.hex_encode_char(char)
+      format('\\u00%02x', char.ord)
+    end
+  end
+end

data/lib/janeway/parser.rb

@@ -29,7 +29,7 @@ module Janeway
     }.freeze
 
     # @param jsonpath [String] jsonpath query to be lexed and parsed
-    # @return [AST::Query]
+    # @return [Query]
     def self.parse(jsonpath)
       raise ArgumentError, "expect jsonpath string, got #{jsonpath.inspect}" unless jsonpath.is_a?(String)
 
@@ -46,7 +46,7 @@ module Janeway
     end
 
     # Parse the token list and create an Abstract Syntax Tree
-    # @return [AST::Query]
+    # @return [Query]
     def parse
       consume
       raise err('JsonPath queries must start with root identifier "$"') unless current.type == :root
@@ -58,7 +58,7 @@ module Janeway
         raise err("Unrecognized expressions after query: #{remaining}")
       end
 
-      AST::Query.new(root_node, @jsonpath)
+      Query.new(root_node, @jsonpath)
     end
 
     private

data/lib/janeway/query.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Janeway
+  # Query holds the abstract syntax tree created by parsing the query.
+  # This can be frozen and passed to multiple threads or ractors for simultaneous use.
+  # No instance members are modified during the interpretation stage.
+  class Query
+    # The original jsonpath query, for use in error messages
+    # @return [String]
+    attr_reader :jsonpath
+
+    # @return [AST::Root]
+    attr_reader :root
+
+    # @param root_node [AST::Root]
+    # @param jsonpath [String]
+    def initialize(root_node, jsonpath)
+      raise ArgumentError, "expect root identifier, got #{root_node.inspect}" unless root_node.is_a?(AST::RootNode)
+      raise ArgumentError, "expect query string, got #{jsonpath.inspect}" unless jsonpath.is_a?(String)
+
+      @root = root_node
+      @jsonpath = jsonpath
+    end
+
+    # Combine this query with input to make an Enumerator.
+    # This can be used to iterate over results with #each, #map, etc.
+    #
+    # @return [Janeway::Enumerator]
+    def on(input)
+      Janeway::Enumerator.new(self, input)
+    end
+
+    def to_s
+      @root.to_s
+    end
+
+    # Return a list of the nodes in the AST.
+    # The AST of a jsonpath query is a straight line, so this is expressible as an array.
+    # The only part of the AST with branches is inside a filter selector, but that doesn't show up here.
+    # @return [Array<Expression>]
+    def node_list
+      nodes = []
+      node = @root
+      loop do
+        nodes << node
+        break unless node.next
+
+        node = node.next
+      end
+      nodes
+    end
+
+    # Queries are considered equal if their ASTs evaluate to the same JSONPath string.
+    #
+    # The string output is generated from the AST and should be considered a "normalized"
+    # form of the query. It may have different whitespace and parentheses than the original
+    # input but will be semantically equivalent.
+    def ==(other)
+      to_s == other.to_s
+    end
+
+    # Print AST in tree format
+    # Every AST class prints a 1-line representation of self, with children on separate lines
+    def tree
+      result = @root.tree(0)
+
+      result.flatten.join("\n")
+    end
+  end
+end

data/lib/janeway/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Janeway
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/janeway.rb

@@ -11,45 +11,33 @@ module Janeway
     INTEGER_MAX = 9_007_199_254_740_991
   end
 
-  # Apply a JSONPath query to the input, and return all matched values.
+  # Pair a jsonpath query with data to make an enumerator.
+  # This can be used to apply the query to the data using Enumerator module
+  # methods such as #each and #map.
   #
-  # @param query [String] JSONPath query
-  # @param input [Hash, Array] ruby object to be searched
-  # @return [Array] all matched objects
-  def self.find_all(query, input)
-    ast = compile(query)
-    Janeway::Interpreter.new(ast).interpret(input)
+  # @param jsonpath [String] jsonpath query
+  # @param data [Array, Hash] input data
+  # @return [Janeway::Enumerator]
+  def self.enum_for(jsonpath, data)
+    query = compile(jsonpath)
+    Janeway::Enumerator.new(query, data)
   end
 
   # Compile a JSONPath query into an Abstract Syntax Tree.
   #
-  # This can be used and re-used later on multiple inputs.
+  # This can be combined with inputs (using #on) to create Enumerators.
+  # @example
+  #     query = Janeway.compile('$.store.books[? length(@.title) > 20]')
+  #     long_title_books = query.on(local_json).search
+  #     query.on(remote_json).each do |book|
+  #       long_title_books << book
+  #     end
   #
   # @param query [String] jsonpath query
   # @return [Janeway::AST::Query]
   def self.compile(query)
     Janeway::Parser.parse(query)
   end
-
-  # Iterate through each value matched by the JSONPath query.
-  #
-  # @param query [String] jsonpath query
-  # @param input [Hash, Array] ruby object to be searched
-  # @yieldparam [Object] matched value
-  # @return [void]
-  def self.each(query, input, &block)
-    raise ArgumentError, "Invalid jsonpath query: #{query.inspect}" unless query.is_a?(String)
-    unless [Hash, Array, String].include?(input.class)
-      raise ArgumentError, "Invalid input, expecting array or hash: #{input.inspect}"
-    end
-    return enum_for(:each, query, input) unless block_given?
-
-    ast = Janeway::Parser.parse(query)
-    interpreter = Janeway::Interpreter.new(ast)
-    yielder = Janeway::Interpreters::Yielder.new(&block)
-    interpreter.push(yielder)
-    interpreter.interpret(input)
-  end
 end
 
 # Require ruby source files in the given dir. Do not recurse to subdirs.