jmespath 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ee5e74d561de8c291c5ff6dbb805638edf8a20a
-  data.tar.gz: 73efab83f2a20e968b53f4d67dfb9bb95195b7a0
+  metadata.gz: d772ec4a8596343c31e4c9a6be128f965a6f46e8
+  data.tar.gz: 08dcbd47adb5a32f0d0b27a3ab161af48bd068c7
 SHA512:
-  metadata.gz: 242d19aae8a4a56715e0a2c6faad19a1a204627efba6e72ac71385c9f5fbc3f46d431b116119283f1753776d9cafd706d2c205342fabd34deb68938d00ae56b5
-  data.tar.gz: 0b29709b7df66280b5727689f2cc1bf35635fab5ddd8df885553d9a2d765000b655845f5816f01e0317e6f5ca189d08531f6fdd1131170274afb71997a911170
+  metadata.gz: bd3806fd6b131304b5199d871cbcfc92d0a5cae5b8c6fe551899a0a076d4868c0a3fb02a7583c19b42d2d0ed81e9ca0b806e4adb18a78c498133104f7b755999
+  data.tar.gz: c7fd708540946c3ed5697478ee114f7942ea0d933329d0f01f9ba326cba7784c40f41431749735c72702e80978d0ee3e1a2902aa24395f2bf9f1d68b3c09c722

data/lib/jmespath.rb

@@ -1,4 +1,5 @@
-require 'multi_json'
+require 'json'
+require 'stringio'
 require 'pathname'
 
 module JMESPath
@@ -7,11 +8,11 @@ module JMESPath
   autoload :Errors, 'jmespath/errors'
   autoload :ExprNode, 'jmespath/expr_node'
   autoload :Lexer, 'jmespath/lexer'
+  autoload :Nodes, 'jmespath/nodes'
   autoload :Parser, 'jmespath/parser'
   autoload :Runtime, 'jmespath/runtime'
   autoload :Token, 'jmespath/token'
   autoload :TokenStream, 'jmespath/token_stream'
-  autoload :TreeInterpreter, 'jmespath/tree_interpreter'
   autoload :VERSION, 'jmespath/version'
 
   class << self
@@ -26,7 +27,7 @@ module JMESPath
       data = case data
         when Hash, Struct then data # check for most common case first
         when Pathname then load_json(data)
-        when IO, StringIO then MultiJson.load(data.read)
+        when IO, StringIO then JSON.load(data.read)
         else data
         end
       Runtime.new.search(expression, data)
@@ -34,7 +35,7 @@ module JMESPath
 
     # @api private
     def load_json(path)
-      MultiJson.load(File.open(path, 'r', encoding: 'UTF-8') { |f| f.read })
+      JSON.load(File.open(path, 'r', encoding: 'UTF-8') { |f| f.read })
     end
 
   end

data/lib/jmespath/errors.rb

@@ -9,6 +9,8 @@ module JMESPath
 
     class InvalidTypeError < Error; end
 
+    class InvalidValueError < Error; end
+
     class InvalidArityError < Error; end
 
     class UnknownFunctionError < Error; end

data/lib/jmespath/lexer.rb

@@ -1,116 +1,323 @@
+require 'json'
+require 'set'
+
 module JMESPath
   # @api private
   class Lexer
 
-    # @api private
-    TOKEN_PATTERNS = {}
-
-    # @api private
-    TOKEN_TYPES = {}
-
-    {
-      '[a-zA-Z_][a-zA-Z_0-9]*'     => :identifier,
-      '\.'                         => :dot,
-      '\*'                         => :star,
-      '\[\]'                       => :flatten,
-      '-?\d+'                      => :number,
-      '\|\|'                       => :or,
-      '\|'                         => :pipe,
-      '\[\?'                       => :filter,
-      '\['                         => :lbracket,
-      '\]'                         => :rbracket,
-      '"(?:\\\\\\\\|\\\\"|[^"])*"' => :quoted_identifier,
-      '`(?:\\\\\\\\|\\\\`|[^`])*`' => :literal,
-      ','                          => :comma,
-      ':'                          => :colon,
-      '@'                          => :current,
-      '&'                          => :expref,
-      '\('                         => :lparen,
-      '\)'                         => :rparen,
-      '\{'                         => :lbrace,
-      '\}'                         => :rbrace,
-      '!='                         => :comparator,
-      '=='                         => :comparator,
-      '<='                         => :comparator,
-      '>='                         => :comparator,
-      '<'                          => :comparator,
-      '>'                          => :comparator,
-      '[ \t]'                      => :skip,
-    }.each.with_index do |(pattern, type), n|
-      TOKEN_PATTERNS[n] = pattern
-      TOKEN_TYPES[n] = type
-    end
+    T_DOT = :dot
+    T_STAR = :star
+    T_COMMA = :comma
+    T_COLON = :colon
+    T_CURRENT = :current
+    T_EXPREF = :expref
+    T_LPAREN = :lparen
+    T_RPAREN = :rparen
+    T_LBRACE = :lbrace
+    T_RBRACE = :rbrace
+    T_LBRACKET = :lbracket
+    T_RBRACKET = :rbracket
+    T_FLATTEN = :flatten
+    T_IDENTIFIER = :identifier
+    T_NUMBER = :number
+    T_QUOTED_IDENTIFIER = :quoted_identifier
+    T_UNKNOWN = :unknown
+    T_PIPE = :pipe
+    T_OR = :or
+    T_FILTER = :filter
+    T_LITERAL = :literal
+    T_EOF = :eof
+    T_COMPARATOR = :comparator
+
+    STATE_IDENTIFIER = 0
+    STATE_NUMBER = 1
+    STATE_SINGLE_CHAR = 2
+    STATE_WHITESPACE = 3
+    STATE_STRING_LITERAL = 4
+    STATE_QUOTED_STRING = 5
+    STATE_JSON_LITERAL = 6
+    STATE_LBRACKET = 7
+    STATE_PIPE = 8
+    STATE_LT = 9
+    STATE_GT = 10
+    STATE_EQ = 11
+    STATE_NOT = 12
 
-    # @api private
-    TOKEN_REGEX = /(#{TOKEN_PATTERNS.values.join(')|(')})/
+    TRANSLATION_TABLE = {
+      '<'  => STATE_LT,
+      '>'  => STATE_GT,
+      '='  => STATE_EQ,
+      '!'  => STATE_NOT,
+      '['  => STATE_LBRACKET,
+      '|'  => STATE_PIPE,
+      '`'  => STATE_JSON_LITERAL,
+      '"'  => STATE_QUOTED_STRING,
+      "'"  => STATE_STRING_LITERAL,
+      '-'  => STATE_NUMBER,
+      '0'  => STATE_NUMBER,
+      '1'  => STATE_NUMBER,
+      '2'  => STATE_NUMBER,
+      '3'  => STATE_NUMBER,
+      '4'  => STATE_NUMBER,
+      '5'  => STATE_NUMBER,
+      '6'  => STATE_NUMBER,
+      '7'  => STATE_NUMBER,
+      '8'  => STATE_NUMBER,
+      '9'  => STATE_NUMBER,
+      ' '  => STATE_WHITESPACE,
+      "\t" => STATE_WHITESPACE,
+      "\n" => STATE_WHITESPACE,
+      "\r" => STATE_WHITESPACE,
+      '.'  => STATE_SINGLE_CHAR,
+      '*'  => STATE_SINGLE_CHAR,
+      ']'  => STATE_SINGLE_CHAR,
+      ','  => STATE_SINGLE_CHAR,
+      ':'  => STATE_SINGLE_CHAR,
+      '@'  => STATE_SINGLE_CHAR,
+      '&'  => STATE_SINGLE_CHAR,
+      '('  => STATE_SINGLE_CHAR,
+      ')'  => STATE_SINGLE_CHAR,
+      '{'  => STATE_SINGLE_CHAR,
+      '}'  => STATE_SINGLE_CHAR,
+      '_'  => STATE_IDENTIFIER,
+      'A'  => STATE_IDENTIFIER,
+      'B'  => STATE_IDENTIFIER,
+      'C'  => STATE_IDENTIFIER,
+      'D'  => STATE_IDENTIFIER,
+      'E'  => STATE_IDENTIFIER,
+      'F'  => STATE_IDENTIFIER,
+      'G'  => STATE_IDENTIFIER,
+      'H'  => STATE_IDENTIFIER,
+      'I'  => STATE_IDENTIFIER,
+      'J'  => STATE_IDENTIFIER,
+      'K'  => STATE_IDENTIFIER,
+      'L'  => STATE_IDENTIFIER,
+      'M'  => STATE_IDENTIFIER,
+      'N'  => STATE_IDENTIFIER,
+      'O'  => STATE_IDENTIFIER,
+      'P'  => STATE_IDENTIFIER,
+      'Q'  => STATE_IDENTIFIER,
+      'R'  => STATE_IDENTIFIER,
+      'S'  => STATE_IDENTIFIER,
+      'T'  => STATE_IDENTIFIER,
+      'U'  => STATE_IDENTIFIER,
+      'V'  => STATE_IDENTIFIER,
+      'W'  => STATE_IDENTIFIER,
+      'X'  => STATE_IDENTIFIER,
+      'Y'  => STATE_IDENTIFIER,
+      'Z'  => STATE_IDENTIFIER,
+      'a'  => STATE_IDENTIFIER,
+      'b'  => STATE_IDENTIFIER,
+      'c'  => STATE_IDENTIFIER,
+      'd'  => STATE_IDENTIFIER,
+      'e'  => STATE_IDENTIFIER,
+      'f'  => STATE_IDENTIFIER,
+      'g'  => STATE_IDENTIFIER,
+      'h'  => STATE_IDENTIFIER,
+      'i'  => STATE_IDENTIFIER,
+      'j'  => STATE_IDENTIFIER,
+      'k'  => STATE_IDENTIFIER,
+      'l'  => STATE_IDENTIFIER,
+      'm'  => STATE_IDENTIFIER,
+      'n'  => STATE_IDENTIFIER,
+      'o'  => STATE_IDENTIFIER,
+      'p'  => STATE_IDENTIFIER,
+      'q'  => STATE_IDENTIFIER,
+      'r'  => STATE_IDENTIFIER,
+      's'  => STATE_IDENTIFIER,
+      't'  => STATE_IDENTIFIER,
+      'u'  => STATE_IDENTIFIER,
+      'v'  => STATE_IDENTIFIER,
+      'w'  => STATE_IDENTIFIER,
+      'x'  => STATE_IDENTIFIER,
+      'y'  => STATE_IDENTIFIER,
+      'z'  => STATE_IDENTIFIER,
+    }
 
-    # @api private
-    JSON_VALUE = /^[\["{]/
+    VALID_IDENTIFIERS = Set.new(%w(
+      A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+      a b c d e f g h i j k l m n o p q r s t u v w x y z
+      _ 0 1 2 3 4 5 6 7 8 9
+    ))
 
-    # @api private
-    JSON_NUMBER = /^\-?[0-9]*(\.[0-9]+)?([e|E][+|\-][0-9]+)?$/
+    NUMBERS = Set.new(%w(0 1 2 3 4 5 6 7 8 9))
+
+    SIMPLE_TOKENS = {
+      '.' => T_DOT,
+      '*' => T_STAR,
+      ']' => T_RBRACKET,
+      ',' => T_COMMA,
+      ':' => T_COLON,
+      '@' => T_CURRENT,
+      '&' => T_EXPREF,
+      '(' => T_LPAREN,
+      ')' => T_RPAREN,
+      '{' => T_LBRACE,
+      '}' => T_RBRACE,
+    }
 
     # @param [String<JMESPath>] expression
     # @return [Array<Hash>]
     def tokenize(expression)
-      offset = 0
+
       tokens = []
-      expression.scan(TOKEN_REGEX).each do |match|
-        match_index = match.find_index { |token| !token.nil? }
-        match_value = match[match_index]
-        type = TOKEN_TYPES[match_index]
-        token = Token.new(type, match_value, offset)
-        if token.type != :skip
-          case token.type
-          when :number then token_number(token, expression, offset)
-          when :literal then token_literal(token, expression, offset)
-          when :quoted_identifier
-            token_quoted_identifier(token, expression, offset)
+      chars = CharacterStream.new(expression.chars)
+
+      while chars.current
+        case TRANSLATION_TABLE[chars.current]
+        when nil
+          tokens << Token.new(
+            T_UNKNOWN,
+            chars.current,
+            chars.position
+          )
+          chars.next
+        when STATE_SINGLE_CHAR
+          # consume simple tokens like ".", ",", "@", etc.
+          tokens << Token.new(
+            SIMPLE_TOKENS[chars.current],
+            chars.current,
+            chars.position
+          )
+          chars.next
+        when STATE_IDENTIFIER
+          start = chars.position
+          buffer = []
+          begin
+            buffer << chars.current
+            chars.next
+          end while VALID_IDENTIFIERS.include?(chars.current)
+          tokens << Token.new(
+            T_IDENTIFIER,
+            buffer.join,
+            start
+          )
+        when STATE_WHITESPACE
+          # skip whitespace
+          chars.next
+        when STATE_LBRACKET
+          # consume "[", "[?" and "[]"
+          position = chars.position
+          actual = chars.next
+          if actual == ']'
+            chars.next
+            tokens << Token.new(T_FLATTEN, '[]', position)
+          elsif actual == '?'
+            chars.next
+            tokens << Token.new(T_FILTER, '[?', position)
+          else
+            tokens << Token.new(T_LBRACKET, '[',  position)
+          end
+        when STATE_STRING_LITERAL
+          # consume raw string literals
+          tokens << inside(chars, "'", T_LITERAL)
+        when STATE_PIPE
+          # consume pipe and OR
+          tokens << match_or(chars, '|', '|', T_OR, T_PIPE)
+        when STATE_JSON_LITERAL
+          # consume JSON literals
+          token = inside(chars, '`', T_LITERAL)
+          if token.type == T_LITERAL
+            token.value = token.value.gsub('\\`', '`')
+            token = parse_json(token)
+          end
+          tokens << token
+        when STATE_NUMBER
+          start = chars.position
+          buffer = []
+          begin
+            buffer << chars.current
+            chars.next
+          end while NUMBERS.include?(chars.current)
+          tokens << Token.new(
+            T_NUMBER,
+            buffer.join.to_i,
+            start
+          )
+        when STATE_QUOTED_STRING
+          # consume quoted identifiers
+          token = inside(chars, '"', T_QUOTED_IDENTIFIER)
+          if token.type == T_QUOTED_IDENTIFIER
+            token.value = "\"#{token.value}\""
+            token = parse_json(token)
           end
           tokens << token
+        when STATE_EQ
+          # consume equals
+          tokens << match_or(chars, '=', '=', T_COMPARATOR, T_UNKNOWN)
+        when STATE_NOT
+          # consume not equals
+          tokens << match_or(chars, '!', '=', T_COMPARATOR, T_UNKNOWN)
+        else
+          # either '<' or '>'
+          # consume less than and greater than
+          tokens << match_or(chars, chars.current, '=', T_COMPARATOR, T_COMPARATOR)
         end
-        offset += match_value.size
-      end
-      tokens << Token.new(:eof, nil, offset)
-      unless expression.size == offset
-        syntax_error('invalid expression', expression, offset) 
       end
+      tokens << Token.new(T_EOF, nil, chars.position)
       tokens
     end
 
     private
 
-    def token_number(token, expression, offset)
-      token[:value] = token[:value].to_i
+    def match_or(chars, current, expected, type, or_type)
+      if chars.next == expected
+        chars.next
+        Token.new(type, current + expected, chars.position - 1)
+      else
+        Token.new(or_type, current, chars.position - 1)
+      end
     end
 
-    def token_literal(token, expression, offset)
-      token[:value] = token[:value][1..-2].lstrip.gsub('\`', '`')
-      token[:value] =
-        case token[:value]
-        when 'true', 'false' then token[:value] == 'true'
-        when 'null' then nil
-        when '' then syntax_error("empty json literal", expression, offset)
-        when JSON_VALUE then decode_json(token[:value], expression, offset)
-        when JSON_NUMBER then decode_json(token[:value], expression, offset)
-        else decode_json('"' + token[:value] + '"', expression, offset)
+    def inside(chars, delim, type)
+      position = chars.position
+      current = chars.next
+      buffer = []
+      while current != delim
+        if current == '\\'
+          buffer << current
+          current = chars.next
+        end
+        if current.nil?
+          # unclosed delimiter
+          return Token.new(T_UNKNOWN, buffer.join, position)
         end
+        buffer << current
+        current = chars.next
+      end
+      chars.next
+      Token.new(type, buffer.join, position)
     end
 
-    def token_quoted_identifier(token, expression, offset)
-      token[:value] = decode_json(token[:value], expression, offset)
+    def parse_json(token)
+      begin
+        token.value = JSON.load(token.value)
+      rescue JSON::ParserError
+        token.type = T_UNKNOWN
+      end
+      token
     end
 
-    def decode_json(json, expression, offset)
-      MultiJson.load(json)
-    rescue MultiJson::ParseError => e
-      syntax_error(e.message, expression, offset)
-    end
+    class CharacterStream
 
-    def syntax_error(message, expression, offset)
-      msg = message + "in #{expression.inspect} at #{offset}"
-      raise Errors::SyntaxError.new(msg)
-    end
+      def initialize(chars)
+        @chars = chars
+        @position = 0
+      end
+
+      def current
+        @chars[@position]
+      end
+
+      def next
+        @position += 1
+        @chars[@position]
+      end
 
+      def position
+        @position
+      end
+
+    end
   end
 end