ruby_json_parser 0.1.0

data/lib/ruby_json_parser/lexer.rb

@@ -0,0 +1,358 @@
+# typed: strict
+# frozen_string_literal: true
+
+require_relative 'token'
+
+class RubyJsonParser
+  # A lexical analyzer (tokenizer) for JSON
+  class Lexer
+    extend T::Sig
+    extend T::Generic
+    include Enumerable
+
+    # Type parameter for `Enumerable`
+    # Declares the type that the lexer returns for tokens
+    Elem = type_member { { fixed: Token } }
+
+    class << self
+      extend T::Sig
+
+      sig { params(source: String).returns(T::Array[Token]) }
+      def lex(source)
+        new(source).to_a
+      end
+    end
+
+    sig { params(source: String).void }
+    def initialize(source)
+      @source = source
+
+      # offset of the first character of the current lexeme
+      @start_cursor = T.let(0, Integer)
+      # offset of the next character
+      @cursor = T.let(0, Integer)
+    end
+
+    sig { returns(Token) }
+    def next
+      return Token.new(Token::END_OF_FILE) unless more_tokens?
+
+      scan_token
+    end
+
+    sig { override.params(block: T.nilable(T.proc.params(arg0: Token).void)).returns(T.untyped) }
+    def each(&block)
+      return enum_for(T.must(__method__)) unless block
+
+      loop do
+        tok = self.next
+        break if tok.type == Token::END_OF_FILE
+
+        block.call(tok)
+      end
+
+      self
+    end
+
+    private
+
+    sig { returns(T::Boolean) }
+    def more_tokens?
+      @cursor < @source.length
+    end
+
+    sig { params(type: Symbol).returns(Token) }
+    def token_with_consumed_value(type)
+      token(type, token_value)
+    end
+
+    sig { params(type: Symbol, value: T.nilable(String)).returns(Token) }
+    def token(type, value = nil)
+      @start_cursor = @cursor
+      Token.new(type, value)
+    end
+
+    # Returns the current token value.
+    sig { returns(String) }
+    def token_value
+      T.must @source[@start_cursor...@cursor]
+    end
+
+    sig { returns([String, T::Boolean]) }
+    def advance_char
+      return '', false unless more_tokens?
+
+      char = next_char
+
+      @cursor += 1
+      [char, true]
+    end
+
+    sig { returns(String) }
+    def next_char
+      T.must @source[@cursor]
+    end
+
+    # Gets the next UTF-8 encoded character
+    # without incrementing the cursor.
+    sig { returns(String) }
+    def peek_char
+      return '' unless more_tokens?
+
+      char, = next_char
+      char
+    end
+
+    # Advance the next `n` characters
+    sig { params(n: Integer).returns(T::Boolean) }
+    def advance_chars(n)
+      n.times do
+        _, ok = advance_char
+        return false unless ok
+      end
+
+      true
+    end
+
+    # Checks if the given character matches
+    # the next UTF-8 encoded character in source code.
+    # If they match, the cursor gets incremented.
+    sig { params(char: String).returns(T::Boolean) }
+    def match_char(char)
+      return false unless more_tokens?
+
+      if peek_char == char
+        advance_char
+        return true
+      end
+
+      false
+    end
+
+    # Consumes the next character if it's from the valid set.
+    sig { params(valid_chars: String).returns(T::Boolean) }
+    def match_chars(valid_chars)
+      return false unless more_tokens?
+
+      p = peek_char
+      if p != '' && valid_chars.include?(p)
+        advance_char
+        return true
+      end
+
+      false
+    end
+
+    # Rewinds the cursor back n chars.
+    sig { params(n: Integer).void }
+    def backup_chars(n)
+      @cursor -= n
+    end
+
+    # Skips the current accumulated token.
+    sig { void }
+    def skip_token
+      @start_cursor = @cursor
+    end
+
+    sig { returns(Token) }
+    def scan_token
+      loop do
+        char, ok = advance_char
+        return token(Token::END_OF_FILE) unless ok
+
+        case char
+        when '['
+          return token(Token::LBRACKET)
+        when ']'
+          return token(Token::RBRACKET)
+        when '{'
+          return token(Token::LBRACE)
+        when '}'
+          return token(Token::RBRACE)
+        when ','
+          return token(Token::COMMA)
+        when ':'
+          return token(Token::COLON)
+        when '.'
+          return token(Token::DOT)
+        when '"'
+          return scan_string
+        when '-'
+          p = peek_char
+          return token(Token::ERROR, 'unexpected EOF') if p == ''
+          return token(Token::ERROR, "unexpected number char: `#{p}`") unless Token::DIGITS.include?(p)
+
+          char, = advance_char
+          return scan_number(char)
+        when ' ', "\n", "\r", "\t"
+          skip_token
+          next
+        else
+          if char.match?(/[[:alpha:]]/)
+            return scan_identifier
+          elsif char.match?(/\d/)
+            return scan_number(char)
+          end
+
+          return token(Token::ERROR, "unexpected char `#{char}`")
+        end
+      end
+    end
+
+    sig { params(char: String).returns(T::Boolean) }
+    def identifier_char?(char)
+      char.match?(/[[:alpha:][:digit:]_]/)
+    end
+
+    sig { returns(Token) }
+    def scan_identifier
+      advance_char while identifier_char?(peek_char)
+
+      value = token_value
+      return token(value.to_sym) if Token::KEYWORDS.include?(value)
+
+      token(Token::ERROR, "unexpected identifier: `#{value}`")
+    end
+
+    sig { void }
+    def consume_digits
+      loop do
+        p = peek_char
+        break if p == '' || !Token::DIGITS.include?(peek_char)
+
+        _, ok = advance_char
+        break unless ok
+      end
+    end
+
+    # Checks if the next `n` characters are from the valid set.
+    sig { params(valid_chars: String, n: Integer).returns(T::Boolean) }
+    def accept_chars(valid_chars, n)
+      result = T.let(true, T::Boolean)
+      n.times do
+        unless match_chars(valid_chars)
+          result = false
+          break
+        end
+      end
+
+      backup_chars(n)
+
+      result
+    end
+
+    sig { params(init_char: String).returns(Token) }
+    def scan_number(init_char)
+      if init_char == '0'
+        p = peek_char
+        if accept_chars(Token::DIGITS, 1)
+          consume_digits
+          return token(
+            Token::ERROR,
+            'illegal trailing zero in number literal',
+          )
+        end
+      end
+
+      consume_digits
+
+      if match_char('.')
+        p = peek_char
+        if p == ''
+          return token(
+            Token::ERROR,
+            'unexpected EOF',
+          )
+        end
+
+        unless Token::DIGITS.include?(p)
+          return token(
+            Token::ERROR,
+            "unexpected char in number literal: `#{p}`",
+          )
+        end
+
+        consume_digits
+      end
+
+      if match_char('e') || match_char('E')
+        match_char('+') || match_char('-')
+        p = peek_char
+        if p == ''
+          return token(
+            Token::ERROR,
+            'unexpected EOF',
+          )
+        end
+        unless Token::DIGITS.include?(p)
+          return token(
+            Token::ERROR,
+            "unexpected char in number literal: `#{p}`",
+          )
+        end
+        consume_digits
+      end
+
+      token_with_consumed_value(Token::NUMBER)
+    end
+
+    sig { void }
+    def swallow_rest_of_the_string
+      loop do
+        # swallow the rest of the string
+        ch, more_tokens = advance_char
+        break if !more_tokens || ch == '"'
+      end
+    end
+
+    sig { returns(Token) }
+    def scan_string
+      value_buffer = String.new
+      loop do
+        char, ok = advance_char
+        return token(Token::ERROR, 'unterminated string literal') unless ok
+        return token(Token::STRING, value_buffer) if char == '"'
+
+        if char != '\\'
+          value_buffer << char
+          next
+        end
+
+        char, ok = advance_char
+        return token(Token::ERROR, 'unterminated string literal') unless ok
+
+        case char
+        when '"'
+          value_buffer << '"'
+        when '\\'
+          value_buffer << '\\'
+        when '/'
+          value_buffer << '/'
+        when 'b'
+          value_buffer << "\b"
+        when 'f'
+          value_buffer << "\f"
+        when 'n'
+          value_buffer << "\n"
+        when 'r'
+          value_buffer << "\r"
+        when 't'
+          value_buffer << "\t"
+        when 'u'
+          unless accept_chars(Token::HEX_DIGITS, 4)
+            swallow_rest_of_the_string
+            return Token.new(Token::ERROR, 'invalid unicode escape')
+          end
+
+          advance_chars(4)
+          last4 = T.must @source[@cursor - 4...@cursor]
+          value_buffer << [last4.hex].pack('U')
+        else
+          swallow_rest_of_the_string
+          return Token.new(Token::ERROR, "invalid escape `\\#{char}`")
+        end
+      end
+    end
+  end
+end

data/lib/ruby_json_parser/parser.rb

@@ -0,0 +1,205 @@
+# typed: strict
+# frozen_string_literal: true
+
+class RubyJsonParser
+  # JSON parser
+  class Parser
+    extend T::Sig
+
+    class << self
+      extend T::Sig
+
+      sig { params(source: String).returns(Result) }
+      def parse(source)
+        new(source).parse
+      end
+
+      private :new
+    end
+
+    sig { params(source: String).void }
+    def initialize(source)
+      # Lexer/Tokenizer that produces tokens
+      @lexer = T.let(Lexer.new(source), Lexer)
+      # Next token used for predicting productions
+      @lookahead = T.let(Token.new(Token::NONE), Token)
+      @errors = T.let([], T::Array[String])
+    end
+
+    sig { returns(Result) }
+    def parse
+      advance # populate @lookahead
+      ast = parse_value
+      Result.new(ast, @errors)
+    end
+
+    private
+
+    sig { returns(AST::Node) }
+    def parse_value
+      case @lookahead.type
+      when Token::FALSE
+        advance
+        AST::FalseLiteralNode.new
+      when Token::TRUE
+        advance
+        AST::TrueLiteralNode.new
+      when Token::NULL
+        advance
+        AST::NullLiteralNode.new
+      when Token::NUMBER
+        tok = advance
+        AST::NumberLiteralNode.new(T.must(tok.value))
+      when Token::STRING
+        tok = advance
+        AST::StringLiteralNode.new(T.must(tok.value))
+      when Token::LBRACKET
+        parse_array
+      when Token::LBRACE
+        parse_object
+      else
+        tok = advance
+        add_error("unexpected token `#{tok}`") if tok.type != Token::ERROR
+        AST::InvalidNode.new(tok)
+      end
+    end
+
+    sig { returns(AST::Node) }
+    def parse_object
+      advance # swallow `{`
+      return AST::ObjectLiteralNode.new([]) if match(Token::RBRACE)
+
+      pairs = parse_key_value_pairs
+      consume(Token::RBRACE)
+      AST::ObjectLiteralNode.new(pairs)
+    end
+
+    sig { returns(T::Array[AST::KeyValuePairNode]) }
+    def parse_key_value_pairs
+      elements = [parse_key_value_pair]
+
+      loop do
+        break if accept(Token::END_OF_FILE, Token::RBRACE)
+        break unless match(Token::COMMA)
+
+        if accept(Token::RBRACE)
+          add_error('illegal trailing comma in object literal')
+          break
+        end
+
+        elements << parse_key_value_pair
+      end
+
+      elements
+    end
+
+    sig { returns(AST::KeyValuePairNode) }
+    def parse_key_value_pair
+      key = parse_value
+      if accept(Token::COMMA, Token::RBRACE, Token::END_OF_FILE)
+        add_error("missing key in object literal for value: `#{key}`")
+        return AST::KeyValuePairNode.new(nil, key)
+      end
+
+      add_error("non-string key in object literal: `#{key}`") unless key.is_a?(AST::StringLiteralNode)
+      consume(Token::COLON)
+      value = parse_value
+
+      AST::KeyValuePairNode.new(key, value)
+    end
+
+    sig { returns(AST::Node) }
+    def parse_array
+      advance # swallow `[`
+      return AST::ArrayLiteralNode.new([]) if match(Token::RBRACKET)
+
+      elements = parse_array_elements
+      consume(Token::RBRACKET)
+      AST::ArrayLiteralNode.new(elements)
+    end
+
+    sig { returns(T::Array[AST::Node]) }
+    def parse_array_elements
+      elements = [parse_value]
+
+      loop do
+        break if accept(Token::END_OF_FILE, Token::RBRACKET)
+        break unless match(Token::COMMA)
+
+        if accept(Token::RBRACKET)
+          add_error('illegal trailing comma in array literal')
+          break
+        end
+
+        elements << parse_value
+      end
+
+      elements
+    end
+
+    # Move over to the next token.
+    sig { returns(Token) }
+    def advance
+      previous = @lookahead
+      @lookahead = @lexer.next
+      handle_error_token(@lookahead) if @lookahead.type == Token::ERROR
+
+      previous
+    end
+
+    # Add the content of an error token to the syntax error list.
+    sig { params(err: Token).void }
+    def handle_error_token(err)
+      msg = err.value
+      return unless msg
+
+      add_error(msg)
+    end
+
+    # Register a syntax error
+    sig { params(err: String).void }
+    def add_error(err)
+      @errors << err
+    end
+
+    # Checks if the next token matches any of the given types,
+    # if so it gets consumed.
+    sig { params(token_types: Symbol).returns(T.nilable(Token)) }
+    def match(*token_types)
+      token_types.each do |type|
+        return advance if accept(type)
+      end
+
+      nil
+    end
+
+    # Checks whether the next token matches any the specified types.
+    sig { params(token_types: Symbol).returns(T::Boolean) }
+    def accept(*token_types)
+      token_types.each do |type|
+        return true if @lookahead.type == type
+      end
+
+      false
+    end
+
+    sig { params(token_type: Symbol).returns([Token, T::Boolean]) }
+    def consume(token_type)
+      return advance, false if @lookahead.type == Token::ERROR
+
+      if @lookahead.type != token_type
+        error_expected(Token.type_to_string(token_type))
+        return advance, false
+      end
+
+      [advance, true]
+    end
+
+    # Adds an error which tells the user that another type of token
+    # was expected.
+    sig { params(expected: String).void }
+    def error_expected(expected)
+      add_error("unexpected `#{@lookahead}`, expected `#{expected}`")
+    end
+  end
+end

data/lib/ruby_json_parser/result.rb

@@ -0,0 +1,43 @@
+# typed: strong
+# frozen_string_literal: true
+
+class RubyJsonParser
+  # The result of parsing a JSON string/file.
+  # Combines an AST (Abstract Syntax Tree) and a list of errors.
+  class Result
+    extend T::Sig
+
+    sig { returns(AST::Node) }
+    attr_reader :ast
+
+    sig { returns(T::Array[String]) }
+    attr_reader :errors
+
+    sig { params(ast: AST::Node, errors: T::Array[String]).void }
+    def initialize(ast, errors)
+      @ast = ast
+      @errors = errors
+    end
+
+    sig { returns(T::Boolean) }
+    def err?
+      @errors.any?
+    end
+
+    sig { returns(String) }
+    def inspect
+      buff = String.new
+      buff << "<RubyJsonParser::Result>\n"
+      if @errors.any?
+        buff << "  !Errors!\n"
+        @errors.each do |err|
+          buff << "    - #{err}\n"
+        end
+        buff << "\n"
+      end
+
+      buff << "  AST:\n"
+      buff << @ast.inspect(2)
+    end
+  end
+end