json5-ruby 0.1.0

data/lib/json5/lexer.rb

@@ -0,0 +1,599 @@
+# frozen_string_literal: true
+
+module JSON5
+  class Lexer
+    SIMPLE_ESCAPES = {
+      0x27 => 0x27,
+      0x22 => 0x22,
+      0x5c => 0x5c,
+      0x62 => 0x08,
+      0x66 => 0x0c,
+      0x6e => 0x0a,
+      0x72 => 0x0d,
+      0x74 => 0x09,
+      0x76 => 0x0b
+    }.freeze
+    KEYWORDS = {
+      "true" => :TRUE,
+      "false" => :FALSE,
+      "null" => :NULL,
+      "Infinity" => :INFINITY,
+      "NaN" => :NAN
+    }.freeze
+    PUNCTUATORS = {
+      0x7b => "{",
+      0x7d => "}",
+      0x5b => "[",
+      0x5d => "]",
+      0x3a => ":",
+      0x2c => ","
+    }.freeze
+    ASCII = ASCIIClassification::TABLE
+    ASCII_WHITESPACE = ASCIIClassification::WHITESPACE
+    ASCII_DIGIT = ASCIIClassification::DIGIT
+    ASCII_HEX_DIGIT = ASCIIClassification::HEX_DIGIT
+    ASCII_IDENTIFIER_START = ASCIIClassification::IDENTIFIER_START
+    ASCII_IDENTIFIER_PART = ASCIIClassification::IDENTIFIER_PART
+    ASCII_QUOTE = ASCIIClassification::QUOTE
+    ASCII_PUNCTUATOR = ASCIIClassification::PUNCTUATOR
+    ASCII_SLASH = ASCIIClassification::SLASH
+    ASCII_BACKSLASH = ASCIIClassification::BACKSLASH
+    ASCII_LINE_TERMINATOR = ASCIIClassification::LINE_TERMINATOR
+    ASCII_STRING_STOP = ASCII_BACKSLASH | ASCII_LINE_TERMINATOR
+    EMPTY_TRIVIA = [].freeze
+
+    attr_reader :input, :last_token, :nesting_depth,
+      :last_token_container, :last_token_nesting_depth
+
+    def initialize(input, limits: Limits.default, diagnostics: nil, filename: nil, preserve_trivia: false)
+      @input = input
+      @limits = limits
+      @diagnostics = DiagnosticSink.build(diagnostics, filename: filename)
+      @filename = filename
+      @preserve_trivia = preserve_trivia
+      @leading_trivia = []
+      @container_stack = []
+      @nesting_depth = 0
+    end
+
+    def next_token
+      skip_trivia
+      @last_token_container = current_container
+      @last_token_nesting_depth = @nesting_depth
+      start_byte = input.index
+      start_line = input.line
+      start_column = input.column
+      kind, value = scan_token
+      finish_token(kind, value, start_byte, start_line, start_column)
+    end
+
+    private
+
+    def scan_token
+      return [:EOF, nil] if input.eof?
+      byte = input.peek_byte
+      flags = ASCII.fetch(byte)
+      return [PUNCTUATORS.fetch(byte), consume_punctuation] unless (flags & ASCII_PUNCTUATOR).zero?
+      return [:STRING, scan_string] unless (flags & ASCII_QUOTE).zero?
+      return [:NUMBER, scan_number] if number_start?(byte)
+      return scan_identifier if identifier_start_byte?(byte) || !(flags & ASCII_BACKSLASH).zero?
+
+      raise_error("unexpected_character", "unexpected character", input.mark, LexError)
+    end
+
+    def finish_token(kind, value, start_byte, start_line, start_column)
+      token = Token.new(
+        kind,
+        value,
+        start_byte,
+        input.index,
+        start_line,
+        start_column,
+        nil,
+        (@preserve_trivia ? @leading_trivia.freeze : EMPTY_TRIVIA),
+        source: input.source
+      )
+      @last_token = token
+      @leading_trivia = []
+      token
+    end
+
+    def consume_punctuation
+      byte = input.peek_byte
+      if byte == 0x7b || byte == 0x5b
+        @container_stack << byte
+        @nesting_depth = @container_stack.length
+        if @limits.max_depth && @nesting_depth > @limits.max_depth
+          raise_error("maximum_depth_exceeded", "nested value exceeds max_depth", input.mark, DepthError)
+        end
+      elsif byte == 0x7d || byte == 0x5d
+        matching_open = byte == 0x7d ? 0x7b : 0x5b
+        @container_stack.pop if @container_stack.last == matching_open
+        @nesting_depth = @container_stack.length
+      end
+      input.consume_byte
+      nil
+    end
+
+    public
+
+    def current_container
+      case @container_stack.last
+      when 0x7b then "{"
+      when 0x5b then "["
+      end
+    end
+
+    private
+
+    def skip_trivia
+      loop do
+        if whitespace?
+          start = input.mark
+          input.consume_codepoint
+          record_trivia(:whitespace, start)
+          next
+        end
+        unless (ascii_flags(input.peek_byte) & ASCII_SLASH).zero?
+          if input.peek_byte(1) == 0x2f
+            start = input.mark
+            scan_line_comment(start)
+            next
+          end
+          if input.peek_byte(1) == 0x2a
+            start = input.mark
+            scan_block_comment(start)
+            next
+          end
+        end
+        break
+      end
+    end
+
+    def whitespace?
+      byte = input.peek_byte
+      return false unless byte
+      return !(ASCII.fetch(byte) & ASCII_WHITESPACE).zero? if byte < 0x80
+
+      codepoint, = input.peek_codepoint
+      codepoint == 0xa0 || codepoint == 0x2028 || codepoint == 0x2029 || codepoint == 0xfeff ||
+        UnicodeTables.space_separator?(codepoint)
+    end
+
+    def scan_line_comment(start)
+      input.consume_ascii(0x2f)
+      input.consume_ascii(0x2f)
+      enforce_comment_limit(start)
+      while !input.eof?
+        codepoint, = input.peek_codepoint
+        break if input.line_terminator?(codepoint)
+        input.consume_codepoint
+        enforce_comment_limit(start)
+      end
+      record_trivia(:line_comment, start)
+    end
+
+    def scan_block_comment(start)
+      input.consume_ascii(0x2f)
+      input.consume_ascii(0x2a)
+      enforce_comment_limit(start)
+      until input.eof?
+        if input.peek_byte == 0x2a && input.peek_byte(1) == 0x2f
+          input.consume_byte
+          input.consume_byte
+          enforce_comment_limit(start)
+          record_trivia(:block_comment, start)
+          return
+        end
+        input.consume_codepoint
+        enforce_comment_limit(start)
+      end
+      raise_error("unterminated_comment", "unterminated block comment", start, LexError)
+    end
+
+    def enforce_comment_limit(start)
+      return unless @limits.max_comment_bytes && input.index - start.byte_offset > @limits.max_comment_bytes
+      raise_error("maximum_comment_size_exceeded", "comment exceeds max_comment_bytes", start, LimitError)
+    end
+
+    def record_trivia(kind, start)
+      return unless @preserve_trivia
+      raw = input.byteslice(start.byte_offset).freeze
+      @leading_trivia << Trivia.new(kind, start.byte_offset, input.index, raw)
+    end
+
+    def scan_identifier
+      start = input.mark
+      units = []
+      escaped = consume_identifier_character(units, first: true)
+      while identifier_continuation?
+        escaped = consume_identifier_character(units, first: false) || escaped
+      end
+      name = ECMAString.new(units)
+      kind = keyword_kind(start.byte_offset, input.index) unless escaped
+      if kind
+        enforce_number_limit(start) if %i[INFINITY NAN].include?(kind)
+        value = if %i[INFINITY NAN].include?(kind)
+                  NumberValue.new(input.byteslice(start.byte_offset))
+                else
+                  name
+                end
+        return [kind, value]
+      end
+      [:IDENTIFIER_NAME, name]
+    end
+
+    def consume_identifier_character(units, first:)
+      start = input.mark
+      if input.peek_byte == 0x5c
+        unit = consume_identifier_escape
+        valid = first ? UnicodeTables.identifier_start?(unit) : UnicodeTables.identifier_part?(unit)
+        raise_error("invalid_identifier_character", "escaped character is not valid in IdentifierName", start, LexError) unless valid
+        units << unit
+        return true
+      end
+      codepoint, = input.peek_codepoint
+      input.consume_codepoint
+      if codepoint > 0xffff
+        raise_error("invalid_identifier_character", "non-BMP identifier characters are not supported", start, LexError)
+      end
+      valid = first ? identifier_start_codepoint?(codepoint) : identifier_part_codepoint?(codepoint)
+      raise_error("invalid_identifier_character", "character is not valid in IdentifierName", start, LexError) unless valid
+      units << codepoint
+      false
+    end
+
+    def keyword_kind(start_byte, end_byte)
+      length = end_byte - start_byte
+      KEYWORDS.each do |word, kind|
+        next unless word.bytesize == length
+        return kind if word.bytes.each_with_index.all? { |byte, index| input.source.getbyte(start_byte + index) == byte }
+      end
+      nil
+    end
+
+    def consume_identifier_escape
+      start = input.mark
+      input.consume_ascii(0x5c)
+      unless input.peek_byte == 0x75
+        raise_error("invalid_identifier_escape", "IdentifierName escape must use \\uHHHH", start, LexError)
+      end
+      input.consume_byte
+      value = 0
+      4.times do
+        byte = input.peek_byte
+        unless hex_digit_byte?(byte)
+          raise_error("invalid_identifier_escape", "IdentifierName escape requires four hex digits", start, LexError)
+        end
+        input.consume_byte
+        value = (value << 4) | hex_digit_value(byte)
+      end
+      value
+    end
+
+    def identifier_continuation?
+      return false if input.eof?
+      return true if input.peek_byte == 0x5c
+      byte = input.peek_byte
+      return !(ASCII.fetch(byte) & ASCII_IDENTIFIER_PART).zero? if byte < 0x80
+
+      codepoint, = input.peek_codepoint
+      identifier_part_codepoint?(codepoint)
+    end
+
+    def identifier_start_byte?(byte)
+      byte >= 0x80 || !(ASCII.fetch(byte) & ASCII_IDENTIFIER_START).zero?
+    end
+
+    def identifier_start_codepoint?(codepoint)
+      return false unless codepoint
+      return !(ASCII.fetch(codepoint) & ASCII_IDENTIFIER_START).zero? if codepoint < 0x80
+
+      UnicodeTables.identifier_start?(codepoint)
+    end
+
+    def identifier_part_codepoint?(codepoint)
+      return false unless codepoint
+      return !(ASCII.fetch(codepoint) & ASCII_IDENTIFIER_PART).zero? if codepoint < 0x80
+
+      UnicodeTables.identifier_part?(codepoint)
+    end
+
+    def scan_string
+      quote = input.consume_byte
+      content_start = input.index
+      stop = ascii_string_stop(content_start, quote)
+      units = consume_ascii_string_prefix(content_start, stop)
+
+      if input.peek_byte == quote
+        input.consume_byte
+        return ECMAString.new(units)
+      end
+
+      loop do
+        raise_error("unterminated_string", "unterminated string", input.mark, LexError) if input.eof?
+        codepoint, = input.peek_codepoint
+        if codepoint == quote
+          input.consume_byte
+          return ECMAString.new(units)
+        end
+        if codepoint == 0x5c
+          scan_escape(units)
+          next
+        end
+        if codepoint == 0x0a || codepoint == 0x0d
+          raise_error("unterminated_string", "line terminator in string", input.mark, LexError)
+        end
+        if codepoint == 0x2028 || codepoint == 0x2029
+          position = input.mark
+          @diagnostics.emit(
+            code: codepoint == 0x2028 ? "unescaped_line_separator_in_string" : "unescaped_paragraph_separator_in_string",
+            message: "unescaped Unicode line separator in string",
+            byte_offset: input.index,
+            end_byte_offset: input.index + 3,
+            line: position.line,
+            column: position.column
+          )
+        end
+        input.consume_codepoint
+        append_codepoint(units, codepoint)
+        enforce_string_limit(units)
+      end
+    end
+
+    def ascii_string_stop(start, quote)
+      index = start
+      while (byte = input.source.getbyte(index))
+        return index if byte >= 0x80 || byte == quote || !(ASCII.fetch(byte) & ASCII_STRING_STOP).zero?
+        index += 1
+      end
+      index
+    end
+
+    def consume_ascii_string_prefix(start, finish)
+      length = finish - start
+      limit = @limits.max_string_code_units
+      if limit && length > limit
+        input.consume_ascii_bytes(limit + 1)
+        raise_string_limit
+      end
+
+      input.consume_ascii_bytes(length)
+      input.source.byteslice(start, length).bytes
+    end
+
+    def scan_escape(units)
+      start = input.mark
+      input.consume_byte
+      raise_error("invalid_escape", "backslash at end of string", start, LexError) if input.eof?
+      codepoint, = input.peek_codepoint
+      if input.line_terminator?(codepoint)
+        input.consume_codepoint
+        return
+      end
+      if SIMPLE_ESCAPES.key?(codepoint)
+        input.consume_byte
+        units << SIMPLE_ESCAPES.fetch(codepoint)
+      elsif codepoint == 0x30
+        input.consume_byte
+        if decimal_digit_byte?(input.peek_byte)
+          raise_error("invalid_escape", "\\0 cannot be followed by a decimal digit", start, LexError)
+        end
+        units << 0
+      elsif codepoint.between?(0x31, 0x39)
+        raise_error("invalid_escape", "decimal and octal escapes are not allowed", start, LexError)
+      elsif codepoint == 0x78
+        units << read_hex_escape(2, start, "x")
+      elsif codepoint == 0x75
+        units << read_hex_escape(4, start, "u")
+      else
+        input.consume_codepoint
+        append_codepoint(units, codepoint)
+      end
+      enforce_string_limit(units)
+    end
+
+    def read_hex_escape(length, start, marker)
+      input.consume_byte
+      value = 0
+      length.times do
+        byte = input.peek_byte
+        unless hex_digit_byte?(byte)
+          raise_error("invalid_escape", "\\#{marker} escape requires #{length} hex digits", start, LexError)
+        end
+        input.consume_byte
+        value = (value << 4) | hex_digit_value(byte)
+      end
+      value
+    end
+
+    def append_codepoint(units, codepoint)
+      if codepoint <= 0xffff
+        units << codepoint
+      else
+        codepoint -= 0x10000
+        units << (0xd800 + (codepoint >> 10))
+        units << (0xdc00 + (codepoint & 0x3ff))
+      end
+    end
+
+    def enforce_string_limit(units)
+      return unless @limits.max_string_code_units && units.length > @limits.max_string_code_units
+      raise_string_limit
+    end
+
+    def raise_string_limit
+      raise_error("maximum_string_size_exceeded", "string exceeds max_string_code_units", input.mark, LimitError)
+    end
+
+    def scan_number
+      start = input.mark
+      if [0x2b, 0x2d].include?(input.peek_byte)
+        consume_number_byte(start)
+        raw = scan_signed_number_body(start)
+        return raw
+      end
+      if input.peek_byte == 0x2e
+        consume_number_byte(start)
+        require_decimal_digits(start)
+        consume_number_byte(start) while decimal_digit_byte?(input.peek_byte)
+        scan_decimal_tail(start)
+        check_number_boundary(start)
+        enforce_number_limit(start)
+        return input.byteslice(start.byte_offset)
+      end
+      scan_unsigned_number(start)
+    end
+
+    def scan_signed_number_body(start)
+      if match_ascii_word?("Infinity") || match_ascii_word?("NaN")
+        consume_number_bytes(input.peek_byte == 0x49 ? 8 : 3, start)
+        check_number_boundary(start)
+        enforce_number_limit(start)
+        return input.byteslice(start.byte_offset)
+      end
+      if input.peek_byte == 0x30 && [0x78, 0x58].include?(input.peek_byte(1))
+        scan_hex_body(start)
+      elsif input.peek_byte == 0x2e
+        consume_number_byte(start)
+        require_decimal_digits(start)
+        consume_number_byte(start) while decimal_digit_byte?(input.peek_byte)
+        scan_decimal_tail(start)
+        check_number_boundary(start)
+        enforce_number_limit(start)
+        input.byteslice(start.byte_offset)
+      elsif input.peek_byte == 0x30
+        consume_number_byte(start)
+        raise_error("leading_zero", "leading zero is not allowed", start, LexError) if decimal_digit_byte?(input.peek_byte)
+        scan_decimal_tail(start)
+        raw = input.byteslice(start.byte_offset)
+        check_number_boundary(start)
+        enforce_number_limit(start)
+        raw
+      elsif decimal_digit_byte?(input.peek_byte)
+        scan_decimal_body(start)
+      else
+        raise_error("invalid_number", "sign must be followed by a JSON5 number", start, LexError)
+      end
+    end
+
+    def scan_unsigned_number(start)
+      if input.peek_byte == 0x30 && [0x78, 0x58].include?(input.peek_byte(1))
+        scan_hex_body(start)
+      elsif input.peek_byte == 0x30
+        consume_number_byte(start)
+        raise_error("leading_zero", "leading zero is not allowed", start, LexError) if decimal_digit_byte?(input.peek_byte)
+        scan_decimal_tail(start)
+        raw = input.byteslice(start.byte_offset)
+        check_number_boundary(start)
+        enforce_number_limit(start)
+        raw
+      elsif input.peek_byte&.between?(0x31, 0x39)
+        scan_decimal_body(start)
+      else
+        raise_error("invalid_number", "invalid number", start, LexError)
+      end
+    end
+
+    def scan_decimal_body(start)
+      consume_number_byte(start) while decimal_digit_byte?(input.peek_byte)
+      scan_decimal_tail(start)
+      raw = input.byteslice(start.byte_offset)
+      check_number_boundary(start)
+      enforce_number_limit(start)
+      raw
+    end
+
+    def scan_decimal_tail(start)
+      if input.peek_byte == 0x2e
+        consume_number_byte(start)
+        consume_number_byte(start) while decimal_digit_byte?(input.peek_byte)
+      end
+      if [0x65, 0x45].include?(input.peek_byte)
+        consume_number_byte(start)
+        consume_number_byte(start) if [0x2b, 0x2d].include?(input.peek_byte)
+        require_decimal_digits(input.mark)
+        consume_number_byte(start) while decimal_digit_byte?(input.peek_byte)
+      end
+    end
+
+    def scan_hex_body(start)
+      consume_number_byte(start)
+      consume_number_byte(start)
+      unless hex_digit_byte?(input.peek_byte)
+        raise_error("invalid_number", "hexadecimal number requires at least one digit", start, LexError)
+      end
+      consume_number_byte(start) while hex_digit_byte?(input.peek_byte)
+      raw = input.byteslice(start.byte_offset)
+      check_number_boundary(start)
+      enforce_number_limit(start)
+      raw
+    end
+
+    def require_decimal_digits(start)
+      raise_error("invalid_number", "number requires a decimal digit", start, LexError) unless decimal_digit_byte?(input.peek_byte)
+    end
+
+    def check_number_boundary(start)
+      return if input.eof?
+      codepoint, = input.peek_codepoint
+      return unless identifier_start_codepoint?(codepoint) || decimal_digit_byte?(codepoint) || input.peek_byte == 0x5c
+      raise_error("invalid_number", "number is followed by an identifier character", start, LexError)
+    end
+
+    def enforce_number_limit(start)
+      return unless @limits.max_number_bytes && input.index - start.byte_offset > @limits.max_number_bytes
+      raise_error("maximum_number_size_exceeded", "number exceeds max_number_bytes", start, LimitError)
+    end
+
+    def consume_number_byte(start)
+      input.consume_byte
+      enforce_number_limit(start)
+    end
+
+    def consume_number_bytes(count, start)
+      count.times { consume_number_byte(start) }
+    end
+
+    def match_ascii_word?(word)
+      input.source.byteslice(input.index, word.bytesize) == word
+    end
+
+    def number_start?(byte)
+      byte == 0x2b || byte == 0x2d || byte == 0x2e || decimal_digit_byte?(byte)
+    end
+
+    def decimal_digit_byte?(byte)
+      !(ascii_flags(byte) & ASCII_DIGIT).zero?
+    end
+
+    def hex_digit_byte?(byte)
+      !(ascii_flags(byte) & ASCII_HEX_DIGIT).zero?
+    end
+
+    def hex_digit_value(byte)
+      return byte - 0x30 if byte <= 0x39
+      return byte - 0x37 if byte <= 0x46
+
+      byte - 0x57
+    end
+
+    def ascii_flags(byte)
+      byte && byte < ASCII.length ? ASCII.fetch(byte) : 0
+    end
+
+    def raise_error(code, message, position, klass)
+      offset = position.byte_offset
+      raise klass.new(
+        message,
+        code: code,
+        filename: @filename,
+        byte_offset: offset,
+        end_byte_offset: [[input.index, offset + 1].max, input.source.bytesize].min,
+        line: position.line,
+        column: position.column,
+        excerpt: input.excerpt_at(offset)
+      )
+    end
+  end
+end

data/lib/json5/limits.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module JSON5
+  class Limits
+    ATTRIBUTES = %i[
+      max_input_bytes
+      max_depth
+      max_string_code_units
+      max_number_bytes
+      max_comment_bytes
+      max_container_entries
+    ].freeze
+
+    attr_reader(*ATTRIBUTES)
+
+    def initialize(max_input_bytes: 64 * 1024 * 1024, max_depth: 512,
+                   max_string_code_units: 16 * 1024 * 1024,
+                   max_number_bytes: 1 * 1024 * 1024,
+                   max_comment_bytes: 16 * 1024 * 1024,
+                   max_container_entries: 10_000_000)
+      values = {
+        max_input_bytes: max_input_bytes,
+        max_depth: max_depth,
+        max_string_code_units: max_string_code_units,
+        max_number_bytes: max_number_bytes,
+        max_comment_bytes: max_comment_bytes,
+        max_container_entries: max_container_entries
+      }
+      values.each do |name, value|
+        next if value.nil?
+        unless value.is_a?(Integer) && value >= 0
+          raise ArgumentError, "#{name} must be non-negative"
+        end
+      end
+      values.each { |name, value| instance_variable_set("@#{name}", value) }
+      freeze
+    end
+
+    def self.default
+      @default ||= new
+    end
+
+    def self.unbounded
+      @unbounded ||= new(**ATTRIBUTES.to_h { |name| [name, nil] })
+    end
+  end
+end