json5-ruby 0.1.0

data/lib/json5/input.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+module JSON5
+  class Input
+    Position = Data.define(:byte_offset, :line, :column)
+
+    attr_reader :source, :index, :line, :column, :line_start_byte
+
+    def initialize(source, limits: Limits.default, filename: nil)
+      unless source.is_a?(String)
+        raise TypeError, "source must be a String"
+      end
+      unless limits.is_a?(Limits)
+        raise ArgumentError, "limits must be a JSON5::Limits"
+      end
+      @filename = filename
+      unless [Encoding::UTF_8, Encoding::US_ASCII].include?(source.encoding)
+        raise EncodingError.new(
+          "source must be UTF-8 or US-ASCII",
+          code: "invalid_encoding",
+          filename: filename,
+          byte_offset: 0,
+          end_byte_offset: [source.bytesize, 1].min,
+          line: 1,
+          column: 1,
+          excerpt: excerpt_for(source, 0)
+        )
+      end
+      if limits.max_input_bytes && source.bytesize > limits.max_input_bytes
+        line, column = position_for_limit(source, limits.max_input_bytes)
+        raise LimitError.new(
+          "input exceeds max_input_bytes",
+          code: "maximum_input_size_exceeded",
+          filename: filename,
+          byte_offset: limits.max_input_bytes,
+          end_byte_offset: limits.max_input_bytes + 1,
+          line: line,
+          column: column,
+          excerpt: limit_excerpt_for(source, limits.max_input_bytes)
+        )
+      end
+      unless source.valid_encoding?
+        offset = first_invalid_utf8_offset(source)
+        line, column = position_at(source, offset)
+        raise EncodingError.new(
+          "source is not valid UTF-8",
+          code: "invalid_encoding",
+          filename: filename,
+          byte_offset: offset,
+          end_byte_offset: [offset + 1, source.bytesize].min,
+          line: line,
+          column: column,
+          excerpt: excerpt_for(source, offset)
+        )
+      end
+
+      @source = source
+      @limits = limits
+      @index = 0
+      @line = 1
+      @column = 1
+      @line_start_byte = 0
+    end
+
+    def eof?
+      index >= source.bytesize
+    end
+
+    def peek_byte(offset = 0)
+      source.getbyte(index + offset)
+    end
+
+    def mark
+      Position.new(index, line, column)
+    end
+
+    def position
+      mark
+    end
+
+    def self.position_for(source, byte_offset)
+      return [nil, nil] unless source.is_a?(String)
+
+      normalized = source.dup
+      normalized.force_encoding(Encoding::UTF_8) unless normalized.encoding == Encoding::UTF_8
+      input = allocate
+      input.send(:position_at, normalized, [[byte_offset, 0].max, normalized.bytesize].min)
+    rescue ArgumentError, TypeError
+      [nil, nil]
+    end
+
+    def byteslice(start_byte, end_byte = index)
+      source.byteslice(start_byte...end_byte)
+    end
+
+    def remaining
+      source.bytesize - index
+    end
+
+    def consume_byte
+      byte = peek_byte
+      return nil unless byte
+      @index += 1
+      @column += 1
+      byte
+    end
+
+    def consume_ascii(expected = nil)
+      byte = peek_byte
+      if expected && byte != expected
+        raise ArgumentError, "expected byte #{expected}, got #{byte.inspect}"
+      end
+      consume_byte
+    end
+
+    def consume_bytes(count)
+      count.times { consume_byte }
+    end
+
+    def consume_ascii_bytes(count)
+      @index += count
+      @column += count
+    end
+
+    def peek_codepoint
+      return [nil, 0] if eof?
+      byte = peek_byte
+      return [byte, 1] if byte < 0x80
+      decode_utf8(index)
+    end
+
+    def consume_codepoint
+      codepoint, width = peek_codepoint
+      return [nil, 0] unless codepoint
+      if codepoint == 0x0d
+        width += 1 if peek_byte(width) == 0x0a
+        @index += width
+        advance_line
+      elsif line_terminator?(codepoint)
+        @index += width
+        advance_line
+      else
+        @index += width
+        @column += 1
+      end
+      [codepoint, width]
+    end
+
+    def line_terminator?(codepoint)
+      codepoint == 0x0a || codepoint == 0x0d || codepoint == 0x2028 || codepoint == 0x2029
+    end
+
+    def excerpt_at(byte_offset, radius: 40)
+      start_byte = [byte_offset - radius, 0].max
+      finish_byte = [byte_offset + radius, source.bytesize].min
+      source.byteslice(start_byte...finish_byte)&.scrub
+    end
+
+    private
+
+    def advance_line
+      @line += 1
+      @column = 1
+      @line_start_byte = index
+    end
+
+    def decode_utf8(offset)
+      first = source.getbyte(offset)
+      width, codepoint, minimum = utf8_shape(first)
+      bytes = (1...width).map { |delta| source.getbyte(offset + delta) }
+      unless bytes.all? { |byte| byte && byte.between?(0x80, 0xbf) }
+        invalid_utf8!(offset)
+      end
+      codepoint = bytes.reduce(codepoint) { |value, byte| (value << 6) | (byte & 0x3f) }
+      invalid_utf8!(offset) if codepoint < minimum || codepoint > 0x10ffff || codepoint.between?(0xd800, 0xdfff)
+      [codepoint, width]
+    end
+
+    def utf8_shape(first)
+      case first
+      when 0xc2..0xdf then [2, first & 0x1f, 0x80]
+      when 0xe0..0xef then [3, first & 0x0f, 0x800]
+      when 0xf0..0xf4 then [4, first & 0x07, 0x10000]
+      else invalid_utf8!(index)
+      end
+    end
+
+    def invalid_utf8!(offset)
+      raise EncodingError.new(
+        "invalid UTF-8 sequence",
+        code: "invalid_encoding",
+        filename: @filename,
+        byte_offset: offset,
+        end_byte_offset: [offset + 1, source.bytesize].min,
+        line: line,
+        column: column,
+        excerpt: excerpt_at(offset)
+      )
+    end
+
+    def excerpt_for(source, byte_offset, radius: 40)
+      start_byte = [byte_offset - radius, 0].max
+      finish_byte = [byte_offset + radius, source.bytesize].min
+      source.byteslice(start_byte...finish_byte)&.dup&.force_encoding(Encoding::UTF_8)&.scrub
+    end
+
+    def limit_excerpt_for(source, byte_offset, radius: 40)
+      start_byte = [byte_offset - radius, 0].max
+      source.byteslice(start_byte...byte_offset + 1)&.dup&.force_encoding(Encoding::UTF_8)&.scrub
+    end
+
+    def first_invalid_utf8_offset(source)
+      offset = 0
+      while offset < source.bytesize
+        first = source.getbyte(offset)
+        if first < 0x80
+          offset += 1
+          next
+        end
+
+        width, codepoint, minimum = case first
+                                    when 0xc2..0xdf then [2, first & 0x1f, 0x80]
+                                    when 0xe0..0xef then [3, first & 0x0f, 0x800]
+                                    when 0xf0..0xf4 then [4, first & 0x07, 0x10000]
+                                    else return offset
+                                    end
+        bytes = (1...width).map { |delta| source.getbyte(offset + delta) }
+        return offset unless bytes.all? { |byte| byte && byte.between?(0x80, 0xbf) }
+
+        codepoint = bytes.reduce(codepoint) { |value, byte| (value << 6) | (byte & 0x3f) }
+        return offset if codepoint < minimum || codepoint > 0x10ffff || codepoint.between?(0xd800, 0xdfff)
+
+        offset += width
+      end
+      source.bytesize
+    end
+
+    def position_at(source, byte_offset)
+      line = 1
+      column = 1
+      index = 0
+      while index < byte_offset
+        packed = valid_codepoint_at(source, index)
+        width = packed & 0x07
+        codepoint = packed >> 3
+        break if index + width > byte_offset
+
+        if codepoint == 0x0d
+          width += 1 if index + width < byte_offset && source.getbyte(index + width) == 0x0a
+          line += 1
+          column = 1
+        elsif line_terminator?(codepoint)
+          line += 1
+          column = 1
+        else
+          column += 1
+        end
+        index += width
+      end
+      [line, column]
+    end
+
+    def valid_codepoint_at(source, offset)
+      first = source.getbyte(offset)
+      return (first << 3) | 1 if first < 0x80
+
+      width, codepoint = case first
+                         when 0xc2..0xdf then [2, first & 0x1f]
+                         when 0xe0..0xef then [3, first & 0x0f]
+                         when 0xf0..0xf4 then [4, first & 0x07]
+                         else raise ArgumentError, "invalid UTF-8 before requested position"
+                         end
+      (1...width).each { |delta| codepoint = (codepoint << 6) | (source.getbyte(offset + delta) & 0x3f) }
+      (codepoint << 3) | width
+    end
+
+    def position_for_limit(source, byte_offset)
+      position_at(source, byte_offset)
+    rescue ArgumentError
+      [nil, nil]
+    end
+  end
+end