json5-ruby 0.1.0

data/lib/json5/line_map.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module JSON5
+  class LineMap
+    def initialize(source)
+      Input.new(source, limits: Limits.unbounded)
+      @source = source
+      @line_starts = build_line_starts.freeze
+      freeze
+    end
+
+    def position(byte_offset)
+      line, column = line_column(byte_offset)
+      Input::Position.new(byte_offset, line, column)
+    end
+
+    def line_column(byte_offset)
+      validate_byte_offset!(byte_offset)
+
+      next_line = @line_starts.bsearch_index { |start_byte| start_byte > byte_offset }
+      if crlf_middle?(byte_offset)
+        return [next_line + 1, 1]
+      end
+
+      line_index = (next_line || @line_starts.length) - 1
+      [line_index + 1, column_at(@line_starts.fetch(line_index), byte_offset)]
+    end
+
+    private
+
+    def build_line_starts
+      starts = [0]
+      index = 0
+      while index < @source.bytesize
+        byte = @source.getbyte(index)
+        if byte == 0x0d
+          index += @source.getbyte(index + 1) == 0x0a ? 2 : 1
+          starts << index
+        elsif byte == 0x0a
+          index += 1
+          starts << index
+        elsif unicode_line_separator_at?(index)
+          index += 3
+          starts << index
+        else
+          index += utf8_width(byte)
+        end
+      end
+      starts
+    end
+
+    def column_at(line_start, byte_offset)
+      column = 1
+      index = line_start
+      while index < byte_offset
+        width = utf8_width(@source.getbyte(index))
+        break if index + width > byte_offset
+
+        index += width
+        column += 1
+      end
+      column
+    end
+
+    def crlf_middle?(byte_offset)
+      byte_offset.positive? &&
+        @source.getbyte(byte_offset) == 0x0a &&
+        @source.getbyte(byte_offset - 1) == 0x0d
+    end
+
+    def unicode_line_separator_at?(index)
+      @source.getbyte(index) == 0xe2 &&
+        @source.getbyte(index + 1) == 0x80 &&
+        [0xa8, 0xa9].include?(@source.getbyte(index + 2))
+    end
+
+    def utf8_width(byte)
+      return 1 if byte < 0x80
+      return 2 if byte < 0xe0
+      return 3 if byte < 0xf0
+
+      4
+    end
+
+    def validate_byte_offset!(byte_offset)
+      raise TypeError, "byte offset must be an Integer" unless byte_offset.is_a?(Integer)
+      return if byte_offset.between?(0, @source.bytesize)
+
+      raise RangeError, "byte offset is outside the source"
+    end
+  end
+end

data/lib/json5/number_value.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module JSON5
+  class NumberValue
+    attr_reader :raw, :kind
+
+    def initialize(raw, kind: nil)
+      @raw = raw.to_s.freeze
+      @kind = (kind || classify(@raw)).to_sym
+      freeze
+    end
+
+    def to_native
+      case kind
+      when :infinity
+        raw.start_with?("-") ? -Float::INFINITY : Float::INFINITY
+      when :nan
+        Float::NAN
+      when :hex
+        sign = raw.start_with?("-") ? -1 : 1
+        sign * Integer(raw.delete_prefix("+").delete_prefix("-"), 16)
+      when :integer
+        return -0.0 if raw == "-0"
+        Integer(raw, 10)
+      else
+        decimal_to_float
+      end
+    end
+
+    def to_f
+      return hex_to_f if kind == :hex
+
+      to_native.to_f
+    end
+
+    private
+
+    def decimal_to_float
+      normalized = raw.dup
+      body_start = normalized.start_with?("+", "-") ? 1 : 0
+      normalized.insert(body_start, "0") if normalized.getbyte(body_start) == 0x2e
+      decimal_point = normalized.index(".", body_start)
+      if decimal_point && [nil, 0x45, 0x65].include?(normalized.getbyte(decimal_point + 1))
+        normalized.insert(decimal_point + 1, "0")
+      end
+      Float(normalized)
+    end
+
+    def classify(value)
+      body = value.delete_prefix("+").delete_prefix("-")
+      return :infinity if body == "Infinity"
+      return :nan if body == "NaN"
+      return :hex if body.start_with?("0x", "0X")
+      return :integer if !body.include?(".") && !body.include?("e") && !body.include?("E")
+
+      :float
+    end
+
+    def hex_to_f
+      negative = raw.start_with?("-")
+      first_byte = raw.getbyte(0)
+      digits_start = first_byte == 0x2b || first_byte == 0x2d ? 3 : 2
+      digit_count = raw.bytesize - digits_start
+      first_nonzero = 0
+      first_nonzero += 1 while first_nonzero < digit_count && raw.getbyte(digits_start + first_nonzero) == 0x30
+      return negative ? -0.0 : 0.0 if first_nonzero == digit_count
+
+      first_digit = raw.getbyte(digits_start + first_nonzero)
+      first_digit -= 0x30 if first_digit <= 0x39
+      first_digit -= 0x37 if first_digit >= 0x41 && first_digit <= 0x46
+      first_digit -= 0x57 if first_digit >= 0x61 && first_digit <= 0x66
+      leading_bits = first_digit.bit_length
+      first_bit = first_nonzero * 4 + (4 - leading_bits)
+      bit_length = digit_count * 4 - first_bit
+      exponent = bit_length - 1
+      return negative ? -Float::INFINITY : Float::INFINITY if exponent > 1023
+
+      significand = 0
+      [bit_length, 53].min.times do |index|
+        significand = (significand << 1) | hex_bit(digits_start, first_bit + index)
+      end
+      if bit_length > 53
+        guard = hex_bit(digits_start, first_bit + 53)
+        sticky = nonzero_hex_bit?(digits_start, first_bit + 54, digit_count * 4)
+        significand += 1 if guard == 1 && (sticky || significand.odd?)
+        if significand == (1 << 53)
+          significand >>= 1
+          exponent += 1
+          return negative ? -Float::INFINITY : Float::INFINITY if exponent > 1023
+        end
+      else
+        significand <<= (53 - bit_length)
+      end
+
+      result = significand.to_f * (2.0**(exponent - 52))
+      negative ? -result : result
+    end
+
+    def nonzero_hex_bit?(digits_start, bit_index, end_bit)
+      while bit_index < end_bit
+        return true if hex_bit(digits_start, bit_index) == 1
+        bit_index += 1
+      end
+      false
+    end
+
+    def hex_bit(digits_start, bit_index)
+      byte = raw.getbyte(digits_start + (bit_index / 4))
+      nibble = if byte <= 0x39
+                 byte - 0x30
+               elsif byte <= 0x46
+                 byte - 0x37
+               else
+                 byte - 0x57
+               end
+      (nibble >> (3 - (bit_index % 4))) & 1
+    end
+  end
+end

data/lib/json5/object_value.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module JSON5
+  class ObjectValue
+    Member = Data.define(:key, :value)
+
+    attr_reader :members
+
+    def initialize(members)
+      @members = members.map { |key, value| Member.new(key, value) }.freeze
+      freeze
+    end
+
+    def [](key)
+      member = members.reverse_each.find { |candidate| candidate.key == key }
+      member&.value
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block
+      members.each(&block)
+    end
+
+    def to_h
+      members.to_h { |member| [member.key, member.value] }
+    end
+  end
+end

data/lib/json5/parser.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+module JSON5
+  class Parser
+    DEFAULT_OPTIONS = {
+      duplicate_keys: :last,
+      number_mode: :native,
+      string_mode: :ruby,
+      lone_surrogate: :error,
+      diagnostics: nil,
+      limits: Limits.default,
+      filename: nil,
+      freeze: false,
+      warn_duplicate_keys: false,
+      preserve_trivia: true,
+      borrow_source: false
+    }.freeze
+    PARSE_OPTION_KEYS = (DEFAULT_OPTIONS.keys - %i[preserve_trivia borrow_source]).freeze
+    DOCUMENT_OPTION_KEYS = %i[diagnostics limits filename preserve_trivia borrow_source].freeze
+
+    def self.parse(source, **options)
+      new(source, **options).parse
+    end
+
+    def self.load(io, **options)
+      unless io.respond_to?(:read)
+        raise TypeError, "io must respond to #read"
+      end
+      validate_option_keys!(options, PARSE_OPTION_KEYS)
+      validate_parse_options!(options)
+      limits = options.fetch(:limits, Limits.default)
+      source = read_source(io, limits, filename: options.fetch(:filename, nil))
+      parse(source, **options)
+    end
+
+    def self.parse_document(source, **options)
+      validate_option_keys!(options, DOCUMENT_OPTION_KEYS)
+      merged = DEFAULT_OPTIONS.merge(options)
+      validate_limits_option!(merged.fetch(:limits))
+      validate_diagnostics_option!(merged.fetch(:diagnostics))
+      validate_boolean_option!(merged.fetch(:preserve_trivia), :preserve_trivia)
+      validate_boolean_option!(merged.fetch(:borrow_source), :borrow_source)
+      unless source.is_a?(String)
+        raise TypeError, "source must be a String"
+      end
+      if merged[:limits].max_input_bytes && source.bytesize > merged[:limits].max_input_bytes
+        Input.new(source, limits: merged.fetch(:limits), filename: merged.fetch(:filename))
+      end
+      owned_source = merged.fetch(:borrow_source) ? source : source.dup.freeze
+      collected_diagnostics = []
+      diagnostics = merged.fetch(:diagnostics)
+      diagnostic_target = case diagnostics
+      when Array
+        ->(diagnostic) { collected_diagnostics << diagnostic; diagnostics << diagnostic }
+      when :ignore
+        :ignore
+      else
+        if diagnostics.respond_to?(:call)
+          ->(diagnostic) { collected_diagnostics << diagnostic; diagnostics.call(diagnostic) }
+        else
+          lambda do |diagnostic|
+            collected_diagnostics << diagnostic
+            DiagnosticSink.write_default_warning(diagnostic)
+          end
+        end
+      end
+      DocumentParser.new(
+        owned_source,
+        limits: merged.fetch(:limits),
+        diagnostics: diagnostic_target,
+        filename: merged.fetch(:filename),
+        preserve_trivia: merged.fetch(:preserve_trivia),
+        collected_diagnostics: collected_diagnostics
+      ).parse
+    end
+
+    def self.read_source(io, limits, filename: nil)
+      external_encoding = io.external_encoding if io.respond_to?(:external_encoding)
+      external_encoding = nil unless [Encoding::UTF_8, Encoding::US_ASCII].include?(external_encoding)
+      allow_binary_chunks = !external_encoding.nil?
+      unless limits.max_input_bytes
+        source = io.read
+        validate_read_chunk!(source, filename: filename, byte_offset: 0, preceding: nil,
+                             allow_binary: allow_binary_chunks)
+        return normalize_read_encoding(source, external_encoding)
+      end
+
+      source = String.new(encoding: Encoding::BINARY)
+      source_encoding = external_encoding
+      chunk_size = [limits.max_input_bytes + 1, 16 * 1024].min
+      while source.bytesize <= limits.max_input_bytes
+        chunk = io.read(chunk_size)
+        break if chunk.nil?
+        validate_read_chunk!(chunk, filename: filename, byte_offset: source.bytesize, preceding: source,
+                             allow_binary: allow_binary_chunks)
+        break if chunk.empty?
+        source_encoding = merged_read_encoding(source_encoding, chunk.encoding)
+        remaining = limits.max_input_bytes + 1 - source.bytesize
+        if chunk.bytesize >= remaining
+          position_bytes = source + chunk.b.byteslice(0, remaining + 3).to_s
+          position_bytes.force_encoding(source_encoding || Encoding::UTF_8)
+          line, column = Input.position_for(position_bytes, limits.max_input_bytes)
+          raise LimitError.new(
+            "input exceeds max_input_bytes",
+            code: "maximum_input_size_exceeded",
+            filename: filename,
+            byte_offset: limits.max_input_bytes,
+            end_byte_offset: limits.max_input_bytes + 1,
+            line: line,
+            column: column,
+            excerpt: limit_excerpt(source, chunk, limits.max_input_bytes)
+          )
+        end
+        source << chunk.b
+      end
+      source.force_encoding(source_encoding || Encoding::UTF_8)
+    end
+
+    def self.normalize_read_encoding(source, external_encoding)
+      return source unless source.encoding == Encoding::ASCII_8BIT && external_encoding
+
+      source.dup.force_encoding(external_encoding)
+    end
+
+    def self.merged_read_encoding(current, chunk_encoding)
+      return current if chunk_encoding == Encoding::ASCII_8BIT
+      return Encoding::UTF_8 if current == Encoding::UTF_8 || chunk_encoding == Encoding::UTF_8
+
+      current || chunk_encoding
+    end
+
+    def self.validate_read_chunk!(chunk, filename:, byte_offset:, preceding:, allow_binary: false)
+      raise TypeError, "io#read must return a String" unless chunk.is_a?(String)
+      return if [Encoding::UTF_8, Encoding::US_ASCII].include?(chunk.encoding) ||
+        (allow_binary && chunk.encoding == Encoding::ASCII_8BIT)
+
+      line, column = Input.position_for(preceding || chunk, byte_offset)
+      raise EncodingError.new(
+        "source must be UTF-8 or US-ASCII",
+        code: "invalid_encoding",
+        filename: filename,
+        byte_offset: byte_offset,
+        end_byte_offset: byte_offset + [chunk.bytesize, 1].min,
+        line: line,
+        column: column,
+        excerpt: limit_excerpt(preceding || String.new(encoding: Encoding::BINARY), chunk, byte_offset)
+      )
+    end
+
+    def self.validate_parse_options!(options)
+      merged = DEFAULT_OPTIONS.merge(options)
+      validate_limits_option!(merged.fetch(:limits))
+      validate_diagnostics_option!(merged.fetch(:diagnostics))
+      validate_boolean_option!(merged.fetch(:freeze), :freeze)
+      validate_boolean_option!(merged.fetch(:warn_duplicate_keys), :warn_duplicate_keys)
+      validate_enum_option!(RubyValueBuilder::VALID_DUPLICATE_POLICIES, merged.fetch(:duplicate_keys), :duplicate_keys)
+      validate_enum_option!(RubyValueBuilder::VALID_NUMBER_MODES, merged.fetch(:number_mode), :number_mode)
+      validate_enum_option!(RubyValueBuilder::VALID_STRING_MODES, merged.fetch(:string_mode), :string_mode)
+      validate_enum_option!([:error, :replace], merged.fetch(:lone_surrogate), :lone_surrogate)
+    end
+
+    def self.limit_excerpt(source, chunk, byte_offset, radius: 40)
+      start_byte = [byte_offset - radius, 0].max
+      finish_byte = byte_offset + 1
+      excerpt = String.new(encoding: Encoding::BINARY)
+      if start_byte < source.bytesize
+        excerpt << source.byteslice(start_byte...[finish_byte, source.bytesize].min).to_s.b
+      end
+      chunk_start = [start_byte - source.bytesize, 0].max
+      chunk_finish = [finish_byte - source.bytesize, 0].max
+      excerpt << chunk.byteslice(chunk_start...chunk_finish).to_s.b
+      excerpt.force_encoding(Encoding::UTF_8).scrub
+    end
+
+    def initialize(source, **options)
+      self.class.send(:validate_option_keys!, options, PARSE_OPTION_KEYS)
+      @options = DEFAULT_OPTIONS.merge(options)
+      validate_options
+      @input = Input.new(
+        source,
+        limits: @options.fetch(:limits),
+        filename: @options.fetch(:filename)
+      )
+    end
+
+    def parse
+      builder = RubyValueBuilder.new(
+        number_mode: @options.fetch(:number_mode),
+        string_mode: @options.fetch(:string_mode),
+        lone_surrogate: @options.fetch(:lone_surrogate),
+        duplicate_keys: @options.fetch(:duplicate_keys),
+        limits: @options.fetch(:limits),
+        diagnostics: @options.fetch(:diagnostics),
+        warn_duplicate_keys: @options.fetch(:warn_duplicate_keys),
+        freeze_values: @options.fetch(:freeze),
+        source: @input.source,
+        filename: @options.fetch(:filename)
+      )
+      lexer = Lexer.new(
+        @input,
+        limits: @options.fetch(:limits),
+        diagnostics: @options.fetch(:diagnostics),
+        filename: @options.fetch(:filename)
+      )
+      @lexer = lexer
+      generated = GeneratedParser.new(lexer, builder)
+      generated.parse_tokens
+    rescue JSON5::Error => error
+      raise enrich_error(error)
+    rescue Ibex::Runtime::ParseError => error
+      raise enrich_error(translate_parse_error(error))
+    end
+
+    private
+
+    def validate_options
+      self.class.send(:validate_parse_options!, @options)
+    end
+
+    def self.validate_option_keys!(options, allowed_keys)
+      unknown_options = options.keys - allowed_keys
+      return if unknown_options.empty?
+
+      raise ArgumentError, "unknown options: #{unknown_options.map(&:to_s).sort.join(', ')}"
+    end
+
+    def self.validate_limits_option!(limits)
+      return if limits.is_a?(Limits)
+
+      raise ArgumentError, "limits must be a JSON5::Limits"
+    end
+
+    def self.validate_diagnostics_option!(diagnostics)
+      return if diagnostics.nil? || diagnostics == :ignore || diagnostics.is_a?(Array) || diagnostics.respond_to?(:call)
+
+      raise ArgumentError, "diagnostics must be an Array, callable, :ignore, or nil"
+    end
+
+    def self.validate_boolean_option!(value, name)
+      return if value == true || value == false
+
+      raise ArgumentError, "#{name} must be true or false"
+    end
+
+    def self.validate_enum_option!(allowed, value, name)
+      return if allowed.include?(value)
+
+      raise ArgumentError, "#{name} must be one of #{allowed.inspect}"
+    end
+
+    def translate_parse_error(error)
+      internal_expected = error.respond_to?(:expected_tokens) ? contextual_expected(error.expected_tokens) : nil
+      internal_unexpected = error.respond_to?(:token_name) ? error.token_name : nil
+      trailing_tokens = internal_unexpected != "$eof" && internal_expected&.include?("$eof") &&
+        @lexer&.last_token_nesting_depth == 0
+      expected = trailing_tokens ? ["end of input"] : PublicTokenNames.expected(internal_expected)
+      unexpected = PublicTokenNames.name(internal_unexpected || "token")
+      token = @lexer&.last_token
+      message = "unexpected #{unexpected}"
+      message += "; expected #{expected.join(', ')}" if expected && !expected.empty?
+      ParseError.new(
+        message,
+        code: trailing_tokens ? "trailing_tokens" : "unexpected_token",
+        filename: @options.fetch(:filename),
+        byte_offset: token&.start_byte || @input.index,
+        end_byte_offset: token&.end_byte || @input.index,
+        line: token&.line || @input.line,
+        column: token&.column || @input.column,
+        unexpected: unexpected,
+        expected: expected
+      )
+    end
+
+    def contextual_expected(expected_tokens)
+      expected = Array(expected_tokens).dup
+      case @lexer&.last_token_container
+      when "{"
+        expected -= ["$eof", "']'"]
+      when "["
+        expected -= ["$eof", "'}'"]
+      end
+      expected
+    end
+
+    def enrich_error(error)
+      return error if error.byte_offset && error.end_byte_offset && error.line && error.column && error.excerpt
+
+      token = @lexer&.last_token
+      offset = error.byte_offset || token&.start_byte || @input.index
+      finish = error.end_byte_offset || token&.end_byte || @input.index
+      error.with_context(
+        filename: error.filename || @options.fetch(:filename),
+        byte_offset: offset,
+        end_byte_offset: finish,
+        line: error.line || token&.line || @input.line,
+        column: error.column || token&.column || @input.column,
+        excerpt: error.excerpt || @input.excerpt_at(offset)
+      )
+    end
+  end
+end

data/lib/json5/public_token_names.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module JSON5
+  module PublicTokenNames
+    VALUE_TOKENS = ["STRING", "NUMBER", "TRUE", "FALSE", "NULL", "INFINITY", "NAN", "'{'", "'['"].freeze
+    MEMBER_NAME_TOKENS = %w[STRING IDENTIFIER_NAME TRUE FALSE NULL INFINITY NAN].freeze
+    NAMES = {
+      "$eof" => "end of input",
+      "EOF" => "end of input",
+      "STRING" => "string",
+      "NUMBER" => "number",
+      "IDENTIFIER_NAME" => "identifier",
+      "TRUE" => "true",
+      "FALSE" => "false",
+      "NULL" => "null",
+      "INFINITY" => "Infinity",
+      "NAN" => "NaN"
+    }.freeze
+
+    module_function
+
+    def name(token_name)
+      internal_name = token_name.to_s
+      return NAMES.fetch(internal_name) if NAMES.key?(internal_name)
+      return internal_name[1...-1] if internal_name.start_with?("'") && internal_name.end_with?("'")
+
+      internal_name.downcase.tr("_", " ")
+    end
+
+    def expected(token_names)
+      remaining = Array(token_names).map(&:to_s)
+      public_names = []
+      if (VALUE_TOKENS - remaining).empty?
+        public_names << "value"
+        remaining -= VALUE_TOKENS
+      elsif (MEMBER_NAME_TOKENS - remaining).empty?
+        public_names << "member name"
+        remaining -= MEMBER_NAME_TOKENS
+      end
+      public_names.concat(remaining.map { |token_name| name(token_name) }).uniq.freeze
+    end
+  end
+end