json5-ruby 0.1.0

data/lib/json5/document_parser.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+module JSON5
+  class DocumentParser
+    MemberParts = Struct.new(:name, :value, :colon, :comma)
+    ElementParts = Struct.new(:elements, :commas)
+
+    def initialize(source, limits:, diagnostics:, filename:, preserve_trivia: true, collected_diagnostics: [])
+      @source = source
+      @limits = limits
+      @input = Input.new(source, limits: limits, filename: filename)
+      @lexer = Lexer.new(
+        @input,
+        limits: limits,
+        diagnostics: diagnostics,
+        filename: filename,
+        preserve_trivia: preserve_trivia
+      )
+      @filename = filename
+      @diagnostics = collected_diagnostics
+      @value_tokens = {}
+      @eof_token = nil
+    end
+
+    def parse
+      root = GeneratedParser.new(@lexer, self).parse_tokens
+      Document.new(
+        source: @source,
+        root: root,
+        diagnostics: @diagnostics,
+        trailing_trivia: @eof_token&.leading_trivia || []
+      )
+    rescue JSON5::Error => error
+      raise enrich_error(error)
+    rescue Ibex::Runtime::ParseError => error
+      raise enrich_error(translate_parse_error(error))
+    end
+
+    def observe_token(token)
+      @last_token = token
+      @value_tokens[token.value.object_id] = token if token.value
+      @eof_token = token if token.kind == :EOF
+    end
+
+    def retain_punctuation_tokens?
+      true
+    end
+
+    def string(value)
+      scalar_string(take_value_token(value))
+    end
+
+    def number(value)
+      scalar_number(take_value_token(value))
+    end
+
+    def literal(value, result)
+      scalar_literal(take_value_token(value), result)
+    end
+
+    def member_name(value)
+      token = take_value_token(value)
+      return scalar_string(token) if token.kind == :STRING
+
+      code_units = if value.is_a?(ECMAString)
+                     value
+                   else
+                     ECMAString.new(value.raw.each_codepoint.to_a)
+                   end
+      IdentifierNameNode.new(
+        code_units: code_units,
+        source: @source,
+        start_byte: token.start_byte,
+        end_byte: token.end_byte,
+        leading_trivia: token.leading_trivia
+      )
+    end
+
+    def member(name, colon, value)
+      MemberParts.new(name, value, colon, nil)
+    end
+
+    def object(members, open, close, trailing_comma = nil)
+      members.last.comma = trailing_comma if trailing_comma && !members.empty?
+      nodes = members.map do |parts|
+        MemberNode.new(
+          name: parts.name,
+          value: parts.value,
+          colon_start_byte: parts.colon.start_byte,
+          colon_end_byte: parts.colon.end_byte,
+          source: @source,
+          start_byte: parts.name.start_byte,
+          end_byte: parts.value.end_byte,
+          trailing_trivia: parts.comma ? parts.comma.leading_trivia : [],
+          colon_leading_trivia: parts.colon.leading_trivia
+        )
+      end
+      ObjectNode.new(
+        members: nodes,
+        source: @source,
+        start_byte: open.start_byte,
+        end_byte: close.end_byte,
+        leading_trivia: open.leading_trivia,
+        trailing_trivia: close.leading_trivia
+      )
+    end
+
+    def array(parts, open, close, trailing_comma = nil)
+      element_parts = parts.is_a?(ElementParts) ? parts : ElementParts.new(parts, [])
+      element_parts.commas << trailing_comma if trailing_comma
+      trailing = element_parts.elements.each_index.map do |index|
+        comma = element_parts.commas[index]
+        comma ? comma.leading_trivia : []
+      end
+      ArrayNode.new(
+        elements: element_parts.elements,
+        element_trailing_trivia: trailing,
+        source: @source,
+        start_byte: open.start_byte,
+        end_byte: close.end_byte,
+        leading_trivia: open.leading_trivia,
+        trailing_trivia: close.leading_trivia
+      )
+    end
+
+    def member_list(member)
+      enforce_entry_limit(1)
+      [member]
+    end
+
+    def append_member(members, member, comma)
+      enforce_entry_limit(members.length + 1)
+      members.last.comma = comma
+      members << member
+    end
+
+    def element_list(element)
+      enforce_entry_limit(1)
+      ElementParts.new([element], [])
+    end
+
+    def append_element(parts, element, comma)
+      enforce_entry_limit(parts.elements.length + 1)
+      parts.commas << comma
+      parts.elements << element
+      parts
+    end
+
+    private
+
+    def take_value_token(value)
+      @value_tokens.delete(value.object_id) || raise("missing scalar token")
+    end
+
+    def scalar_string(token)
+      StringNode.new(
+        code_units: token.value,
+        quote: @source.getbyte(token.start_byte),
+        escape_spans: string_escape_spans(token),
+        source: @source,
+        start_byte: token.start_byte,
+        end_byte: token.end_byte,
+        leading_trivia: token.leading_trivia
+      )
+    end
+
+    def scalar_number(token)
+      value = token.value.is_a?(NumberValue) ? token.value : NumberValue.new(token.value)
+      classification = %i[integer float].include?(value.kind) ? :decimal : value.kind
+      NumberNode.new(
+        classification: classification,
+        sign: value.raw.start_with?("-") ? :- : value.raw.start_with?("+") ? :+ : nil,
+        source: @source,
+        start_byte: token.start_byte,
+        end_byte: token.end_byte,
+        leading_trivia: token.leading_trivia
+      )
+    end
+
+    def scalar_literal(token, value)
+      LiteralNode.new(
+        value: value,
+        source: @source,
+        start_byte: token.start_byte,
+        end_byte: token.end_byte,
+        leading_trivia: token.leading_trivia
+      )
+    end
+
+    def string_escape_spans(token)
+      spans = []
+      index = token.start_byte + 1
+      content_end = token.end_byte - 1
+      while index < content_end
+        unless @source.getbyte(index) == 0x5c
+          index += utf8_width(@source.getbyte(index))
+          next
+        end
+
+        start_byte = index
+        index = escape_end_byte(index + 1)
+        spans << ByteSpan.new(start_byte, index)
+      end
+      spans
+    end
+
+    def escape_end_byte(marker_byte)
+      marker = @source.getbyte(marker_byte)
+      return marker_byte + 3 if marker == 0x78
+      return marker_byte + 5 if marker == 0x75
+      return marker_byte + 2 if marker == 0x0d && @source.getbyte(marker_byte + 1) == 0x0a
+
+      marker_byte + utf8_width(marker)
+    end
+
+    def utf8_width(byte)
+      return 1 if byte < 0x80
+      return 2 if byte < 0xe0
+      return 3 if byte < 0xf0
+
+      4
+    end
+
+    def enforce_entry_limit(count)
+      return unless @limits.max_container_entries && count > @limits.max_container_entries
+
+      token = @last_token
+      offset = token&.start_byte || @input.index
+      raise LimitError.new(
+        "container exceeds max_container_entries",
+        code: "maximum_container_entries_exceeded",
+        filename: @filename,
+        byte_offset: offset,
+        end_byte_offset: token&.end_byte || offset,
+        line: token&.line || @input.line,
+        column: token&.column || @input.column,
+        excerpt: @input.excerpt_at(offset)
+      )
+    end
+
+    def translate_parse_error(error)
+      internal_expected = error.respond_to?(:expected_tokens) ? contextual_expected(error.expected_tokens) : nil
+      internal_unexpected = error.respond_to?(:token_name) ? error.token_name : nil
+      trailing_tokens = internal_unexpected != "$eof" && internal_expected&.include?("$eof") &&
+        @lexer.last_token_nesting_depth == 0
+      expected = trailing_tokens ? ["end of input"] : PublicTokenNames.expected(internal_expected)
+      unexpected = PublicTokenNames.name(internal_unexpected || "token")
+      token = @lexer.last_token
+      message = "unexpected #{unexpected}"
+      message += "; expected #{expected.join(', ')}" if expected && !expected.empty?
+      ParseError.new(
+        message,
+        code: trailing_tokens ? "trailing_tokens" : "unexpected_token",
+        filename: @filename,
+        byte_offset: token&.start_byte || @input.index,
+        end_byte_offset: token&.end_byte || @input.index,
+        line: token&.line || @input.line,
+        column: token&.column || @input.column,
+        unexpected: unexpected,
+        expected: expected
+      )
+    end
+
+    def contextual_expected(expected_tokens)
+      expected = Array(expected_tokens).dup
+      case @lexer.last_token_container
+      when "{"
+        expected -= ["$eof", "']'"]
+      when "["
+        expected -= ["$eof", "'}'"]
+      end
+      expected
+    end
+
+    def enrich_error(error)
+      return error if error.byte_offset && error.end_byte_offset && error.line && error.column && error.excerpt
+
+      token = @lexer.last_token
+      offset = error.byte_offset || token&.start_byte || @input.index
+      finish = error.end_byte_offset || token&.end_byte || @input.index
+      error.with_context(
+        filename: error.filename || @filename,
+        byte_offset: offset,
+        end_byte_offset: finish,
+        line: error.line || token&.line || @input.line,
+        column: error.column || token&.column || @input.column,
+        excerpt: error.excerpt || @input.excerpt_at(offset)
+      )
+    end
+  end
+end

data/lib/json5/ecma_string.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module JSON5
+  class ECMAString
+    attr_reader :code_units
+
+    def initialize(code_units)
+      unless code_units.all? { |unit| unit.is_a?(Integer) && unit.between?(0, 0xffff) }
+        raise ArgumentError, "code units must be UTF-16 integers"
+      end
+      @code_units = code_units.dup.freeze
+      freeze
+    end
+
+    def each_code_unit(&block)
+      return enum_for(__method__) unless block
+      code_units.each(&block)
+    end
+
+    def length
+      code_units.length
+    end
+
+    def empty?
+      code_units.empty?
+    end
+
+    def eql?(other)
+      other.is_a?(ECMAString) && code_units == other.code_units
+    end
+    alias == eql?
+
+    def hash
+      code_units.hash
+    end
+
+    def lone_surrogate?
+      index = 0
+      while index < code_units.length
+        unit = code_units[index]
+        if surrogate_pair_at?(index)
+          index += 2
+          next
+        end
+        return true if high_surrogate?(unit) || low_surrogate?(unit)
+
+        index += 1
+      end
+      false
+    end
+
+    def to_utf8(lone_surrogate: :error)
+      result = String.new(encoding: Encoding::UTF_8)
+      index = 0
+      while index < code_units.length
+        unit = code_units[index]
+        if surrogate_pair_at?(index)
+          codepoint = 0x10000 + ((unit - 0xd800) << 10) + code_units[index + 1] - 0xdc00
+          append_codepoint(result, codepoint)
+          index += 2
+          next
+        end
+
+        if high_surrogate?(unit) || low_surrogate?(unit)
+          case lone_surrogate
+          when :error
+            raise ConversionError.new(
+              "lone surrogate cannot be represented as UTF-8",
+              code: "lone_surrogate"
+            )
+          when :replace
+            append_codepoint(result, 0xfffd)
+          else
+            raise ArgumentError, "unknown lone_surrogate policy: #{lone_surrogate.inspect}"
+          end
+        else
+          append_codepoint(result, unit)
+        end
+        index += 1
+      end
+      result
+    end
+
+    def to_s
+      to_utf8
+    end
+
+    private
+
+    def high_surrogate?(unit)
+      unit.between?(0xd800, 0xdbff)
+    end
+
+    def low_surrogate?(unit)
+      unit.between?(0xdc00, 0xdfff)
+    end
+
+    def surrogate_pair_at?(index)
+      high_surrogate?(code_units[index]) && index + 1 < code_units.length && low_surrogate?(code_units[index + 1])
+    end
+
+    def append_codepoint(result, codepoint)
+      result << [codepoint].pack("U")
+    end
+  end
+end

data/lib/json5/errors.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module JSON5
+  class Error < StandardError
+    attr_reader :code, :filename, :byte_offset, :end_byte_offset, :line,
+      :column, :unexpected, :expected, :excerpt
+
+    def initialize(message = nil, code: "parse_error", filename: nil,
+                   byte_offset: nil, end_byte_offset: nil, line: nil,
+                   column: nil, unexpected: nil, expected: nil, excerpt: nil)
+      @code = code.to_s.freeze
+      @filename = filename
+      @byte_offset = byte_offset
+      @end_byte_offset = end_byte_offset
+      @line = line
+      @column = column
+      @unexpected = unexpected
+      @expected = expected&.map(&:to_s)&.freeze
+      @excerpt = excerpt
+      super(message || @code)
+    end
+
+    def to_h
+      {
+        code: code,
+        message: message,
+        filename: filename,
+        byte_offset: byte_offset,
+        end_byte_offset: end_byte_offset,
+        line: line,
+        column: column,
+        unexpected: unexpected,
+        expected: expected,
+        excerpt: excerpt
+      }
+    end
+
+    def with_context(filename: self.filename, byte_offset: self.byte_offset,
+                     end_byte_offset: self.end_byte_offset, line: self.line,
+                     column: self.column, excerpt: self.excerpt)
+      return self if self.byte_offset && self.end_byte_offset && self.line && self.column && self.excerpt
+
+      enriched = self.class.new(
+        message,
+        code: code,
+        filename: filename,
+        byte_offset: byte_offset,
+        end_byte_offset: end_byte_offset,
+        line: line,
+        column: column,
+        unexpected: unexpected,
+        expected: expected,
+        excerpt: excerpt
+      )
+      enriched.set_backtrace(backtrace) if backtrace
+      enriched
+    end
+  end
+
+  class EncodingError < Error; end
+  class LexError < Error; end
+  class ParseError < Error; end
+  class DuplicateKeyError < ParseError; end
+  class DepthError < Error; end
+  class LimitError < Error; end
+  class ConversionError < Error; end
+end

data/lib/json5/generated_parser.ibex.json

@@ -0,0 +1,196 @@
+{
+  "ibex_manifest": "generation",
+  "schema_version": 1,
+  "input": {
+    "root": "grammar/json5.y",
+    "sha256": "bf1302cb95d92012fa9d462837b4bb7b0f165d13cea80e1e1724d98c383ee230",
+    "files": [
+      {
+        "path": "grammar/json5.y",
+        "sha256": "ca1490daccedcc6067afc95d175173bd3453c416007555938ed934d74cbd7183",
+        "bytesize": 2345
+      }
+    ]
+  },
+  "options": {
+    "algorithm": "lalr",
+    "automaton_ir": {
+      "algorithm": "lalr1",
+      "construction_authority": "grammar_contract",
+      "entry_construction": "shared",
+      "schema_version": 1
+    },
+    "counterexample_max_configurations": 50000,
+    "counterexample_max_tokens": 32,
+    "cst_trivia": "leading",
+    "effective_configuration": [
+      {
+        "key": "actions.omit_calls",
+        "value": null,
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "build.debug",
+        "value": false,
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "build.executable",
+        "value": null,
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "build.frozen_strings",
+        "value": true,
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "cli"
+        },
+        "explicit": true,
+        "canonical": true
+      },
+      {
+        "key": "cst.trivia",
+        "value": "leading",
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "grammar.mode",
+        "value": "default",
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "parser.algorithm",
+        "value": "lalr",
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "cli"
+        },
+        "explicit": true,
+        "canonical": true
+      },
+      {
+        "key": "parser.entries",
+        "value": "shared",
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "parser.superclass",
+        "value": null,
+        "owner": "grammar-contract",
+        "policy": "fixed",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "runtime.embedded",
+        "value": true,
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "cli"
+        },
+        "explicit": true,
+        "canonical": true
+      },
+      {
+        "key": "source.line_mapping",
+        "value": "actions",
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      },
+      {
+        "key": "table.representation",
+        "value": "compact",
+        "owner": "project-build",
+        "policy": "build",
+        "origin": {
+          "kind": "builtin"
+        },
+        "explicit": false,
+        "canonical": true
+      }
+    ],
+    "embedded": true,
+    "emit": "ruby",
+    "frozen": true,
+    "grammar_ir": {
+      "digest": "sha256:ad3f3c01a5895b10f17bd4c512aa8f073f651d9b7a623a6df9a4b5536c5d1a18",
+      "parser_contract": {
+        "algorithm": {
+          "explicit": false,
+          "loc": null,
+          "value": null
+        },
+        "cst_trivia": {
+          "explicit": false,
+          "loc": null,
+          "value": null
+        },
+        "entries": {
+          "explicit": false,
+          "loc": null,
+          "value": null
+        }
+      },
+      "schema_version": 1
+    },
+    "line_convert": true,
+    "mode": "default",
+    "table": "compact"
+  },
+  "artifacts": [
+    {
+      "kind": "parser",
+      "path": "lib/json5/generated_parser.rb",
+      "sha256": "2ae44411e7a2f97a35d9b3a1083921baa47615f7e69e0a94f1751866d762b722",
+      "bytesize": 392768
+    }
+  ]
+}