red_quilt 0.6.1

data/lib/red_quilt/inline/lexer.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+require_relative "html_entities"
+
+module RedQuilt
+  module Inline
+    # Scans a byte range of the document source and emits inline tokens
+    # into a caller-owned Tokens storage.
+    #
+    # The lexer never copies the source string; all positions are absolute
+    # byte offsets into @source. The caller is responsible for clearing the
+    # Tokens storage between invocations if it is being reused.
+    class Lexer
+      # Bytes whose appearance ends a TEXT run. Anything not in this set is
+      # plain text content. Newline is included so LINE_ENDING gets its own
+      # token.
+      SPECIAL_BYTES = begin
+        a = Array.new(256, false)
+        # *, _, `, [, ], !, <, &, \, \n, ~ (GFM strikethrough)
+        [0x2A, 0x5F, 0x60, 0x5B, 0x5D, 0x21, 0x3C, 0x26, 0x5C, 0x0A, 0x7E].each { |b| a[b] = true }
+        a.freeze
+      end
+      # Same set as SPECIAL_BYTES, for String#byteindex to jump over long
+      # plain-text runs at C speed.
+      SPECIAL_BYTE_RE = /[*_`\[\]!<&\\\n~]/
+
+      # Anchored regexes for StringScanner#scan (still used by
+      # scan_angle / scan_amp). StringScanner anchors at the current pos,
+      # so no `\G` is needed.
+      #
+      # URI autolink rejects every ASCII control char (U+0000-U+001F, U+007F)
+      # plus space (U+0020); CommonMark 6.5 forbids ASCII control characters,
+      # space, <, or >.
+      URI_AUTOLINK_RE = /<([A-Za-z][A-Za-z0-9+.-]{1,31}:[^<>\u0000-\u0020\u007F]*)>/
+      EMAIL_AUTOLINK_RE = /<([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>/
+      # CommonMark spec 6.6 "Raw HTML": six forms — open tag, closing tag,
+      # HTML comment, processing instruction, declaration, CDATA section.
+      # Attribute values are allowed to span lines.
+      # HTML tag separators are restricted to space/tab/CR/LF per spec --
+      # \s would also match form feed (U+000C) and vertical tab (U+000B),
+      # which CommonMark disallows.
+      HTML_OPEN_TAG_RE = %r{<[A-Za-z][A-Za-z0-9-]*(?:[ \t\r\n]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t\r\n]*=[ \t\r\n]*(?:"[^"]*"|'[^']*'|[^ \t\r\n"'=<>`]+))?)*[ \t\r\n]*/?>}
+      HTML_CLOSING_TAG_RE = %r{</[A-Za-z][A-Za-z0-9-]*[ \t\r\n]*>}
+      # Comment: `<!-->`, `<!--->`, or `<!-- text -->` where text doesn't
+      # start with `>` or `->`, end with `-`, or contain `--`.
+      HTML_COMMENT_RE = %r{<!-->|<!--->|<!--(?!>)(?!->)[\s\S]*?(?<!-)-->}
+      HTML_PROC_INST_RE = %r{<\?[\s\S]*?\?>}
+      HTML_DECLARATION_RE = %r{<![A-Za-z][^>]*>}
+      HTML_CDATA_RE = %r{<!\[CDATA\[[\s\S]*?\]\]>}
+      # Entity regex and decoder live on the enclosing Inline module so
+      # the same digit-count caps and U+FFFD replacement apply across
+      # the lexer, the inline builder, and the reference-definition
+      # parser. See lib/red_quilt/inline/html_entities.rb.
+
+      def initialize(source)
+        @source = source
+        # A binary-encoded view for String#byteindex hot paths (byteindex
+        # on a UTF-8 string raises when the offset falls inside a
+        # multibyte sequence; binary treats every byte as its own char).
+        @source_b = source.b
+        @ss = StringScanner.new(source)
+      end
+
+      # Scans @source[start_byte...end_byte] and emits tokens.
+      # Returns the tokens object that was passed in.
+      def lex_into(tokens, start_byte, end_byte)
+        @ss.pos = start_byte
+        @start = start_byte
+        @end = end_byte
+        scan(tokens)
+        tokens
+      end
+
+      private
+
+      def scan(tokens)
+        # Hot loop. `pos` is the source of truth during the scan; @ss.pos
+        # is only synced when entering scan_angle / scan_amp (which still
+        # use StringScanner for the regex match) and at loop exit. The
+        # other scan_* helpers take `pos` as an arg and return the new
+        # position, so the round-trip through @ss.pos is avoided.
+        pos = @ss.pos
+        end_pos = @end
+        while pos < end_pos
+          byte = @source.getbyte(pos)
+          case byte
+          when 0x0A # \n
+            pos = scan_line_ending(tokens, pos)
+          when 0x5C # \\ (backslash)
+            pos = scan_backslash(tokens, pos, end_pos)
+          when 0x60 # `
+            pos = scan_code_delimiter(tokens, pos, end_pos)
+          when 0x2A # *
+            pos = scan_delim_run(tokens, pos, end_pos, "*", 0x2A)
+          when 0x5F # _
+            pos = scan_delim_run(tokens, pos, end_pos, "_", 0x5F)
+          when 0x7E # ~ (GFM strikethrough)
+            pos = scan_delim_run(tokens, pos, end_pos, "~", 0x7E)
+          when 0x5B # [
+            tokens.emit(TokenKind::LBRACKET, start_byte: pos, end_byte: pos + 1)
+            pos += 1
+          when 0x5D # ]
+            tokens.emit(TokenKind::RBRACKET, start_byte: pos, end_byte: pos + 1)
+            pos += 1
+          when 0x21 # !
+            pos = scan_bang(tokens, pos, end_pos)
+          when 0x3C # <
+            @ss.pos = pos
+            scan_angle(tokens)
+            pos = @ss.pos
+          when 0x26 # &
+            @ss.pos = pos
+            scan_amp(tokens)
+            pos = @ss.pos
+          else
+            # Inlined scan_text. Always make progress: consume the
+            # current byte, then byteindex against the binary view to
+            # leap to the next special byte at C speed.
+            start = pos
+            pos += 1
+            if pos < end_pos
+              next_special = @source_b.byteindex(SPECIAL_BYTE_RE, pos)
+              pos = next_special.nil? || next_special >= end_pos ? end_pos : next_special
+            end
+            tokens.emit(TokenKind::TEXT, start_byte: start, end_byte: pos)
+          end
+        end
+        @ss.pos = pos
+      end
+
+      def scan_line_ending(tokens, pos)
+        # Count trailing ASCII spaces immediately before the newline; the
+        # builder uses this to decide softbreak vs hardbreak (>= 2 spaces).
+        trailing_spaces = 0
+        i = pos - 1
+        while i >= 0 && @source.getbyte(i) == 0x20
+          trailing_spaces += 1
+          i -= 1
+        end
+        new_pos = pos + 1
+        tokens.emit(TokenKind::LINE_ENDING,
+                    start_byte: pos, end_byte: new_pos,
+                    int1: trailing_spaces)
+        new_pos
+      end
+
+      def scan_backslash(tokens, pos, end_pos)
+        nxt_pos = pos + 1
+        if nxt_pos >= end_pos
+          tokens.emit(TokenKind::TEXT, start_byte: pos, end_byte: nxt_pos)
+          return nxt_pos
+        end
+
+        nxt = @source.getbyte(nxt_pos)
+        if nxt == 0x0A
+          # "\\\n" → hardbreak (backslash form). int2 = 1 signals the form.
+          tokens.emit(TokenKind::LINE_ENDING,
+                      start_byte: pos, end_byte: nxt_pos + 1,
+                      int1: 0, int2: 1)
+          nxt_pos + 1
+        elsif Inline.ascii_punct_byte?(nxt)
+          tokens.emit(TokenKind::ESCAPED_CHAR,
+                      start_byte: pos, end_byte: nxt_pos + 1,
+                      str1: nxt.chr)
+          nxt_pos + 1
+        else
+          tokens.emit(TokenKind::TEXT, start_byte: pos, end_byte: nxt_pos)
+          nxt_pos
+        end
+      end
+
+      def scan_code_delimiter(tokens, pos, end_pos)
+        # Manual byte loop. Backtick runs are usually short (1-3 bytes),
+        # so a regex skip's setup cost outweighs the per-byte compare.
+        i = pos
+        while i < end_pos && @source.getbyte(i) == 0x60
+          i += 1
+        end
+        tokens.emit(TokenKind::CODE_DELIMITER,
+                    start_byte: pos, end_byte: i,
+                    int1: i - pos)
+        i
+      end
+
+      def scan_delim_run(tokens, pos, end_pos, char, byte)
+        i = pos
+        while i < end_pos && @source.getbyte(i) == byte
+          i += 1
+        end
+        count = i - pos
+        prev_char = Flanking.char_before(@source, pos, @start)
+        next_char = Flanking.char_at(@source, i, end_pos)
+        can_open, can_close = Flanking.can_open_close(char, prev_char, next_char)
+        # A run that can neither open nor close (e.g. underscores inside
+        # a word) can never participate in emphasis, so emit it as plain
+        # TEXT to allow text coalescing with neighbours.
+        if !can_open && !can_close
+          tokens.emit(TokenKind::TEXT, start_byte: pos, end_byte: i)
+          return i
+        end
+        flags = (can_open ? 0b10 : 0) | (can_close ? 0b01 : 0)
+        tokens.emit(TokenKind::DELIM_RUN,
+                    start_byte: pos, end_byte: i,
+                    int1: byte, int2: count, int3: flags)
+        i
+      end
+
+      def scan_bang(tokens, pos, end_pos)
+        if pos + 1 < end_pos && @source.getbyte(pos + 1) == 0x5B # [
+          tokens.emit(TokenKind::BANG_LBRACKET, start_byte: pos, end_byte: pos + 2)
+          pos + 2
+        else
+          tokens.emit(TokenKind::TEXT, start_byte: pos, end_byte: pos + 1)
+          pos + 1
+        end
+      end
+
+      def scan_angle(tokens)
+        start = @ss.pos
+        if scan_within_end(URI_AUTOLINK_RE)
+          tokens.emit(TokenKind::AUTOLINK_URI,
+                      start_byte: start, end_byte: @ss.pos,
+                      str1: @ss[1])
+        elsif scan_within_end(EMAIL_AUTOLINK_RE)
+          tokens.emit(TokenKind::AUTOLINK_EMAIL,
+                      start_byte: start, end_byte: @ss.pos,
+                      str1: @ss[1])
+        elsif (matched = scan_within_end(HTML_OPEN_TAG_RE)) ||
+              (matched = scan_within_end(HTML_CLOSING_TAG_RE)) ||
+              (matched = scan_within_end(HTML_COMMENT_RE)) ||
+              (matched = scan_within_end(HTML_PROC_INST_RE)) ||
+              (matched = scan_within_end(HTML_DECLARATION_RE)) ||
+              (matched = scan_within_end(HTML_CDATA_RE))
+          tokens.emit(TokenKind::HTML_INLINE,
+                      start_byte: start, end_byte: @ss.pos,
+                      str1: matched)
+        else
+          @ss.pos += 1
+          tokens.emit(TokenKind::TEXT, start_byte: start, end_byte: @ss.pos)
+        end
+      end
+
+      def scan_amp(tokens)
+        start = @ss.pos
+        if (matched = scan_within_end(Inline::ENTITY_RE))
+          tokens.emit(TokenKind::ENTITY,
+                      start_byte: start, end_byte: @ss.pos,
+                      str1: decode_entity(matched))
+        else
+          @ss.pos += 1
+          tokens.emit(TokenKind::TEXT, start_byte: start, end_byte: @ss.pos)
+        end
+      end
+
+      # StringScanner#scan but constrained to @end. Returns the matched
+      # string on success (rewinding when the match extends past @end),
+      # nil otherwise.
+      def scan_within_end(regex)
+        before = @ss.pos
+        matched = @ss.scan(regex)
+        return nil unless matched
+
+        if @ss.pos > @end
+          @ss.pos = before
+          return nil
+        end
+        matched
+      end
+
+      # Decodes a single entity reference using the shared Inline
+      # decoder, which enforces the spec's digit limits and U+FFFD
+      # replacement for U+0000 / surrogates / out-of-range codepoints.
+      def decode_entity(raw)
+        Inline.decode_entity(raw)
+      end
+    end
+  end
+end

data/lib/red_quilt/inline/link_scanner.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module RedQuilt
+  module Inline
+    # Pure byte-level scanner for link / image tails: inline link bodies
+    # `(dest "title")`, bracketed reference labels `[label]`, and link
+    # destination URI normalization. Operates only on the document source
+    # string -- no arena, token stream, or parser state -- so it can be
+    # exercised in isolation. Inline::Builder owns one instance and feeds
+    # it absolute byte offsets.
+    class LinkScanner
+      NIL_PAIR = [nil, nil].freeze
+      # Bytes left verbatim by normalize_uri: ASCII alphanumerics plus the
+      # URL sub-delims / reserved chars that the spec keeps unencoded.
+      # Everything else is percent-encoded.
+      URL_SAFE_BYTE = begin
+        a = Array.new(256, false)
+        (0x30..0x39).each { |b| a[b] = true } # 0-9
+        (0x41..0x5A).each { |b| a[b] = true } # A-Z
+        (0x61..0x7A).each { |b| a[b] = true } # a-z
+        "-._~:/?#@!$&'()*+,;=".each_byte { |b| a[b] = true }
+        a.freeze
+      end
+
+      def initialize(source)
+        @source = source
+      end
+
+      # Parses an inline link body `(dest "title")` starting at the byte
+      # right after the link's closing `]`. Returns a hash with
+      # `:end_byte`, `:destination`, `:title` on success, or nil if the
+      # bytes don't form a valid inline link tail.
+      def inline_link(start_byte)
+        return nil unless byte_at(start_byte) == 0x28
+
+        pos = start_byte + 1
+        pos = skip_link_whitespace(pos)
+        return nil if pos.nil?
+
+        raw_dest = nil
+        next_byte = byte_at(pos)
+        if next_byte && next_byte != 0x29 && !link_tail_whitespace_byte?(next_byte) && next_byte != 0x0A
+          dest_result = parse_link_destination(pos)
+          return nil unless dest_result
+
+          raw_dest, pos = dest_result
+        end
+
+        ws_end = skip_link_whitespace(pos)
+        return nil if ws_end.nil?
+
+        raw_title = nil
+        if ws_end > pos
+          opener_byte = byte_at(ws_end)
+          if opener_byte && (opener_byte == 0x22 || opener_byte == 0x27 || opener_byte == 0x28)
+            title_result = parse_link_title(ws_end)
+            return nil unless title_result
+
+            raw_title, pos = title_result
+            pos = skip_link_whitespace(pos)
+            return nil if pos.nil?
+          else
+            pos = ws_end
+          end
+        else
+          pos = ws_end
+        end
+
+        return nil unless byte_at(pos) == 0x29
+
+        destination = raw_dest ? normalize_uri(raw_dest) : ""
+        title = raw_title ? decode_link_entities(raw_title) : nil
+        { end_byte: pos + 1, destination: destination, title: title }
+      end
+
+      # Reads a bracketed reference label `[label]` starting at start_byte
+      # (which must point at the `[`). Returns [label, after_byte] or
+      # NIL_PAIR when the label is malformed or over-long.
+      def reference_label(start_byte)
+        return NIL_PAIR unless @source.getbyte(start_byte) == 0x5B
+
+        i = start_byte + 1
+        while i < @source.bytesize
+          b = @source.getbyte(i)
+          if b == 0x5D
+            label = @source.byteslice(start_byte + 1, i - start_byte - 1).to_s
+            return NIL_PAIR if ReferenceDefinition.label_too_long?(label)
+
+            return [label, i + 1]
+          elsif b == 0x5B
+            # An unescaped `[` inside a reference label voids the form.
+            return NIL_PAIR
+          elsif b == 0x5C && i + 1 < @source.bytesize
+            i += 2
+            next
+          end
+          i += 1
+        end
+        NIL_PAIR
+      end
+
+      # Percent-encodes bytes not in the URL-safe set, decodes HTML
+      # entities first, and preserves (uppercasing) existing `%XX`.
+      def normalize_uri(raw)
+        decoded = decode_link_entities(raw)
+        bytes = decoded.b
+        result = +""
+        i = 0
+        size = bytes.bytesize
+        while i < size
+          b = bytes.getbyte(i)
+          if b == 0x25 && i + 2 < size &&
+             hex_byte?(bytes.getbyte(i + 1)) && hex_byte?(bytes.getbyte(i + 2))
+            result << "%"
+            result << bytes.getbyte(i + 1).chr.upcase
+            result << bytes.getbyte(i + 2).chr.upcase
+            i += 3
+          elsif URL_SAFE_BYTE[b]
+            # All URL-safe bytes are ASCII, so appending the integer
+            # codepoint matches b.chr without allocating a 1-char string.
+            result << b
+            i += 1
+          else
+            result << format("%%%02X", b)
+            i += 1
+          end
+        end
+        result
+      end
+
+      private
+
+      # Consume ASCII whitespace starting at start_byte. Returns the
+      # position of the first non-whitespace byte, or nil if a blank line
+      # was crossed (link inner whitespace may span at most one newline).
+      def skip_link_whitespace(start_byte)
+        pos = start_byte
+        newlines = 0
+        while pos < @source.bytesize
+          b = @source.getbyte(pos)
+          if b == 0x0A
+            newlines += 1
+            return nil if newlines > 1
+          elsif !link_tail_whitespace_byte?(b)
+            break
+          end
+          pos += 1
+        end
+        pos
+      end
+
+      def parse_link_destination(start_byte)
+        if byte_at(start_byte) == 0x3C
+          parse_angle_bracket_destination(start_byte)
+        else
+          parse_raw_destination(start_byte)
+        end
+      end
+
+      # `<...>` form. Returns [string_with_backslash_escapes_applied, end_pos]
+      # or nil. Inside angles, `\` followed by ASCII punctuation escapes that
+      # punctuation; unescaped `<`, `>` or newlines bail the parse.
+      def parse_angle_bracket_destination(start_byte)
+        pos = start_byte + 1
+        result = String.new
+        while pos < @source.bytesize
+          b = @source.getbyte(pos)
+          case b
+          when 0x3E
+            return [result, pos + 1]
+          when 0x3C, 0x0A
+            return nil
+          when 0x5C
+            nb = @source.getbyte(pos + 1)
+            if nb && Inline.ascii_punct_byte?(nb)
+              result << nb
+              pos += 2
+              next
+            end
+            result << b
+          else
+            result << b
+          end
+          pos += 1
+        end
+        nil
+      end
+
+      # Raw destination: characters until ASCII whitespace, an ASCII
+      # control char, or an unbalanced `)`. Parens are allowed if balanced
+      # or backslash-escaped.
+      def parse_raw_destination(start_byte)
+        pos = start_byte
+        depth = 0
+        result = String.new
+        while pos < @source.bytesize
+          b = @source.getbyte(pos)
+          if b == 0x5C
+            nb = @source.getbyte(pos + 1)
+            if nb && Inline.ascii_punct_byte?(nb)
+              result << nb
+              pos += 2
+              next
+            end
+            result << b
+            pos += 1
+            next
+          end
+
+          break if link_tail_whitespace_byte?(b) || b < 0x20 || b == 0x7F
+
+          if b == 0x28
+            depth += 1
+          elsif b == 0x29
+            break if depth.zero?
+
+            depth -= 1
+          end
+
+          result << b
+          pos += 1
+        end
+
+        return nil if pos == start_byte
+        return nil if depth != 0
+
+        [result, pos]
+      end
+
+      # Parses a title delimited by `"`, `'`, or `(...)`. Returns
+      # [unescaped_string, end_pos] or nil. Backslash escapes apply for
+      # ASCII punctuation; a blank line inside a title voids the match.
+      def parse_link_title(start_byte)
+        opener = @source.getbyte(start_byte)
+        closer = case opener
+                 when 0x22 then 0x22
+                 when 0x27 then 0x27
+                 when 0x28 then 0x29
+                 else return nil
+                 end
+        balanced = opener == 0x28
+
+        pos = start_byte + 1
+        result = String.new
+        while pos < @source.bytesize
+          b = @source.getbyte(pos)
+          if b == 0x5C
+            nb = @source.getbyte(pos + 1)
+            if nb && Inline.ascii_punct_byte?(nb)
+              result << nb
+              pos += 2
+              next
+            end
+            result << b
+            pos += 1
+            next
+          end
+
+          if b == 0x0A
+            # Blank line (newline followed by only whitespace + newline) is forbidden.
+            look = pos + 1
+            while look < @source.bytesize && (@source.getbyte(look) == 0x20 || @source.getbyte(look) == 0x09)
+              look += 1
+            end
+            return nil if look < @source.bytesize && @source.getbyte(look) == 0x0A
+
+            result << b
+            pos += 1
+            next
+          end
+
+          # Inside `(...)` titles, an unescaped opening `(` invalidates the match.
+          return nil if balanced && b == 0x28
+
+          if b == closer
+            return [result, pos + 1]
+          end
+
+          result << b
+          pos += 1
+        end
+        nil
+      end
+
+      def decode_link_entities(raw)
+        # An entity reference always contains `&`; skip the regex scan and
+        # the new-string allocation when there's nothing to decode.
+        return raw unless raw.include?("&")
+
+        raw.gsub(Inline::ENTITY_RE) { |m| Inline.decode_entity(m) }
+      end
+
+      def byte_at(pos)
+        return nil if pos < 0 || pos >= @source.bytesize
+
+        @source.getbyte(pos)
+      end
+
+      # Whitespace allowed as a link-tail separator per CommonMark 6.3:
+      # "spaces, tabs, and up to one line ending". Line endings are
+      # counted by the caller, so this predicate intentionally matches
+      # only space and tab -- it must NOT match form feed (U+000C) or
+      # vertical tab (U+000B) the way the generic \s class does.
+      def link_tail_whitespace_byte?(b)
+        b == 0x20 || b == 0x09
+      end
+
+      def hex_byte?(b)
+        (b >= 0x30 && b <= 0x39) ||
+          (b >= 0x41 && b <= 0x46) ||
+          (b >= 0x61 && b <= 0x66)
+      end
+    end
+  end
+end

data/lib/red_quilt/inline/token_kind.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RedQuilt
+  module Inline
+    module TokenKind
+      TEXT           = 1
+      ENTITY         = 2
+      ESCAPED_CHAR   = 3
+      LINE_ENDING    = 4
+      CODE_DELIMITER = 5
+      DELIM_RUN      = 6
+      LBRACKET       = 7
+      BANG_LBRACKET  = 8
+      RBRACKET       = 9
+      AUTOLINK_URI   = 10
+      AUTOLINK_EMAIL = 11
+      HTML_INLINE    = 12
+
+      NAMES = {
+        TEXT => :text,
+        ENTITY => :entity,
+        ESCAPED_CHAR => :escaped_char,
+        LINE_ENDING => :line_ending,
+        CODE_DELIMITER => :code_delimiter,
+        DELIM_RUN => :delim_run,
+        LBRACKET => :lbracket,
+        BANG_LBRACKET => :bang_lbracket,
+        RBRACKET => :rbracket,
+        AUTOLINK_URI => :autolink_uri,
+        AUTOLINK_EMAIL => :autolink_email,
+        HTML_INLINE => :html_inline,
+      }.freeze
+
+      def self.name(kind)
+        NAMES[kind]
+      end
+    end
+  end
+end

data/lib/red_quilt/inline/tokens.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module RedQuilt
+  module Inline
+    # Parallel-array storage for the inline token stream.
+    #
+    # InlineTokens is intended to be allocated once per document and reused
+    # across paragraphs by calling #clear between inline targets. Array#clear
+    # resets length to 0 while preserving internal capacity, so subsequent
+    # paragraphs avoid reallocating the underlying buffers.
+    class Tokens
+      def initialize
+        @kind = []
+        @start_byte = []
+        @end_byte = []
+        @int1 = []
+        @int2 = []
+        @int3 = []
+        @str1 = []
+      end
+
+      def emit(kind, start_byte:, end_byte:, int1: 0, int2: 0, int3: 0, str1: nil)
+        id = @kind.length
+        @kind[id] = kind
+        @start_byte[id] = start_byte
+        @end_byte[id] = end_byte
+        @int1[id] = int1
+        @int2[id] = int2
+        @int3[id] = int3
+        @str1[id] = str1
+        id
+      end
+
+      def clear
+        @kind.clear
+        @start_byte.clear
+        @end_byte.clear
+        @int1.clear
+        @int2.clear
+        @int3.clear
+        @str1.clear
+        self
+      end
+
+      def length
+        @kind.length
+      end
+
+      def empty?
+        @kind.empty?
+      end
+
+      def kind(id) = @kind.[](id)
+      def start_byte(id) = @start_byte.[](id)
+      def end_byte(id) = @end_byte.[](id)
+      def int1(id) = @int1.[](id)
+      def int2(id) = @int2.[](id)
+      def int3(id) = @int3.[](id)
+      def str1(id) = @str1.[](id)
+
+      def each_id
+        return enum_for(:each_id) unless block_given?
+
+        id = 0
+        last = @kind.length
+        while id < last
+          yield id
+          id += 1
+        end
+      end
+    end
+  end
+end