hypertabular 0.7.0-x86_64-linux-musl

data/lib/hypertabular/delimited_reader.rb

@@ -0,0 +1,309 @@
+module HyperTabular
+  # Delimited text — CSV, TSV, any single-byte ASCII separator — read a batch at a time
+  # into typed columns, every cell a HyperCast verdict.
+  #
+  #   plan = [HyperTabular::Column.i32(0), HyperTabular::Column.text(1), HyperTabular::Column.f64(2)]
+  #   HyperTabular::DelimitedReader.open("orders.csv", HyperTabular::Dialect::CSV, plan) do |reader|
+  #     while (batch = reader.read)
+  #       ids = batch.values(0)                        # a column at a time: [1, 2, nil, 4, ...]
+  #       batch.rows.times do |row|
+  #         case batch.get(2, row)                     # or a cell at a time, as HyperCast's union
+  #         in HyperCast::Success(value:) then total += value
+  #         in HyperCast::Fault(reason:) then warn "#{reason}: #{batch.raw(2, row).inspect}"
+  #         end
+  #       end
+  #     end
+  #   end
+  #
+  # The native core (libhypertabular) owns no memory and reads no files. It is handed a
+  # chunk of input and the buffers to fill, casts each plan column in one native loop, and
+  # says how many rows it wrote and how many bytes it is finished with. Everything else is
+  # here: this class holds the input, has one value array and one verdict array per column
+  # allocated once and reused for every batch, and puts what the core did not consume back
+  # in front of it. The native boundary is crossed once per batch, not once per cell, and a
+  # column comes out of its buffer in one String#unpack.
+  #
+  # A value that does not cast is that cell's verdict, and the read goes on. Input that is
+  # not rows of cells at all — a record of the wrong width, a quote never closed — is a
+  # TabularError, raised after every intact row before it has been delivered.
+  #
+  # Each #read returns a Batch, which owns what it shows: it stays good after the next
+  # #read. Not thread-safe.
+  class DelimitedReader
+    # The row ceiling: a single record larger than this is a :row_too_long TabularError.
+    MAX_ROW_BYTES = 1 << 30
+
+    # How much is asked of an IO at a time, until a record does not fit.
+    DEFAULT_BUFFER_BYTES = 256 * 1024
+
+    # Rows per batch unless told otherwise.
+    DEFAULT_BATCH_ROWS = 4096
+
+    # Encodings whose bytes already are the UTF-8 (or byte-identical) form the core reads.
+    BYTE_COMPATIBLE = HyperCast::Interop::BYTE_COMPATIBLE
+
+    # Opens a file of UTF-8 delimited text. With a block, yields the reader, closes it —
+    # and the file — when the block ends, and returns the block's value; without one,
+    # returns the reader, whose #close closes the file.
+    def self.open(path, dialect, plan, batch_rows: DEFAULT_BATCH_ROWS, buffer_bytes: DEFAULT_BUFFER_BYTES)
+      file = File.open(path, "rb")
+      begin
+        reader = new(file, dialect, plan, batch_rows: batch_rows, buffer_bytes: buffer_bytes, close_source: true)
+      ensure
+        # The file is ours to close when no reader came of it, whatever went wrong.
+        file.close if reader.nil?
+      end
+      return reader unless block_given?
+
+      begin
+        yield reader
+      ensure
+        reader.close
+      end
+    end
+
+    # Reads UTF-8 delimited text from +source+: a String, read in place — nothing is copied
+    # — or an IO (anything with #readpartial or #read), read forward only through a buffer
+    # of +buffer_bytes+ that grows when a record does not fit it. An IO is read as bytes;
+    # one opened in text mode on Windows should be in binmode first.
+    #
+    # +dialect+ is the declared Dialect and +plan+ the output Columns, in output order.
+    # +batch_rows+ is the most rows one #read delivers. +close_source+ says whether #close
+    # also closes the IO.
+    #
+    # A dialect the scanner cannot honour, or a plan that is not Columns, is an
+    # ArgumentError. With a header declared the header record is read here, so a header
+    # that is structurally broken is a TabularError here.
+    def initialize(source, dialect, plan, batch_rows: DEFAULT_BATCH_ROWS, buffer_bytes: DEFAULT_BUFFER_BYTES,
+                   close_source: false)
+      raise ArgumentError, "dialect must be a HyperTabular::Dialect; got #{dialect.inspect}" unless
+        dialect.is_a?(Dialect)
+      raise ArgumentError, "batch_rows must be a positive Integer; got #{batch_rows.inspect}" unless
+        batch_rows.is_a?(Integer) && batch_rows.positive?
+      raise ArgumentError, "buffer_bytes must be a positive Integer; got #{buffer_bytes.inspect}" unless
+        buffer_bytes.is_a?(Integer) && buffer_bytes.positive?
+
+      @plan = Array(plan).dup.freeze
+      @plan.each_with_index do |column, index|
+        raise ArgumentError, "plan column #{index} must be a HyperTabular::Column; got #{column.inspect}" unless
+          column.is_a?(Column)
+      end
+      @dialect = dialect
+      @batch_rows = batch_rows
+      @buffer_bytes = buffer_bytes
+      @close_source = close_source
+      # Cell-table entries one row takes: the widest ordinal the plan reads, plus two.
+      per_row = (@plan.map(&:ordinal).max || -1) + 2
+      @kernel = Runtime::Delimited.start(dialect.packed, @plan.map(&:packed), @plan.map(&:value_bytes),
+                                         batch_rows, per_row)
+      raise ArgumentError, "separator #{dialect.separator.inspect} is not tab or printable ASCII other than '\"'" if
+        @kernel.nil?
+
+      @start = 0
+      @closed = false
+      @failure = nil
+      take(source)
+      @header = dialect.has_header ? read_header : nil
+    end
+
+    # The header's names, when the dialect declares a header: frozen UTF-8 Strings, quotes
+    # resolved. Empty for an input with no record at all; nil when the dialect declares no
+    # header.
+    attr_reader :header
+
+    # The plan: the output Columns, in output order. Column +i+ of every batch is +plan[i]+.
+    attr_reader :plan
+
+    # The declared Dialect.
+    attr_reader :dialect
+
+    # The most rows one #read delivers.
+    attr_reader :batch_rows
+
+    # Records finished so far — the header and skipped blank lines included.
+    def records
+      @kernel.records
+    end
+
+    # Reads the next batch: a Batch of up to #batch_rows rows, or nil once the input is
+    # exhausted. A TabularError when the input is structurally broken — raised after every
+    # intact row before the break has been delivered, and the same error again on every
+    # later call. IOError on a closed reader.
+    def read
+      raise IOError, "closed reader" if @closed
+      raise @failure if @failure
+
+      loop do
+        length, last = window
+        raise structural unless @kernel.fill(@start, length, last) == Runtime::Delimited::OK
+
+        consumed = @kernel.consumed
+        if @kernel.rows.positive?
+          batch = made(@kernel.rows)
+          @start += consumed
+          return batch
+        end
+        @start += consumed
+        return nil if last && (consumed == length || consumed.zero?)
+
+        refill if consumed.zero?
+      end
+    end
+
+    # Every remaining row, batch after batch: yields one frozen Array per row, a verdict
+    # per plan column. Without a block, an Enumerator. Raises what #read raises.
+    def each_row
+      return to_enum(:each_row) unless block_given?
+
+      while (batch = read)
+        columns = Array.new(@plan.size) { |column| batch.verdicts(column) }
+        batch.rows.times { |row| yield columns.map { |column| column[row] }.freeze }
+      end
+      self
+    end
+
+    # Closes the IO if this reader was told to (+close_source+, or DelimitedReader.open).
+    # Batches already read stay good. Safe to call twice.
+    def close
+      return if @closed
+
+      @closed = true
+      @io.close if @close_source && @io
+      nil
+    end
+
+    # True once #close has been called.
+    def closed?
+      @closed
+    end
+
+    # The reader in a line — not its buffers.
+    def inspect
+      "#<#{self.class.name} columns=#{@plan.size} records=#{records}#{' closed' if @closed}>"
+    end
+
+    private
+
+    # Takes the source: a String becomes the whole input, an IO the thing to refill from.
+    def take(source)
+      if source.is_a?(String)
+        @buffer = utf8(source)
+        @eof = true
+      elsif defined?(::Pathname) && source.is_a?(::Pathname)
+        # It has a #read, and one that starts the file over on every call.
+        raise ArgumentError, "source is a Pathname; DelimitedReader.open reads a path"
+      elsif source.respond_to?(:readpartial) || source.respond_to?(:read)
+        @io = source
+        @partial = source.respond_to?(:readpartial)
+        @buffer = String.new(encoding: Encoding::UTF_8)
+        @eof = false
+      else
+        raise ArgumentError, "source must be a String or an IO; got #{source.class}"
+      end
+      @kernel.attach(@buffer)
+    end
+
+    # The text as a frozen String of UTF-8 bytes that is this reader's alone: the caller's
+    # own when it is already that; otherwise a copy-on-write duplicate, which shares the
+    # bytes and is immune to what the caller does to theirs next. Only a foreign encoding
+    # pays a transcode.
+    def utf8(text)
+      return text if text.frozen? && text.encoding == Encoding::UTF_8
+      return text.dup.force_encoding(Encoding::UTF_8).freeze if BYTE_COMPATIBLE.include?(text.encoding)
+
+      text.encode(Encoding::UTF_8).freeze
+    end
+
+    # What the core is to read next — how many bytes from @start — and whether nothing
+    # follows them.
+    def window
+      length = @buffer.bytesize - @start
+      length > MAX_ROW_BYTES ? [MAX_ROW_BYTES, false] : [length, @eof]
+    end
+
+    # Puts the unfinished record at the front of the buffer and reads more behind it —
+    # at least as much again as is pending, so a record that does not fit doubles the
+    # buffer rather than creeping up on it.
+    def refill
+      pending = @buffer.bytesize - @start
+      raise too_long if @io.nil? || pending >= MAX_ROW_BYTES
+
+      chunk = more([@buffer_bytes, pending].max)
+      if chunk.nil? || chunk.empty?
+        @eof = true
+        return
+      end
+      chunk = chunk.dup if chunk.frozen?
+      @buffer = @buffer.byteslice(@start, pending) if @start.positive?
+      @buffer << chunk.force_encoding(Encoding::UTF_8)
+      @start = 0
+      @kernel.attach(@buffer)
+    end
+
+    # Up to +bytes+ more of the IO, or nil at its end. #readpartial where there is one, so
+    # a pipe or a socket hands over what it has rather than waiting to fill the request.
+    def more(bytes)
+      @partial ? @io.readpartial(bytes) : @io.read(bytes)
+    rescue EOFError
+      nil
+    end
+
+    # The batch the core just wrote, copied out of the kernel's arrays: its spans index the
+    # input from @start, where the core was handed it.
+    def made(rows)
+      columns = @kernel.columns
+      Batch.new(@plan, rows, Array.new(@plan.size) { |column| columns.values(column, rows) },
+                Array.new(@plan.size) { |column| columns.verdicts(column, rows) },
+                @kernel.cells, @kernel.per_row, @buffer, @start, @kernel.arena, workbook: false)
+    end
+
+    # Reads the header record: its names, or none for an input with no record at all.
+    def read_header
+      loop do
+        length, last = window
+        raise structural unless @kernel.header(@start, length, last) == Runtime::Delimited::OK
+
+        consumed = @kernel.consumed
+        if @kernel.rows.positive?
+          names = header_names
+          @start += consumed
+          return names
+        end
+        @start += consumed
+        # An empty input has no header and no rows; the width is unknown.
+        return [].freeze if last && (consumed == length || consumed.zero?)
+
+        refill if consumed.zero?
+      end
+    end
+
+    # The names the core just located: in the input as written, or — flagged — unescaped
+    # in the arena.
+    def header_names
+      spans = @kernel.names
+      arena = nil
+      Array.new(@kernel.rows) do |index|
+        offset = spans[index * 2]
+        length = spans[index * 2 + 1]
+        name =
+          if length < Runtime::Delimited::SPAN_FLAG
+            @buffer.byteslice(@start + offset, length)
+          else
+            arena ||= @kernel.arena.force_encoding(Encoding::UTF_8)
+            arena.byteslice(offset, length & Runtime::Delimited::SPAN_LENGTH)
+          end
+        name.freeze
+      end.freeze
+    end
+
+    # The structural failure the core just reported, kept: it is final.
+    def structural
+      @failure = TabularError.from(@kernel.failure)
+    end
+
+    # The failure of a record that outgrew the ceiling, kept: it is final too.
+    def too_long
+      @failure = TabularError.new(:row_too_long, @kernel.records, @kernel.line, @kernel.offset)
+    end
+  end
+end

data/lib/hypertabular/dialect.rb

@@ -0,0 +1,42 @@
+module HyperTabular
+  # How the text is delimited, declared by the caller. Nothing is sniffed: the separator is
+  # stated, quoting is stated, the header is stated — the same stance HyperCast's NumFormat
+  # takes for numeric notation.
+  #
+  # +separator+ is one byte: tab, or any printable ASCII character except the double quote.
+  # +quoting+ says whether " quotes cells (RFC 4180, "" for a literal quote); off, a quote
+  # is an ordinary byte. +has_header+ says whether the first record is a header, given by
+  # DelimitedReader#header and never delivered as a row. +skip_blank_lines+ says whether a
+  # completely empty line is skipped rather than read as a one-cell row.
+  #
+  #   HyperTabular::Dialect::CSV
+  #   HyperTabular::Dialect.new(separator: ";", has_header: false)
+  Dialect = Data.define(:separator, :quoting, :has_header, :skip_blank_lines) do
+    # Everything but the separator defaults to what CSV means: quoted, with a header, blank
+    # lines skipped. A separator that is not a one-byte String is a caller bug
+    # (ArgumentError); which bytes the scanner can honour is the core's to say, and it says
+    # so when a reader is built.
+    def initialize(separator:, quoting: true, has_header: true, skip_blank_lines: true)
+      raise ArgumentError, "separator must be a one-byte String; got #{separator.inspect}" unless
+        separator.is_a?(String) && separator.bytesize == 1
+
+      super(separator: -separator, quoting: quoting ? true : false,
+            has_header: has_header ? true : false, skip_blank_lines: skip_blank_lines ? true : false)
+    end
+
+    # The four bytes of the core's RawDialect: separator, quoting, skip_blank_lines, and
+    # the scan engine left to the core's own choice.
+    def packed
+      [separator.getbyte(0), quoting ? 1 : 0, skip_blank_lines ? 1 : 0, 0].pack("C4")
+    end
+  end
+
+  # Comma-separated, quoted, with a header, blank lines skipped.
+  Dialect::CSV = Dialect.new(separator: ",")
+
+  # Tab-separated, otherwise as CSV.
+  Dialect::TSV = Dialect.new(separator: "\t")
+
+  # Pipe-separated, otherwise as CSV.
+  Dialect::PSV = Dialect.new(separator: "|")
+end

data/lib/hypertabular/runtime/columns.rb

@@ -0,0 +1,46 @@
+module HyperTabular
+  module Runtime
+    # A plan as the core takes it, and the arrays the core casts it into: one packed
+    # ColumnSpec per plan column, one value array and one verdict array per column sized
+    # for one batch, and the ColumnBuffer table that names them — allocated once and reused
+    # for every batch. A delimited read and a sheet each own one.
+    class Columns
+      # Bytes in one verdict (CellVerdict: offset, len, reason — three u32).
+      VERDICT_BYTES = 12
+      # ColumnSpec: ordinal, door, param, then HyperCast's 32-byte RawNumFormat.
+      SPEC_BYTES = 44
+
+      # The packed specs (nil for no columns), the ColumnBuffer table (likewise), how many
+      # columns there are, and the most rows a batch holds.
+      attr_reader :specs, :table, :count, :batch_rows
+
+      # +specs+ is one packed ColumnSpec per plan column and +sizes+ the bytes one value of
+      # each takes.
+      def initialize(specs, sizes, batch_rows)
+        @sizes = sizes
+        @count = specs.size
+        @batch_rows = batch_rows
+        return if specs.empty?
+
+        @specs = Runtime.buffer(SPEC_BYTES * @count)
+        @specs[0, SPEC_BYTES * @count] = specs.join
+        @values = sizes.map { |size| Runtime.buffer(size * batch_rows) }
+        @verdicts = sizes.map { Runtime.buffer(VERDICT_BYTES * batch_rows) }
+        addresses = @values.zip(@verdicts).flatten.map(&:to_i).pack("J*")
+        @table = Runtime.buffer(addresses.bytesize)
+        @table[0, addresses.bytesize] = addresses
+      end
+
+      # A column's value array for the first +rows+ rows, copied out as the core wrote it.
+      def values(column, rows)
+        @values[column][0, @sizes[column] * rows]
+      end
+
+      # A column's verdict array for the first +rows+ rows — offset, len, reason per row —
+      # copied out as the core wrote it.
+      def verdicts(column, rows)
+        @verdicts[column][0, VERDICT_BYTES * rows]
+      end
+    end
+  end
+end

data/lib/hypertabular/runtime/delimited.rb

@@ -0,0 +1,204 @@
+module HyperTabular
+  module Runtime
+    # The core's delimited reader as one object: every byte of native memory a read needs
+    # and every native call it makes. This class is the whole Fiddle crossing — the reader
+    # above it (DelimitedReader) is plain Ruby that never sees a pointer — and so it is the
+    # one thing a compiled extension would replace: same methods, bytes in and bytes out.
+    #
+    # The memory is allocated once and reused for every batch: the state block, the plan,
+    # one value array and one verdict array per column, the cell table, the arena. The core
+    # keeps none of it between calls beyond what it writes into the state block.
+    class Delimited
+      # The call did what it could; the result says how far it got.
+      OK = 0
+      # A caller bug, never a data verdict.
+      ERR_CONTRACT = -1
+      # The data is structurally broken; the failure says where.
+      ERR_STRUCTURE = -2
+      # The arena cannot hold what one row needs.
+      ERR_ARENA = -3
+      # The cell table (or the header's name table) cannot hold one row.
+      ERR_CELLS = -4
+
+      # The flag in the top bit of a span's length. On a cell-table entry: the cell has ""
+      # inside and has to be unescaped to be read. On a text value or a header name: the
+      # bytes are in the arena rather than in the input.
+      SPAN_FLAG = 1 << 31
+      # A span's length without its flag.
+      SPAN_LENGTH = SPAN_FLAG - 1
+
+      # Bytes in one span.
+      SPAN_BYTES = 8
+      # Filled: rows, consumed, arena_used, needed, then Failure (code, line, record, byte,
+      # expected, found).
+      FILLED_BYTES = 64
+      FILLED = "Q<4L<2Q<2L<2".freeze
+      # Buffers: seven pointer and size pairs (window, arena, cells, row, strings, table,
+      # kinds), as every workbook call takes them. A delimited fill reads the arena and the
+      # cell table from it and nothing else.
+      BUFFERS = "Q<14".freeze
+      BUFFERS_BYTES = 112
+
+      ARENA_BYTES = 4096
+      NAMES = 64
+
+      CONTRACT = "hypertabular: libhypertabular reported a contract violation — a binding bug, " \
+                 "please report it".freeze
+
+      # What the last call wrote: rows (or header names), input bytes finished with, arena
+      # bytes written, and — after ERR_STRUCTURE — the failure as
+      # [code, line, record, byte, expected, found].
+      attr_reader :rows, :consumed, :arena_used, :failure
+
+      # The plan's arrays (Columns), and the cell-table entries one row takes: the widest
+      # ordinal the plan reads, plus two — or more, if the core asked for more.
+      attr_reader :columns, :per_row
+
+      # The loaded core's version word, major << 16 | minor << 8 | patch.
+      def self.version
+        Runtime.function(:hypertabular_version).call
+      end
+
+      # +dialect+ is the four bytes of a RawDialect; +specs+ one packed ColumnSpec per plan
+      # column and +sizes+ the bytes one value of each takes; +per_row+ the cell-table
+      # entries one row takes. Nil when the core refuses the dialect.
+      def self.start(dialect, specs, sizes, batch_rows, per_row)
+        reader = new(specs, sizes, batch_rows, per_row)
+        reader.send(:init, dialect) ? reader : nil
+      end
+
+      def initialize(specs, sizes, batch_rows, per_row)
+        @header = Runtime.function(:hypertabular_delimited_header)
+        @fill = Runtime.function(:hypertabular_delimited_fill)
+        @state = Runtime.buffer(Runtime.function(:hypertabular_delimited_state_size).call)
+        @batch_rows = batch_rows
+        @columns = Columns.new(specs, sizes, batch_rows)
+        @per_row = per_row
+        @cramped = false
+        @cells_cap = per_row * batch_rows
+        @cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
+        @arena_cap = ARENA_BYTES
+        @arena = Runtime.buffer(@arena_cap)
+        @out = Runtime.buffer(FILLED_BYTES)
+        @buffers = Runtime.buffer(BUFFERS_BYTES)
+        @rows = @consumed = @arena_used = 0
+      end
+
+      # Names the String the calls that follow read: +start+ and +offset+ below index its
+      # bytes. It is held where it is, not copied, and must not be modified until another
+      # one is attached.
+      def attach(input)
+        @pin = Runtime.pin(input)
+        @base = @pin.to_i
+      end
+
+      # Reads the next record of the attached input, +length+ bytes from +start+, as a
+      # header. OK or ERR_STRUCTURE; +rows+ is then the number of names.
+      def header(start, length, last)
+        @names_cap ||= NAMES
+        @names ||= Runtime.buffer(SPAN_BYTES * @names_cap)
+        loop do
+          code = @header.call(@state, @base + start, length, last ? 1 : 0,
+                              @names, @names_cap, @arena, @arena_cap, @out)
+          needed = finished
+          case code
+          when OK, ERR_STRUCTURE then return code
+          when ERR_CELLS
+            @names_cap = needed
+            @names = Runtime.buffer(SPAN_BYTES * @names_cap)
+          when ERR_ARENA then grow_arena(needed)
+          else raise CONTRACT
+          end
+        end
+      end
+
+      # The header's names as the core located them: offset and flagged length, a pair per
+      # name, offsets relative to the +start+ the header was read at (or into the arena,
+      # for a flagged one).
+      def names
+        @names[0, SPAN_BYTES * @rows].unpack("L<*")
+      end
+
+      # Fills every column from the attached input, +length+ bytes from +start+ — the one
+      # native call a batch makes. OK or ERR_STRUCTURE.
+      #
+      # The core ends a batch early when the arena fills, so a batch that came back short
+      # with the arena half used or more has the next one start with it doubled: escaped
+      # text costs a few batches, not one per row.
+      def fill(start, length, last)
+        grow_arena(@arena_cap * 2) if @cramped
+        @cramped = false
+        loop do
+          code = @fill.call(@state, @base + start, length, last ? 1 : 0,
+                            @columns.specs, @columns.table, @columns.count, @batch_rows,
+                            buffers, @out)
+          needed = finished
+          case code
+          when OK
+            @cramped = @rows.positive? && @rows < @batch_rows && @arena_used * 2 >= @arena_cap
+            return code
+          when ERR_STRUCTURE then return code
+          when ERR_CELLS
+            @per_row = [@per_row, needed].max
+            @cells_cap = @per_row * @batch_rows
+            @cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
+          when ERR_ARENA then grow_arena(needed)
+          else raise CONTRACT
+          end
+        end
+      end
+
+      # The cell table for the batch in hand — #per_row spans a row — copied out.
+      def cells
+        @cells[0, SPAN_BYTES * @per_row * @rows]
+      end
+
+      # The arena as the last call left it: the unescaped text of every flagged span.
+      def arena
+        @arena[0, @arena_used]
+      end
+
+      # Records finished so far — the header and skipped blank lines included.
+      def records
+        @state[8, 8].unpack1("Q<")
+      end
+
+      # One-based line number of the next unread byte.
+      def line
+        @state[4, 4].unpack1("L<")
+      end
+
+      # Absolute byte offset of the next unread byte.
+      def offset
+        @state[16, 8].unpack1("Q<")
+      end
+
+      private
+
+      def init(dialect)
+        raw = Runtime.buffer(4)
+        raw[0, 4] = dialect
+        Runtime.function(:hypertabular_delimited_init).call(@state, raw) == OK
+      end
+
+      # The arena and the cell table as the Buffers block a fill takes them — packed again
+      # for every call, since either may have been grown since the last.
+      def buffers
+        @buffers[0, BUFFERS_BYTES] = [0, 0, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, 0, 0, 0, 0, 0, 0, 0, 0]
+                                     .pack(BUFFERS)
+        @buffers
+      end
+
+      # Reads what the call wrote to its Filled block; returns `needed`.
+      def finished
+        @rows, @consumed, @arena_used, needed, *@failure = @out[0, FILLED_BYTES].unpack(FILLED)
+        needed
+      end
+
+      def grow_arena(needed)
+        @arena_cap = [needed, @arena_cap * 2].max
+        @arena = Runtime.buffer(@arena_cap)
+      end
+    end
+  end
+end