hypertabular 0.7.0-x64-mingw-ucrt

data/lib/hypertabular/runtime/workbook.rb

@@ -0,0 +1,299 @@
+module HyperTabular
+  module Runtime
+    # The core's workbook reader as objects: every byte of native memory a workbook read
+    # needs and every native call it makes — the Fiddle crossing behind Workbook and Sheet,
+    # which never see a pointer.
+    #
+    # A workbook call works in buffers the caller hands it (a Buffers block of seven pointer
+    # and size pairs), and when one is too small it says which and how large, having undone
+    # nothing: the buffer is grown with what it held kept, and the same call made again.
+    module Book
+      class << self
+        # For the specs: every buffer a workbook call works in starts with room for one
+        # element and no shared-strings bound is asked for, so every call that can stop and
+        # resume does — the grow-and-keep path exercised mid-part.
+        attr_accessor :stingy
+
+        # For the specs: how many times the window, the arena and the cell table were grown,
+        # as [window, arena, cells].
+        attr_accessor :grown
+      end
+      self.stingy = false
+      self.grown = [0, 0, 0]
+
+      OK = Delimited::OK
+      ERR_STRUCTURE = Delimited::ERR_STRUCTURE
+      ERR_ARENA = Delimited::ERR_ARENA
+      ERR_CELLS = Delimited::ERR_CELLS
+      # The window is too small to inflate and tokenize in.
+      ERR_WINDOW = -5
+
+      # The smallest window the core works in.
+      WINDOW_MIN = 64 * 1024
+      # Bytes in one span, and in one slot of the row a read assembles.
+      SPAN_BYTES = 8
+      SLOT_BYTES = 16
+      # Buffers: seven pointer and size pairs — the same block a delimited fill takes.
+      BUFFERS = Delimited::BUFFERS
+      BUFFERS_BYTES = Delimited::BUFFERS_BYTES
+      # Opened: format, epoch, strings_bytes, strings_count, needed, then Failure.
+      OPENED = "L<2Q<3L<2Q<2L<2".freeze
+      OPENED_BYTES = 64
+      # Filled: rows, consumed, arena_used, needed, then Failure.
+      FILLED = Delimited::FILLED
+      FILLED_BYTES = Delimited::FILLED_BYTES
+
+      # The workbook's tables as a sheet's calls are handed them: strings, the span of each,
+      # the kind of each cell format. Pointers and sizes.
+      Tables = Struct.new(:strings, :strings_len, :table, :table_len, :kinds, :kinds_len)
+
+      # The buffers a call to the core may ask to have grown: one set for a workbook while
+      # it opens, one for each sheet.
+      class Scratch
+        # The buffers, each a Fiddle::Pointer, and their sizes (in bytes, spans, slots).
+        attr_reader :window, :arena, :cells, :arena_cap, :cells_cap
+
+        def initialize(window, arena, cells, row)
+          @window_cap = window
+          @window = Runtime.buffer([window, 1].max)
+          @arena_cap = arena
+          @arena = Runtime.buffer([arena, 1].max)
+          @cells_cap = cells
+          @cells = Runtime.buffer([SPAN_BYTES * cells, 1].max)
+          @row_cap = row
+          @row = Runtime.buffer([SLOT_BYTES * row, 1].max)
+          @buffers = Runtime.buffer(BUFFERS_BYTES)
+        end
+
+        # This scratch as the core takes it, with the workbook's tables when a sheet is
+        # being read.
+        def buffers(tables = nil)
+          words = [@window.to_i, @window_cap, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, @row.to_i, @row_cap]
+          words.concat(tables ? tables.to_a : [0] * 6)
+          @buffers[0, BUFFERS_BYTES] = words.pack(BUFFERS)
+          @buffers
+        end
+
+        # Makes the window at least +needed+ bytes, what it held kept.
+        def grow_window(needed)
+          @window, @window_cap = grown(@window, @window_cap, needed, 1)
+        end
+
+        # Makes the arena at least +needed+ bytes, what it held kept.
+        def grow_arena(needed)
+          @arena, @arena_cap = grown(@arena, @arena_cap, needed, 1)
+        end
+
+        # Makes the cell table at least +needed+ spans, what it held kept.
+        def grow_cells(needed)
+          @cells, @cells_cap = grown(@cells, @cells_cap, needed, SPAN_BYTES)
+        end
+
+        # Makes +call+ (a block taking the Buffers block and the out block) until it stops
+        # asking for room, growing the buffer it names each time. Returns the code it ended
+        # on and the Filled fields: rows, consumed, arena_used, needed, then the failure.
+        def drive(out, tables = nil)
+          loop do
+            code = yield buffers(tables)
+            filled = out[0, FILLED_BYTES].unpack(FILLED)
+            case code
+            when ERR_WINDOW then grow_window(filled[3]).then { Book.grown[0] += 1 }
+            when ERR_ARENA then grow_arena(filled[3]).then { Book.grown[1] += 1 }
+            when ERR_CELLS then grow_cells(filled[3]).then { Book.grown[2] += 1 }
+            else return [code, filled]
+            end
+          end
+        end
+
+        private
+
+        def grown(old, capacity, needed, unit)
+          length = [needed, capacity + 1].max
+          larger = Runtime.buffer(unit * length)
+          larger[0, unit * capacity] = old[0, unit * capacity] if capacity.positive?
+          [larger, length]
+        end
+      end
+
+      # A workbook opened in memory: its state, its sheets, its tables.
+      class Opened
+        CALLS = {
+          sheets: :hypertabular_workbook_sheets, strings: :hypertabular_workbook_strings,
+          styles: :hypertabular_workbook_styles
+        }.freeze
+
+        # :xlsx or :ods; the date system's code (1 for 1900, 2 for 1904); the sheets as
+        # [name, hidden, part, index]; the shared strings' bytes; the state template every
+        # sheet copies; and the tables a sheet's calls are handed.
+        attr_reader :format, :epoch, :sheets, :strings, :state, :tables, :container, :length
+
+        # Opens +container+, a String the workbook holds unmodified for its whole life. Raises
+        # the failure as [code, line, record, byte, expected, found] in a StructureError.
+        def initialize(container)
+          @container_string = container
+          @container = Runtime.pin(container)
+          @length = container.bytesize
+          @state_size = Runtime.function(:hypertabular_workbook_state_size).call
+          @state = Runtime.buffer(@state_size)
+          @out = Runtime.buffer([OPENED_BYTES, FILLED_BYTES].max)
+          scratch = Book.stingy ? Scratch.new(1, 1, 1, 0) : Scratch.new(WINDOW_MIN, 1024, 64, 0)
+          strings_bytes = open(scratch)
+          @sheets = listed(scratch)
+          load_tables(scratch, strings_bytes)
+        end
+
+        # A copy of the opened state, which the core allows: a sheet's read of its own.
+        def copy_of_state
+          copy = Runtime.buffer(@state_size)
+          copy[0, @state_size] = @state[0, @state_size]
+          copy
+        end
+
+        private
+
+        def open(scratch)
+          call = Runtime.function(:hypertabular_workbook_open)
+          loop do
+            code = call.call(@state, @container, @length, scratch.buffers, @out)
+            format, epoch, strings_bytes, _count, needed, *failure = @out[0, OPENED_BYTES].unpack(OPENED)
+            case code
+            when OK
+              @format = format == 2 ? :ods : :xlsx
+              @epoch = epoch
+              return strings_bytes
+            when ERR_WINDOW then scratch.grow_window(needed)
+            when ERR_ARENA then scratch.grow_arena(needed)
+            when ERR_STRUCTURE then raise StructureError, failure
+            else raise Delimited::CONTRACT
+            end
+          end
+        end
+
+        def run(scratch, name)
+          call = Runtime.function(CALLS.fetch(name))
+          code, filled = scratch.drive(@out) { |buffers| call.call(@state, @container, @length, buffers, @out) }
+          Book.settle(code, filled)
+        end
+
+        # The sheets the core listed: three spans each — name, part, then one whose offset's
+        # low bit says hidden and whose length is the sheet's index.
+        def listed(scratch)
+          rows = run(scratch, :sheets).first
+          spans = scratch.cells[0, SPAN_BYTES * 3 * rows].unpack("L<*")
+          arena = scratch.arena[0, scratch.arena_cap]
+          Array.new(rows) do |index|
+            name_at, name_len, part_at, part_len, flags, sheet = spans[index * 6, 6]
+            [arena.byteslice(name_at, name_len).force_encoding(Encoding::UTF_8).freeze, flags.odd?,
+             arena.byteslice(part_at, part_len), sheet]
+          end.freeze
+        end
+
+        def load_tables(scratch, strings_bytes)
+          # The shared strings take no more room than their part inflates to; asking for it
+          # once saves growing into it.
+          bound = [strings_bytes, 1 << 28].min
+          scratch.grow_arena(bound) if !Book.stingy && scratch.arena_cap < bound
+          rows, _consumed, arena_used = run(scratch, :strings)
+          @strings = scratch.arena[0, arena_used].force_encoding(Encoding::UTF_8).freeze
+          @strings_buffer = copy(scratch.arena, arena_used)
+          @table = copy(scratch.cells, SPAN_BYTES * rows)
+          table_len = rows
+          kinds_len = run(scratch, :styles).first
+          @kinds = copy(scratch.arena, kinds_len)
+          @tables = Tables.new(@strings_buffer.to_i, arena_used, @table.to_i, table_len, @kinds.to_i, kinds_len)
+        end
+
+        def copy(from, bytes)
+          buffer = Runtime.buffer([bytes, 1].max)
+          buffer[0, bytes] = from[0, bytes] if bytes.positive?
+          buffer
+        end
+      end
+
+      # One sheet being read: its own copy of the state, its plan's arrays, its scratch.
+      class Reading
+        # The plan's arrays, and the cell-table entries one row takes: one per plan column,
+        # and one for the row's number.
+        attr_reader :columns, :per_row, :header
+
+        def initialize(book, sheet, has_header, skip_empty_rows, specs, sizes, batch_rows, width)
+          @book = book
+          @state = book.copy_of_state
+          @columns = Columns.new(specs, sizes, batch_rows)
+          @per_row = specs.size + 1
+          @scratch = Book.stingy ? Scratch.new(1, 1, 1, width) : Scratch.new(0, 4096, batch_rows * @per_row, width)
+          @out = Runtime.buffer(FILLED_BYTES)
+          _name, _hidden, part, index = sheet
+          code = Runtime.function(:hypertabular_workbook_sheet).call(
+            @state, book.container, book.length, Runtime.pin(part), part.bytesize, index,
+            has_header ? 1 : 0, skip_empty_rows ? 1 : 0, @out
+          )
+          Book.settle(code, @out[0, FILLED_BYTES].unpack(FILLED))
+          @header = read_header if has_header
+        end
+
+        # The next batch's rows, its cell table, the arena as far as the core wrote into it
+        # (which is as far as any of the batch's spans reach), and the failure — [code, line,
+        # record, byte, expected, found] — that came with them, if one did: the caller raises
+        # it after them. The table and the arena are copies.
+        def fill
+          call = Runtime.function(:hypertabular_workbook_fill)
+          code, filled = @scratch.drive(@out, @book.tables) do |buffers|
+            call.call(@state, @book.container, @book.length, @columns.specs, @columns.table,
+                      @columns.count, @columns.batch_rows, buffers, @out)
+          end
+          raise Delimited::CONTRACT unless [OK, ERR_STRUCTURE].include?(code)
+
+          rows, _consumed, arena_used = filled
+          [rows, @scratch.cells[0, SPAN_BYTES * @per_row * rows], @scratch.arena[0, arena_used],
+           code == OK ? nil : filled[4..]]
+        end
+
+        private
+
+        def read_header
+          call = Runtime.function(:hypertabular_workbook_header)
+          code, filled = @scratch.drive(@out, @book.tables) do |buffers|
+            call.call(@state, @book.container, @book.length, buffers, @out)
+          end
+          rows = Book.settle(code, filled).first
+          spans = @scratch.cells[0, SPAN_BYTES * rows].unpack("L<*")
+          arena = nil
+          Array.new(rows) do |index|
+            offset = spans[index * 2]
+            length = spans[index * 2 + 1]
+            name =
+              if length < Delimited::SPAN_FLAG
+                @book.strings.byteslice(offset, length)
+              else
+                arena ||= @scratch.arena[0, @scratch.arena_cap].force_encoding(Encoding::UTF_8)
+                arena.byteslice(offset, length & Delimited::SPAN_LENGTH)
+              end
+            name.freeze
+          end.freeze
+        end
+      end
+
+      # A structural failure the core reported, as [code, line, record, byte, expected,
+      # found], on its way to becoming a TabularError.
+      class StructureError < StandardError
+        attr_reader :failure
+
+        def initialize(failure)
+          @failure = failure
+          super("structural failure #{failure.first}")
+        end
+      end
+
+      # What a call's code means: the Filled fields, or a structural failure. Anything else
+      # is this binding's bug.
+      def self.settle(code, filled)
+        case code
+        when OK then filled
+        when ERR_STRUCTURE then raise StructureError, filled[4..]
+        else raise Delimited::CONTRACT
+        end
+      end
+    end
+  end
+end

data/lib/hypertabular/runtime.rb

@@ -0,0 +1,142 @@
+# Autoloaded, not required, as in hypercast: Fiddle loads the first time the backend runs,
+# and every path there goes through `functions` first, so a missing library is reported as
+# missing_library_message rather than as whatever touching Fiddle raises first.
+autoload :Fiddle, "fiddle"
+
+module HyperTabular
+  # Fiddle plumbing for the native libhypertabular shared library — dlopen/dlsym plus raw
+  # C-ABI calls, no runtime bridge. A gem's files are plain files on disk once installed, so
+  # native/{rid}/{lib} is dlopen'ed directly — no extraction.
+  module Runtime
+    NATIVE_DIR = File.join(__dir__, "native")
+
+    # Every export of the library (rust/src/kernel/exports.rs), with its C signature spelled
+    # in Symbols — resolved to Fiddle types only in load_functions, so nothing here touches
+    # Fiddle until the backend actually runs.
+    EXPORTS = {
+      hypertabular_version: [[], :uint32],
+      hypertabular_delimited_state_size: [[], :size],
+      # (state, dialect)
+      hypertabular_delimited_init: [%i[pointer pointer], :int32],
+      # (state, input, input_len, last, names, names_cap, arena, arena_cap, out)
+      hypertabular_delimited_header:
+        [%i[pointer pointer size uint32 pointer size pointer size pointer], :int32],
+      # (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out) —
+      # buffers is the Buffers block the workbook calls take, of which a delimited fill reads
+      # only the arena and the cell table.
+      hypertabular_delimited_fill:
+        [%i[pointer pointer size uint32 pointer pointer size size pointer pointer], :int32],
+      # (cell, len, out, cap)
+      hypertabular_delimited_unescape: [%i[pointer size pointer size], :size],
+      hypertabular_workbook_state_size: [[], :size],
+      # (state, container, container_len, buffers, out)
+      hypertabular_workbook_open: [%i[pointer pointer size pointer pointer], :int32],
+      hypertabular_workbook_sheets: [%i[pointer pointer size pointer pointer], :int32],
+      hypertabular_workbook_strings: [%i[pointer pointer size pointer pointer], :int32],
+      hypertabular_workbook_styles: [%i[pointer pointer size pointer pointer], :int32],
+      # (state, container, container_len, part, part_len, index, has_header, skip_empty_rows, out)
+      hypertabular_workbook_sheet:
+        [%i[pointer pointer size pointer size uint32 uint32 uint32 pointer], :int32],
+      hypertabular_workbook_header: [%i[pointer pointer size pointer pointer], :int32],
+      # (state, container, container_len, specs, columns, column_count, max_rows, buffers, out)
+      hypertabular_workbook_fill: [%i[pointer pointer size pointer pointer size size pointer pointer], :int32]
+    }.freeze
+
+    @mutex = Mutex.new
+    @functions = nil
+
+    class << self
+      # The export's Fiddle::Function, for the caller to invoke directly.
+      def function(symbol)
+        functions.fetch(symbol)
+      end
+
+      # Every native allocation goes through here, and loads the library first: that is what
+      # keeps a missing library reported as missing_library_message instead of as whatever
+      # touching Fiddle raises first. Zeroed, freed with the object that holds it.
+      def buffer(size)
+        functions
+        Fiddle::Pointer.malloc(size, Fiddle::RUBY_FREE)
+      end
+
+      # A pointer to a String's own bytes — nothing is copied. While the returned object is
+      # alive the String is held where it is: Fiddle marks what it wraps as unmovable, so
+      # the garbage collector's compaction cannot relocate an embedded String under a native
+      # call. The address is good until the String is next modified.
+      def pin(string)
+        functions
+        Fiddle::Pointer[string]
+      end
+
+      # A quoted delimited cell — the bytes a flagged cell-table entry names — with its
+      # quotes resolved, as the core cast it.
+      def unescape(quoted)
+        input = pin(quoted)
+        out = buffer([quoted.bytesize, 1].max)
+        written = function(:hypertabular_delimited_unescape).call(input, quoted.bytesize, out, quoted.bytesize)
+        out[0, written]
+      end
+
+      private
+
+      # The shared library to dlopen: this install's native/{rid}/{lib}, or — the
+      # development loop — the in-repo cargo build, exactly what the other bindings' local
+      # staging does. Nil when neither exists.
+      def library_path
+        HyperCast::Interop.library_path("hypertabular", NATIVE_DIR, File.expand_path("../../..", __dir__))
+      end
+
+      # Why Fiddle found nothing to load. A precompiled platform gem is the one install where
+      # that is by design rather than a gap: it carries only its Magnus extensions, and the
+      # Fiddle backend is reached there only by forcing it (HYPERTABULAR_PURE) or because none
+      # of its extensions loaded — a gem RubyGems matched to a Ruby it was not built for. So
+      # that case names its fix, the universal gem, which carries every platform's library,
+      # instead of a missing path that reads like a packaging bug — hypercast's wording for
+      # the same case. Both arguments are parameters only so the specs can ask for every
+      # wording.
+      def missing_library_message(gem_platform = Gem.loaded_specs["hypertabular"]&.platform,
+                                  forced = ENV.key?("HYPERTABULAR_PURE"))
+        rid, lib_name = HyperCast::NativePlatform.rid_and_library_name(library: "hypertabular")
+        missing = File.join(NATIVE_DIR, rid, lib_name)
+        if gem_platform && gem_platform.to_s != Gem::Platform::RUBY
+          reason =
+            if forced
+              "HYPERTABULAR_PURE forces the Fiddle backend (unset it to use the extension)"
+            else
+              "none of its extensions loads on this Ruby (#{RUBY_VERSION}, #{RUBY_PLATFORM})"
+            end
+          "hypertabular: this #{gem_platform} platform gem carries only Magnus extensions, no " \
+            "Fiddle library, and #{reason}. The universal gem has the Fiddle backend for every " \
+            "platform: `gem install hypertabular --platform ruby`, or Bundler's " \
+            "force_ruby_platform (#{missing} not found)"
+        else
+          "hypertabular: #{missing} not found (unsupported platform, or this gem was built " \
+            "without a native library for it)"
+        end
+      end
+
+      # Loaded lazily and exactly once; the native library and its function pointers live
+      # for the process's lifetime (never dlclose'd). The unsynchronized read is the fast
+      # path; the benign race re-checks under the lock.
+      def functions
+        @functions || @mutex.synchronize { @functions ||= load_functions }
+      end
+
+      def load_functions
+        path = library_path
+        raise LoadError, missing_library_message if path.nil?
+
+        handle = Fiddle.dlopen(path)
+        types = {
+          pointer: Fiddle::TYPE_VOIDP, size: Fiddle::TYPE_SIZE_T,
+          uint32: Fiddle::TYPE_UINT32_T, int32: Fiddle::TYPE_INT32_T
+        }
+        EXPORTS.to_h do |name, (arguments, result)|
+          [name, Fiddle::Function.new(handle[name.to_s], arguments.map { |type|
+            types.fetch(type)
+          }, types.fetch(result))]
+        end
+      end
+    end
+  end
+end

data/lib/hypertabular/tabular_error.rb

@@ -0,0 +1,99 @@
+module HyperTabular
+  # A structural failure: the input is not rows of cells — a record of the wrong width,
+  # input that ends inside a quoted cell, a workbook whose container or parts cannot be
+  # read. Never a cell's verdict: a value that does not cast is a HyperCast::Fault in its
+  # column, and the read goes on. A structural failure ends the input, after every intact
+  # row before it has been delivered.
+  #
+  # For a workbook, #record is the part the failure is in, #line the sheet row and #byte the
+  # offset within the part's inflated bytes.
+  #
+  #   begin
+  #     reader.read
+  #   rescue HyperTabular::TabularError => e
+  #     case e
+  #     in { kind: :column_count, line:, expected:, found: }
+  #       warn "line #{line}: #{found} cells, not #{expected}"
+  #     in { kind: :unclosed_quote, line: }
+  #       warn "line #{line}: a quote is never closed"
+  #     end
+  #   end
+  class TabularError < StandardError
+    # What the core's failure codes mean.
+    KINDS = {
+      1 => :unclosed_quote, 2 => :column_count, 3 => :row_too_long, 16 => :not_a_zip, 17 => :container,
+      18 => :encrypted, 19 => :method, 20 => :missing_part, 21 => :xml, 22 => :deflate,
+      23 => :not_a_workbook, 24 => :shared_string, 25 => :too_large
+    }.freeze
+
+    # What is wrong: :unclosed_quote (the input ended inside a quoted cell), :column_count
+    # (a record's cell count disagrees with the first record's), :row_too_long (a single
+    # record is larger than DelimitedReader::MAX_ROW_BYTES); for a workbook :not_a_zip,
+    # :container (the zip's own structure is broken), :encrypted, :method (a part is
+    # compressed by neither stored nor deflate), :missing_part, :xml (a part ends inside an
+    # XML construct), :deflate (a part is not a whole deflate stream), :not_a_workbook,
+    # :shared_string (a cell names a shared string the table does not have) or :too_large.
+    attr_reader :kind
+
+    # Zero-based index of the offending record — the header and skipped blank lines
+    # included.
+    attr_reader :record
+
+    # One-based line the offending record starts on.
+    attr_reader :line
+
+    # Absolute byte offset of the offending record's start, a byte-order mark included.
+    attr_reader :byte
+
+    # Cells in the first record, for :column_count; 0 otherwise.
+    attr_reader :expected
+
+    # Cells in this record, for :column_count; 0 otherwise.
+    attr_reader :found
+
+    # The failure the core reported, as [code, line, record, byte, expected, found]. A code
+    # this binding does not know is the container's: the most general refusal.
+    def self.from(failure)
+      code, line, record, byte, expected, found = failure
+      new(KINDS.fetch(code, :container), record, line, byte, expected, found)
+    end
+
+    # Raised by the readers; nothing else builds one.
+    def initialize(kind, record, line, byte, expected = 0, found = 0)
+      @kind = kind
+      @record = record
+      @line = line
+      @byte = byte
+      @expected = expected
+      @found = found
+      super(describe)
+    end
+
+    # The failure as a Hash, which is what lets `case error in { kind:, line: }` match.
+    def deconstruct_keys(_keys)
+      { kind: kind, record: record, line: line, byte: byte, expected: expected, found: found }
+    end
+
+    private
+
+    # The message, in the words the other bindings use.
+    def describe
+      at = "#{record} (line #{line}, byte #{byte})"
+      case kind
+      when :column_count then "Record #{at} has #{found} cells; the first record had #{expected}."
+      when :unclosed_quote then "The input ended inside a quoted cell in record #{at}."
+      when :row_too_long then "Record #{at} exceeds the #{DelimitedReader::MAX_ROW_BYTES}-byte row ceiling."
+      when :not_a_zip then "The workbook is not a zip file."
+      when :encrypted then "The workbook is encrypted."
+      when :method then "Part #{record} of the workbook is compressed by method #{found}, neither stored nor deflate."
+      when :missing_part then "Part #{record}, which the workbook cannot be read without, is missing."
+      when :xml then "Part #{record} of the workbook ends inside an XML construct (byte #{byte})."
+      when :deflate then "Part #{record} of the workbook is not a whole deflate stream (byte #{byte})."
+      when :not_a_workbook then "The zip is neither an XLSX nor an ODS workbook."
+      when :shared_string then "Row #{line} names shared string #{found}; the table has #{expected}."
+      when :too_large then "The workbook holds more text than a batch can address."
+      else "The workbook's zip structure is broken."
+      end
+    end
+  end
+end