hypertabular-wasm 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,6 +15,8 @@ module HyperTabular
15
15
  OK = 0
16
16
  # A caller bug, never a data verdict.
17
17
  ERR_CONTRACT = -1
18
+ # The sizes hypertabular_batch_check was asked about are refused.
19
+ ERR_BATCH = -6
18
20
  # The data is structurally broken; the failure says where.
19
21
  ERR_STRUCTURE = -2
20
22
  # The arena cannot hold what one row needs.
@@ -36,8 +38,7 @@ module HyperTabular
36
38
  FILLED_BYTES = 64
37
39
  FILLED = "Q<4L<2Q<2L<2".freeze
38
40
  # Buffers: seven pointer and size pairs (window, arena, cells, row, strings, table,
39
- # kinds), as every workbook call takes them. A delimited fill reads the arena and the
40
- # cell table from it and nothing else.
41
+ # kinds), as the workbook calls but a fill take them.
41
42
  BUFFERS = "Q<14".freeze
42
43
  BUFFERS_BYTES = 112
43
44
 
@@ -71,46 +72,55 @@ module HyperTabular
71
72
  # The cell-table entries the reader holds.
72
73
  def cell_entries = @cells_cap
73
74
 
74
- # The loaded core's version word, major << 16 | minor << 8 | patch.
75
- def self.version
76
- Runtime.function(:hypertabular_version).call
77
- end
75
+ # The two entry points the Magnus extension also defines, in a module this class extends
76
+ # so the extension's definitions go in front of them instead of redefining them (see
77
+ # Runtime::FiddleBackend).
78
+ module FiddleBackend
79
+ # The loaded core's version word, major << 16 | minor << 8 | patch.
80
+ def version
81
+ Runtime.function(:hypertabular_version).call
82
+ end
78
83
 
79
- # +dialect+ is the four bytes of a RawDialect. Nil when the core refuses it.
80
- def self.start(dialect)
81
- reader = new
82
- reader.send(:init, dialect) ? reader : nil
84
+ # +options+ is the bytes of a RawDelimitedOptions and +tokens+ the dialect's tokens as the
85
+ # core takes them, or nil. Nil when the core refuses them.
86
+ def start(options, tokens = nil)
87
+ reader = new
88
+ reader.send(:init, options, tokens) ? reader : nil
89
+ end
83
90
  end
84
91
 
92
+ extend FiddleBackend
93
+
85
94
  def initialize
86
95
  @header = Runtime.function(:hypertabular_delimited_header)
87
- @fill = Runtime.function(:hypertabular_delimited_fill)
96
+ @fill = Runtime.function(:hypertabular_delimited_fill_with)
88
97
  @state = Runtime.buffer(Runtime.function(:hypertabular_delimited_state_size).call)
89
98
  @cramped = false
90
99
  @arena_cap = ARENA_BYTES
91
100
  @arena = Runtime.buffer(@arena_cap)
92
101
  @out = Runtime.buffer(FILLED_BYTES)
93
- @buffers = Runtime.buffer(BUFFERS_BYTES)
102
+ @buffers = Runtime.buffer(Columns::BUFFERS_WITH_BYTES)
94
103
  @rows = @consumed = @arena_used = 0
95
104
  end
96
105
 
97
- # Takes the plan: +specs+ is one packed ColumnSpec per plan column and +sizes+ the
98
- # bytes one value of each takes, +batch_rows+ the most rows a fill writes and +per_row+
99
- # the cell-table entries one row takes. Once, before the first #fill.
100
- def bind(specs, sizes, batch_rows, per_row)
106
+ # Takes the plan: +specs+ is one Column.specs entry per plan column and +sizes+ the
107
+ # bytes one value of each takes, +batch_rows+ the most rows a fill writes, +per_row+ the
108
+ # cell-table entries one row takes, +notes+ whether each column keeps its notes, +widths+
109
+ # whether each row's width is kept. Once, before the first #fill.
110
+ def bind(specs, sizes, batch_rows, per_row, notes = true, widths = false)
101
111
  @batch_rows = batch_rows
102
- @columns = Columns.new(specs, sizes, batch_rows)
103
- @per_row = per_row
104
- @cells_cap = per_row * Delimited.cell_rows(per_row, batch_rows)
105
- @cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
112
+ @columns = Columns.new(specs, sizes, batch_rows, notes, widths)
113
+ grow_cells(per_row, batch_rows)
106
114
  end
107
115
 
108
116
  # Names the String the calls that follow read: +start+ and +offset+ below index its
109
117
  # bytes. It is held where it is, not copied, and must not be modified until another
110
- # one is attached.
118
+ # one is attached. The one before is let go of only once this one is held, so that an
119
+ # exception along the way leaves the calls reading the one before, still held.
111
120
  def attach(input)
112
- @pin = Runtime.pin(input)
113
- @base = @pin.to_i
121
+ pin = Runtime.pin(input)
122
+ @base = pin.to_i
123
+ @pin = pin
114
124
  end
115
125
 
116
126
  # Reads the next record of the attached input, +length+ bytes from +start+, as a
@@ -125,8 +135,8 @@ module HyperTabular
125
135
  case code
126
136
  when OK, ERR_STRUCTURE then return code
127
137
  when ERR_CELLS
138
+ @names = Runtime.buffer(SPAN_BYTES * needed)
128
139
  @names_cap = needed
129
- @names = Runtime.buffer(SPAN_BYTES * @names_cap)
130
140
  when ERR_ARENA then grow_arena(needed)
131
141
  else raise CONTRACT
132
142
  end
@@ -162,9 +172,7 @@ module HyperTabular
162
172
  return code
163
173
  when ERR_STRUCTURE then return code
164
174
  when ERR_CELLS
165
- @per_row = [@per_row, needed].max
166
- @cells_cap = @per_row * Delimited.cell_rows(@per_row, @batch_rows)
167
- @cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
175
+ grow_cells([@per_row, needed].max, @batch_rows)
168
176
  when ERR_ARENA then grow_arena(needed)
169
177
  else raise CONTRACT
170
178
  end
@@ -203,17 +211,25 @@ module HyperTabular
203
211
 
204
212
  private
205
213
 
206
- def init(dialect)
207
- raw = Runtime.buffer(4)
208
- raw[0, 4] = dialect
209
- Runtime.function(:hypertabular_delimited_init).call(@state, raw) == OK
214
+ def init(options, tokens)
215
+ raw = Runtime.buffer(options.bytesize)
216
+ raw[0, options.bytesize] = options
217
+ unless tokens.nil?
218
+ @tokens_len = tokens.bytesize
219
+ @tokens = Runtime.buffer(@tokens_len)
220
+ @tokens[0, @tokens_len] = tokens
221
+ end
222
+ Runtime.function(:hypertabular_delimited_init_with).call(@state, raw) == OK
210
223
  end
211
224
 
212
- # The arena and the cell table as the Buffers block a fill takes them — packed again
213
- # for every call, since either may have been grown since the last.
225
+ # The arena, the cell table, the tokens and the row widths as the BuffersWith block a fill
226
+ # takes them — packed again for every call, since either of the first two may have been grown
227
+ # since the last.
214
228
  def buffers
215
- @buffers[0, BUFFERS_BYTES] = [0, 0, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, 0, 0, 0, 0, 0, 0, 0, 0]
216
- .pack(BUFFERS)
229
+ @buffers[0, Columns::BUFFERS_WITH_BYTES] = Columns.word(Columns::BUFFERS_WITH_BYTES) +
230
+ [0, 0, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, 0, 0,
231
+ 0, 0, 0, 0, 0, 0, @tokens.to_i, @tokens_len.to_i,
232
+ @columns.widths_address].pack("J*")
217
233
  @buffers
218
234
  end
219
235
 
@@ -223,9 +239,19 @@ module HyperTabular
223
239
  needed
224
240
  end
225
241
 
242
+ # Each capacity moves only with the block behind it: the allocation can raise and the
243
+ # caller can read on, and the next call hands the core this capacity with this block.
244
+ def grow_cells(per_row, batch_rows)
245
+ capacity = per_row * Delimited.cell_rows(per_row, batch_rows)
246
+ @cells = Runtime.buffer(SPAN_BYTES * capacity)
247
+ @cells_cap = capacity
248
+ @per_row = per_row
249
+ end
250
+
226
251
  def grow_arena(needed)
227
- @arena_cap = [needed, @arena_cap * 2].max
228
- @arena = Runtime.buffer(@arena_cap)
252
+ capacity = [needed, @arena_cap * 2].max
253
+ @arena = Runtime.buffer(capacity)
254
+ @arena_cap = capacity
229
255
  end
230
256
  end
231
257
  end
@@ -5,8 +5,9 @@ module HyperTabular
5
5
  # which never see a pointer.
6
6
  #
7
7
  # A workbook call works in buffers the caller hands it (a Buffers block of seven pointer
8
- # and size pairs), and when one is too small it says which and how large, having undone
9
- # nothing: the buffer is grown with what it held kept, and the same call made again.
8
+ # and size pairs, or a BuffersWith with tokens and row widths for a fill), and when one is
9
+ # too small it says which and how large, having undone nothing: the buffer is grown with
10
+ # what it held kept, and the same call made again.
10
11
  module Book
11
12
  class << self
12
13
  # For the specs: every buffer a workbook call works in starts with room for one
@@ -78,6 +79,18 @@ module HyperTabular
78
79
  @buffers
79
80
  end
80
81
 
82
+ # As #buffers, for a fill: with the sheet's +tokens+ (a Pointer, or nil) and their length, and
83
+ # the address the row widths go, or 0. Packed again for every call, since any of them may
84
+ # have been grown since the last.
85
+ def buffers_with(tables, tokens, tokens_len, widths)
86
+ @buffers_with ||= Runtime.buffer(Columns::BUFFERS_WITH_BYTES)
87
+ @buffers_with[0, Columns::BUFFERS_WITH_BYTES] =
88
+ Columns.word(Columns::BUFFERS_WITH_BYTES) +
89
+ [@window.to_i, @window_cap, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, @row.to_i, @row_cap,
90
+ *tables.to_a, tokens.to_i, tokens_len, widths].pack("J*")
91
+ @buffers_with
92
+ end
93
+
81
94
  # Makes the window at least +needed+ bytes, what it held kept.
82
95
  def grow_window(needed)
83
96
  @window, @window_cap = grown(@window, @window_cap, needed, 1)
@@ -108,7 +121,8 @@ module HyperTabular
108
121
 
109
122
  # Makes +call+ (a block taking the Buffers block and the out block) until it stops
110
123
  # asking for room, growing the buffer it names each time. Returns the code it ended
111
- # on and the Filled fields: rows, consumed, arena_used, needed, then the failure.
124
+ # on and the Filled fields: rows, consumed, arena_used, needed, then the failure. A fill's
125
+ # block makes its own BuffersWith, from the buffers as they are.
112
126
  def drive(out, tables = nil)
113
127
  loop do
114
128
  code = yield buffers(tables)
@@ -234,28 +248,37 @@ module HyperTabular
234
248
  # and one for the row's number. Nil until #bind.
235
249
  attr_reader :columns, :per_row
236
250
 
237
- # Positions a copy of the workbook's state on +sheet+ (an entry of Opened#sheets).
238
- def initialize(book, sheet, has_header, skip_empty_rows, batch_rows)
251
+ # Positions a copy of the workbook's state on +sheet+ (an entry of Opened#sheets), as
252
+ # +options+, the bytes of a RawSheetOptions, say. +tokens+ is the sheet's tokens as the core
253
+ # takes them, or nil.
254
+ def initialize(book, sheet, options, batch_rows, tokens = nil)
239
255
  @book = book
240
256
  @state = book.copy_of_state
241
257
  @batch_rows = batch_rows
242
258
  @scratch = Book.stingy ? Scratch.new(1, 1, 1, 0) : Scratch.new(0, 4096, 64, 0)
243
259
  @out = Runtime.buffer(FILLED_BYTES)
260
+ unless tokens.nil?
261
+ @tokens_len = tokens.bytesize
262
+ @tokens = Runtime.buffer(@tokens_len)
263
+ @tokens[0, @tokens_len] = tokens
264
+ end
244
265
  _name, _hidden, part, index = sheet
245
- code = Runtime.function(:hypertabular_workbook_sheet).call(
246
- @state, book.container, book.length, Runtime.pin(part), part.bytesize, index,
247
- has_header ? 1 : 0, skip_empty_rows ? 1 : 0, @out
266
+ raw = Runtime.buffer(options.bytesize)
267
+ raw[0, options.bytesize] = options
268
+ code = Runtime.function(:hypertabular_workbook_sheet_with).call(
269
+ @state, book.container, book.length, Runtime.pin(part), part.bytesize, index, raw, @out
248
270
  )
249
271
  Book.settle(code, @out[0, FILLED_BYTES].unpack(FILLED))
250
272
  end
251
273
 
252
- # Takes the plan: +specs+ is one packed ColumnSpec per plan column, +sizes+ the bytes
253
- # one value of each takes, and +width+ the slots a row needs to reach every source
254
- # column the plan reads. Once, before the first #fill. The core keeps a header row's
274
+ # Takes the plan: +specs+ is one Column.specs entry per plan column, +sizes+ the bytes
275
+ # one value of each takes, +width+ the slots a row needs to reach every source
276
+ # column the plan reads, +notes+ whether each column keeps its notes and +widths+ whether
277
+ # each row's width is kept. Once, before the first #fill. The core keeps a header row's
255
278
  # cells in the row's slots — a row the sheet repeats (an ODS number-rows-repeated) is
256
279
  # delivered again from them — so the slots are only ever grown here, never replaced.
257
- def bind(specs, sizes, width)
258
- @columns = Columns.new(specs, sizes, @batch_rows)
280
+ def bind(specs, sizes, width, notes = true, widths = false)
281
+ @columns = Columns.new(specs, sizes, @batch_rows, notes, widths)
259
282
  @per_row = specs.size + 1
260
283
  # A stingy scratch is left to grow into the cell table the hard way, mid-read.
261
284
  @scratch.reserve_cells(@batch_rows * @per_row) unless Book.stingy
@@ -267,8 +290,9 @@ module HyperTabular
267
290
  # record, byte, expected, found] — that came with them, if one did: the caller raises
268
291
  # it after them. The table and the arena are copies.
269
292
  def fill
270
- call = Runtime.function(:hypertabular_workbook_fill)
271
- code, filled = @scratch.drive(@out, @book.tables) do |buffers|
293
+ call = Runtime.function(:hypertabular_workbook_fill_with)
294
+ code, filled = @scratch.drive(@out, @book.tables) do
295
+ buffers = @scratch.buffers_with(@book.tables, @tokens, @tokens_len.to_i, @columns.widths_address)
272
296
  call.call(@state, @book.container, @book.length, @columns.specs, @columns.table,
273
297
  @columns.count, @columns.batch_rows, buffers, @out)
274
298
  end
@@ -16,16 +16,30 @@ module HyperTabular
16
16
  EXPORTS = {
17
17
  hypertabular_version: [[], :uint32],
18
18
  hypertabular_delimited_state_size: [[], :size],
19
- # (state, dialect)
19
+ # (state, dialect): 1.0's, which no read takes; a spec holds it to what it always read.
20
20
  hypertabular_delimited_init: [%i[pointer pointer], :int32],
21
+ # (state, options), the RawDelimitedOptions every read starts from.
22
+ hypertabular_delimited_init_with: [%i[pointer pointer], :int32],
23
+ # (encoding, input, input_len, last, out, cap, progress)
24
+ hypertabular_transcode: [%i[uint32 pointer size uint32 pointer size pointer], :int32],
21
25
  # (state, input, input_len, last, names, names_cap, arena, arena_cap, out)
22
26
  hypertabular_delimited_header:
23
27
  [%i[pointer pointer size uint32 pointer size pointer size pointer], :int32],
24
- # (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out) —
25
- # buffers is the Buffers block the workbook calls take, of which a delimited fill reads
26
- # only the arena and the cell table.
28
+ # (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out):
29
+ # 1.0's, which no read takes, reading every format Western; a spec holds it to that.
27
30
  hypertabular_delimited_fill:
28
31
  [%i[pointer pointer size uint32 pointer pointer size size pointer pointer], :int32],
32
+ # (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out) —
33
+ # buffers is the BuffersWith block, of which a delimited fill reads the arena, the cell
34
+ # table, the tokens and the row widths.
35
+ hypertabular_delimited_fill_with:
36
+ [%i[pointer pointer size uint32 pointer pointer size size pointer pointer], :int32],
37
+ # (columns, column_count, rows, row_widths, expected_width, long_rows, cursor, out, cap, written)
38
+ hypertabular_faults:
39
+ [%i[pointer size size pointer uint32 uint32 pointer pointer size pointer], :int32],
40
+ # (input, input_len, arena, arena_len, cells, typed, rows, columns, formats, out)
41
+ hypertabular_suggest:
42
+ [%i[pointer size pointer size pointer pointer size size pointer pointer], :int32],
29
43
  # (cell, len, out, cap)
30
44
  hypertabular_delimited_unescape: [%i[pointer size pointer size], :size],
31
45
  hypertabular_workbook_state_size: [[], :size],
@@ -34,17 +48,180 @@ module HyperTabular
34
48
  hypertabular_workbook_sheets: [%i[pointer pointer size pointer pointer], :int32],
35
49
  hypertabular_workbook_strings: [%i[pointer pointer size pointer pointer], :int32],
36
50
  hypertabular_workbook_styles: [%i[pointer pointer size pointer pointer], :int32],
37
- # (state, container, container_len, part, part_len, index, has_header, skip_empty_rows, out)
51
+ # (state, container, container_len, part, part_len, index, has_header, skip_empty_rows, out):
52
+ # 1.0's, which no read takes.
38
53
  hypertabular_workbook_sheet:
39
54
  [%i[pointer pointer size pointer size uint32 uint32 uint32 pointer], :int32],
55
+ # (state, container, container_len, part, part_len, index, options, out)
56
+ hypertabular_workbook_sheet_with:
57
+ [%i[pointer pointer size pointer size uint32 pointer pointer], :int32],
40
58
  hypertabular_workbook_header: [%i[pointer pointer size pointer pointer], :int32],
41
- # (state, container, container_len, specs, columns, column_count, max_rows, buffers, out)
42
- hypertabular_workbook_fill: [%i[pointer pointer size pointer pointer size size pointer pointer], :int32]
59
+ # (state, container, container_len, specs, columns, column_count, max_rows, buffers, out):
60
+ # 1.0's, which no read takes.
61
+ hypertabular_workbook_fill: [%i[pointer pointer size pointer pointer size size pointer pointer], :int32],
62
+ # (state, container, container_len, specs, columns, column_count, max_rows, buffers, out) —
63
+ # buffers is the BuffersWith block.
64
+ hypertabular_workbook_fill_with:
65
+ [%i[pointer pointer size pointer pointer size size pointer pointer], :int32],
66
+ # (text, len, out)
67
+ hypertabular_range_parse: [%i[pointer size pointer], :int32],
68
+ hypertabular_batch_check: [%i[pointer size uint64 uint64 uint32 pointer], :int32]
43
69
  }.freeze
44
70
 
45
71
  @mutex = Mutex.new
46
72
  @functions = nil
47
73
 
74
+ # The entry points the Magnus extension also defines, the Fiddle way. They live in a module
75
+ # Runtime extends rather than on its singleton class, so the extension's
76
+ # define_singleton_method puts its definition in front of each: an override through
77
+ # the ancestor chain, not a redefinition, and `ruby -w` has nothing to say about it (1.0.0
78
+ # printed "method redefined" for it, and for Delimited.start and .version, on every
79
+ # require under -w). With the extension loaded no call ever reaches it.
80
+ module FiddleBackend
81
+ # A quoted delimited cell — the bytes a flagged cell-table entry names — with its
82
+ # quotes resolved, as the core cast it.
83
+ def unescape(quoted)
84
+ input = pin(quoted)
85
+ out = buffer([quoted.bytesize, 1].max)
86
+ written = function(:hypertabular_delimited_unescape).call(input, quoted.bytesize, out, quoted.bytesize)
87
+ out[0, written]
88
+ end
89
+
90
+ # The index of the first of +specs+ (Column.specs' form) the core refuses, or nil. A fill of
91
+ # no rows judges a column's door, what it declares, its format, date options, vocabulary and
92
+ # tokens and constraints, and writes nothing. It answers on a state of its own, so a sheet's
93
+ # plan is asked the same way.
94
+ def refused(specs)
95
+ state = buffer(function(:hypertabular_delimited_state_size).call)
96
+ options = buffer(Dialect::OPTIONS_BYTES)
97
+ options[0, Dialect::OPTIONS_BYTES] = Dialect::CSV.packed_with
98
+ raise Delimited::CONTRACT unless
99
+ function(:hypertabular_delimited_init_with).call(state, options) == Delimited::OK
100
+
101
+ value = buffer(32)
102
+ verdict = buffer(Columns::VERDICT_BYTES)
103
+ # A constrained column without checks is refused, so each is asked with them.
104
+ check = buffer(Columns::CHECK_BYTES)
105
+ table = buffer(Columns::COLUMN_BUFFER_BYTES)
106
+ table[0, Columns::COLUMN_BUFFER_BYTES] = Columns.column_buffer(value, verdict, nil, check)
107
+ memory = buffer(Columns::BUFFERS_WITH_BYTES)
108
+ memory[0, Columns::WORD] = Columns.word(Columns::BUFFERS_WITH_BYTES)
109
+ out = buffer(Delimited::FILLED_BYTES)
110
+ fill = function(:hypertabular_delimited_fill_with)
111
+ specs.index do |spec|
112
+ lists = []
113
+ bytes = Columns.spec_with(spec, lists)
114
+ held = buffer(bytes.bytesize)
115
+ held[0, bytes.bytesize] = bytes
116
+ fill.call(state, nil, 0, 0, held, table, 1, 0, memory, out) == Delimited::ERR_CONTRACT
117
+ end
118
+ end
119
+
120
+ # The next step of a declared encoding's decode: +length+ bytes of +input+ from +start+, with
121
+ # +last+ saying nothing follows them, into at most +capacity+ bytes of UTF-8. Answers
122
+ # [consumed, written, failed, failure_at, out], +out+ a binary String of its own.
123
+ def transcode(encoding, input, start, length, last, capacity)
124
+ out = buffer(capacity)
125
+ progress = buffer(TRANSCODED_BYTES)
126
+ base = pin(input)
127
+ raise Delimited::CONTRACT unless
128
+ function(:hypertabular_transcode).call(encoding, base.to_i + start, length, last ? 1 : 0, out, capacity,
129
+ progress) == Delimited::OK
130
+
131
+ consumed, written, failure_at, failed = progress[0, TRANSCODED_BYTES].unpack("Q<3L<")
132
+ [consumed, written, failed, failure_at, out[0, written]]
133
+ end
134
+
135
+ # Every RawProblem of a batch, as one binary String: +verdicts+ is each column's array as the
136
+ # core wrote it, +notes+ the same or nil, +checks+ each column's RawCheck array or nil (and
137
+ # nil for every column of a plan that declares no constraints), and +widths+ the row widths
138
+ # or nil.
139
+ def faults(verdicts, notes, checks, rows, widths, expected, long_rows)
140
+ pinned = verdicts.map { |column| pin(column) }
141
+ noted = notes&.map { |column| pin(column) }
142
+ checked = checks&.map { |column| column && pin(column) }
143
+ table = buffer([Columns::COLUMN_BUFFER_BYTES * verdicts.size, 1].max)
144
+ verdicts.each_index do |column|
145
+ table[Columns::COLUMN_BUFFER_BYTES * column, Columns::COLUMN_BUFFER_BYTES] =
146
+ Columns.column_buffer(nil, pinned[column], noted && noted[column], checked && checked[column])
147
+ end
148
+ held = widths && pin(widths)
149
+ cursor = buffer(8)
150
+ written = buffer(Columns::WORD)
151
+ out = buffer(FAULTS_BYTES)
152
+ call = function(:hypertabular_faults)
153
+ listed = "".b
154
+ loop do
155
+ code = call.call(table, verdicts.size, rows, held || 0, expected, long_rows, cursor, out,
156
+ FAULTS_BYTES / PROBLEM_BYTES, written)
157
+ raise Delimited::CONTRACT unless code == Delimited::OK
158
+
159
+ entries = written[0, Columns::WORD].unpack1("J")
160
+ listed << out[0, PROBLEM_BYTES * entries]
161
+ return listed if entries < FAULTS_BYTES / PROBLEM_BYTES
162
+ end
163
+ end
164
+
165
+ # What the core makes of a sample: door, param, status, matched and sampled for each of
166
+ # +columns+, flat, or nil if it refuses. +input+ is the sample's bytes, +cells+ its spans a
167
+ # column after another, +typed+ a byte for each or nil, +formats+ a packed format for each.
168
+ def suggest(input, cells, typed, rows, columns, formats)
169
+ out = buffer([SUGGESTION_BYTES * columns, 1].max)
170
+ pins = [input, cells, typed, formats].map { |text| text && pin(text) }
171
+ code = function(:hypertabular_suggest).call(pins[0], input.bytesize, nil, 0, pins[1], pins[2] || 0, rows,
172
+ columns, pins[3], out)
173
+ code == Delimited::OK ? out[0, SUGGESTION_BYTES * columns].unpack("L<*") : nil
174
+ end
175
+
176
+ # The core's hypertabular_batch_check: [code, needed, limit, buffer_bytes, refused,
177
+ # sample_rows, width]. The Magnus extension defines its own, as it does range_parse.
178
+ def batch_check(specs, batch_rows, buffer_bytes, flags)
179
+ lists = []
180
+ packed = specs.map { |spec| Columns.spec_with(spec, lists) }.join
181
+ held = buffer(packed.bytesize)
182
+ held[0, packed.bytesize] = packed unless packed.empty?
183
+ out = buffer(BATCH_CHECK_BYTES)
184
+ code = function(:hypertabular_batch_check).call(held, specs.size, batch_rows, buffer_bytes, flags, out)
185
+ needed, limit, start, sample_rows, width, refused = out[0, BATCH_CHECK_BYTES].unpack("Q<5L<")
186
+ [code, needed, limit, start, refused, sample_rows, width]
187
+ end
188
+
189
+ # A range's four corners as the core reads them from +text+, or nil if it isn't one.
190
+ def range_parse(text)
191
+ out = buffer(RANGE_BYTES)
192
+ return nil unless function(:hypertabular_range_parse).call(pin(text), text.bytesize, out) == Delimited::OK
193
+
194
+ out[0, RANGE_BYTES].unpack("L<4")
195
+ end
196
+ end
197
+
198
+ # Flags to hypertabular_batch_check (RawBatchCheck in the core).
199
+ BATCH_NOTES = 1
200
+ BATCH_WIDTHS = 2
201
+ BATCH_SHEET = 4
202
+ # RawBatchCheck's refused codes.
203
+ BATCH_ROWS_ZERO = 1
204
+ BUFFER_BYTES_ZERO = 2
205
+ ROW_TOO_WIDE = 4
206
+ # Bytes in one RawBatchCheck.
207
+ BATCH_CHECK_BYTES = 48
208
+ # Bytes in one RawTranscoded.
209
+ TRANSCODED_BYTES = 32
210
+ # Bytes in one RawProblem.
211
+ PROBLEM_BYTES = 36
212
+ # The room one hypertabular_faults call lists into.
213
+ FAULTS_BYTES = PROBLEM_BYTES * 256
214
+ # Bytes in one RawSuggestion.
215
+ SUGGESTION_BYTES = 20
216
+ # Bytes in one RawRange.
217
+ RANGE_BYTES = 16
218
+ # The allocator behind every buffer, which has to be the one RUBY_FREE frees: Ruby's own in
219
+ # CRuby, and libc's in Fiddle's FFI backend, the one every other Ruby takes, whose RUBY_FREE is
220
+ # libc's free.
221
+ ZEROING = RUBY_ENGINE == "ruby" ? "ruby_xcalloc" : "calloc"
222
+
223
+ extend FiddleBackend
224
+
48
225
  class << self
49
226
  # The export's Fiddle::Function, for the caller to invoke directly.
50
227
  def function(symbol)
@@ -54,9 +231,63 @@ module HyperTabular
54
231
  # Every native allocation goes through here, and loads the library first: that is what
55
232
  # keeps a missing library reported as missing_library_message instead of as whatever
56
233
  # touching Fiddle raises first. Zeroed, freed with the object that holds it.
234
+ #
235
+ # Many of these are sized by a caller, so nothing here may touch the memory: pages nobody
236
+ # reads or writes stay the OS's. Fiddle::Pointer.malloc writes every byte. ruby_xcalloc asks
237
+ # calloc, counts the bytes towards the next collection as Pointer.malloc does, pairs with
238
+ # the RUBY_FREE it has always been freed with, and raises NoMemoryError for what it can't get.
239
+ #
240
+ # Fiddle::Pointer takes its size as a C long, 32 bits on 64-bit Windows. A size past that is
241
+ # refused here, before the allocation it would otherwise strand.
57
242
  def buffer(size)
58
243
  functions
59
- Fiddle::Pointer.malloc(size, Fiddle::RUBY_FREE)
244
+ raise NoMemoryError, "failed to allocate memory" if size >= 1 << ((Fiddle::SIZEOF_LONG * 8) - 1)
245
+
246
+ # A zero-sized request still answers an address of its own. TruffleRuby and JRuby's
247
+ # allocator is calloc, which answers null where ruby_xcalloc raises.
248
+ address = zeroing.call(1, [size, 1].max).to_i
249
+ raise NoMemoryError, "failed to allocate memory" if address.zero?
250
+
251
+ Fiddle::Pointer.new(address, size, Fiddle::RUBY_FREE)
252
+ end
253
+
254
+ # Asks the core whether a read may ask for this much, before anything is allocated for it:
255
+ # the size to start the input buffer at, or an ArgumentError in the core's terms. +specs+ is
256
+ # Column.specs' answer, [] where no plan is known yet, and +flags+ is batch_flags' answer. What
257
+ # is here is only what a count has to be for the core to be asked at all.
258
+ def check_sizes(specs, batch_rows, buffer_bytes, flags)
259
+ counted("batch_rows", batch_rows)
260
+ counted("buffer_bytes", buffer_bytes)
261
+ code, needed, limit, start, refused, _, width = batch_check(specs, batch_rows, buffer_bytes, flags)
262
+ return start if code == Delimited::OK
263
+ raise Delimited::CONTRACT unless code == Delimited::ERR_BATCH
264
+
265
+ raise ArgumentError, case refused
266
+ when BATCH_ROWS_ZERO then "batch_rows must be at least 1"
267
+ when BUFFER_BYTES_ZERO then "buffer_bytes must be at least 1"
268
+ when ROW_TOO_WIDE
269
+ "a row of #{width} columns asks for #{needed} bytes, past the limit of #{limit}"
270
+ else "batch_rows asks for #{needed} bytes, past the limit of #{limit}"
271
+ end
272
+ end
273
+
274
+ # The rows of a sampler's batch for a sampler wanting +wanted+ rows: the core's answer, however
275
+ # many batches that takes. Only a row past the limit is refused, in its words.
276
+ def sample_rows(specs, wanted, flags)
277
+ _, _, _, _, _, rows, = batch_check(specs, wanted, 1, flags)
278
+ check_sizes(specs, wanted, 1, flags) if rows.zero?
279
+ rows
280
+ end
281
+
282
+ # The flags batch_check takes, for a read that keeps +notes+ and +widths+.
283
+ def batch_flags(notes, widths, sheet: false)
284
+ (notes ? BATCH_NOTES : 0) | (widths ? BATCH_WIDTHS : 0) | (sheet ? BATCH_SHEET : 0)
285
+ end
286
+
287
+ private def counted(name, count)
288
+ raise ArgumentError, "#{name} must be an Integer; got #{count.inspect}" unless count.is_a?(Integer)
289
+ raise ArgumentError, "#{name} must not be negative; got #{count}" if count.negative?
290
+ raise ArgumentError, "#{name} must fit in 64 bits; got #{count}" if count >= 1 << 64
60
291
  end
61
292
 
62
293
  # A pointer to a String's own bytes — nothing is copied. While the returned object is
@@ -68,16 +299,22 @@ module HyperTabular
68
299
  Fiddle::Pointer[string]
69
300
  end
70
301
 
71
- # A quoted delimited cell — the bytes a flagged cell-table entry names — with its
72
- # quotes resolved, as the core cast it.
73
- def unescape(quoted)
74
- input = pin(quoted)
75
- out = buffer([quoted.bytesize, 1].max)
76
- written = function(:hypertabular_delimited_unescape).call(input, quoted.bytesize, out, quoted.bytesize)
77
- out[0, written]
302
+ private
303
+
304
+ # The allocator, resolved beside the exports, so a Ruby that doesn't have it is a LoadError
305
+ # that available? answers for, not a failure at the first read. ruby_xcalloc needs the lock
306
+ # Fiddle would otherwise let go of around a call.
307
+ def zeroing
308
+ functions
309
+ @zeroing
78
310
  end
79
311
 
80
- private
312
+ def load_zeroing
313
+ Fiddle::Function.new(Fiddle.dlopen(nil)[ZEROING], [Fiddle::TYPE_SIZE_T, Fiddle::TYPE_SIZE_T],
314
+ Fiddle::TYPE_VOIDP, need_gvl: true)
315
+ rescue Fiddle::DLError => e
316
+ raise LoadError, "HyperTabular's Fiddle backend can't find #{ZEROING} in this Ruby: #{e.message}"
317
+ end
81
318
 
82
319
  # The shared library to dlopen: this install's native/{rid}/{lib}, or — the
83
320
  # development loop — the in-repo cargo build, exactly what the other bindings' local
@@ -127,9 +364,10 @@ module HyperTabular
127
364
  raise LoadError, missing_library_message if path.nil?
128
365
 
129
366
  handle = Fiddle.dlopen(path)
367
+ @zeroing = load_zeroing
130
368
  types = {
131
369
  pointer: Fiddle::TYPE_VOIDP, size: Fiddle::TYPE_SIZE_T,
132
- uint32: Fiddle::TYPE_UINT32_T, int32: Fiddle::TYPE_INT32_T
370
+ uint32: Fiddle::TYPE_UINT32_T, int32: Fiddle::TYPE_INT32_T, uint64: Fiddle::TYPE_UINT64_T
133
371
  }
134
372
  EXPORTS.to_h do |name, (arguments, result)|
135
373
  [name, Fiddle::Function.new(handle[name.to_s], arguments.map { |type|