hypertabular-wasm 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE-UNICODE +39 -0
- data/README.md +278 -34
- data/ext/hypertabular_native/3.4/hypertabular_native.a.gz +0 -0
- data/ext/hypertabular_native/4.0/hypertabular_native.a.gz +0 -0
- data/lib/hypertabular/batch.rb +219 -24
- data/lib/hypertabular/column.rb +166 -21
- data/lib/hypertabular/delimited_reader.rb +319 -94
- data/lib/hypertabular/dialect.rb +92 -10
- data/lib/hypertabular/guarded.rb +116 -0
- data/lib/hypertabular/header.rb +59 -22
- data/lib/hypertabular/row.rb +17 -4
- data/lib/hypertabular/runtime/columns.rb +78 -12
- data/lib/hypertabular/runtime/delimited.rb +63 -37
- data/lib/hypertabular/runtime/workbook.rb +39 -15
- data/lib/hypertabular/runtime.rb +255 -17
- data/lib/hypertabular/suggest.rb +283 -0
- data/lib/hypertabular/tabular_error.rb +9 -3
- data/lib/hypertabular/workbook.rb +185 -45
- data/lib/hypertabular.rb +14 -8
- metadata +7 -3
|
@@ -15,6 +15,8 @@ module HyperTabular
|
|
|
15
15
|
OK = 0
|
|
16
16
|
# A caller bug, never a data verdict.
|
|
17
17
|
ERR_CONTRACT = -1
|
|
18
|
+
# The sizes hypertabular_batch_check was asked about are refused.
|
|
19
|
+
ERR_BATCH = -6
|
|
18
20
|
# The data is structurally broken; the failure says where.
|
|
19
21
|
ERR_STRUCTURE = -2
|
|
20
22
|
# The arena cannot hold what one row needs.
|
|
@@ -36,8 +38,7 @@ module HyperTabular
|
|
|
36
38
|
FILLED_BYTES = 64
|
|
37
39
|
FILLED = "Q<4L<2Q<2L<2".freeze
|
|
38
40
|
# Buffers: seven pointer and size pairs (window, arena, cells, row, strings, table,
|
|
39
|
-
# kinds), as
|
|
40
|
-
# cell table from it and nothing else.
|
|
41
|
+
# kinds), as the workbook calls but a fill take them.
|
|
41
42
|
BUFFERS = "Q<14".freeze
|
|
42
43
|
BUFFERS_BYTES = 112
|
|
43
44
|
|
|
@@ -71,46 +72,55 @@ module HyperTabular
|
|
|
71
72
|
# The cell-table entries the reader holds.
|
|
72
73
|
def cell_entries = @cells_cap
|
|
73
74
|
|
|
74
|
-
# The
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
# The two entry points the Magnus extension also defines, in a module this class extends
|
|
76
|
+
# so the extension's definitions go in front of them instead of redefining them (see
|
|
77
|
+
# Runtime::FiddleBackend).
|
|
78
|
+
module FiddleBackend
|
|
79
|
+
# The loaded core's version word, major << 16 | minor << 8 | patch.
|
|
80
|
+
def version
|
|
81
|
+
Runtime.function(:hypertabular_version).call
|
|
82
|
+
end
|
|
78
83
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
# +options+ is the bytes of a RawDelimitedOptions and +tokens+ the dialect's tokens as the
|
|
85
|
+
# core takes them, or nil. Nil when the core refuses them.
|
|
86
|
+
def start(options, tokens = nil)
|
|
87
|
+
reader = new
|
|
88
|
+
reader.send(:init, options, tokens) ? reader : nil
|
|
89
|
+
end
|
|
83
90
|
end
|
|
84
91
|
|
|
92
|
+
extend FiddleBackend
|
|
93
|
+
|
|
85
94
|
def initialize
|
|
86
95
|
@header = Runtime.function(:hypertabular_delimited_header)
|
|
87
|
-
@fill = Runtime.function(:
|
|
96
|
+
@fill = Runtime.function(:hypertabular_delimited_fill_with)
|
|
88
97
|
@state = Runtime.buffer(Runtime.function(:hypertabular_delimited_state_size).call)
|
|
89
98
|
@cramped = false
|
|
90
99
|
@arena_cap = ARENA_BYTES
|
|
91
100
|
@arena = Runtime.buffer(@arena_cap)
|
|
92
101
|
@out = Runtime.buffer(FILLED_BYTES)
|
|
93
|
-
@buffers = Runtime.buffer(
|
|
102
|
+
@buffers = Runtime.buffer(Columns::BUFFERS_WITH_BYTES)
|
|
94
103
|
@rows = @consumed = @arena_used = 0
|
|
95
104
|
end
|
|
96
105
|
|
|
97
|
-
# Takes the plan: +specs+ is one
|
|
98
|
-
# bytes one value of each takes, +batch_rows+ the most rows a fill writes
|
|
99
|
-
#
|
|
100
|
-
|
|
106
|
+
# Takes the plan: +specs+ is one Column.specs entry per plan column and +sizes+ the
|
|
107
|
+
# bytes one value of each takes, +batch_rows+ the most rows a fill writes, +per_row+ the
|
|
108
|
+
# cell-table entries one row takes, +notes+ whether each column keeps its notes, +widths+
|
|
109
|
+
# whether each row's width is kept. Once, before the first #fill.
|
|
110
|
+
def bind(specs, sizes, batch_rows, per_row, notes = true, widths = false)
|
|
101
111
|
@batch_rows = batch_rows
|
|
102
|
-
@columns = Columns.new(specs, sizes, batch_rows)
|
|
103
|
-
|
|
104
|
-
@cells_cap = per_row * Delimited.cell_rows(per_row, batch_rows)
|
|
105
|
-
@cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
|
|
112
|
+
@columns = Columns.new(specs, sizes, batch_rows, notes, widths)
|
|
113
|
+
grow_cells(per_row, batch_rows)
|
|
106
114
|
end
|
|
107
115
|
|
|
108
116
|
# Names the String the calls that follow read: +start+ and +offset+ below index its
|
|
109
117
|
# bytes. It is held where it is, not copied, and must not be modified until another
|
|
110
|
-
# one is attached.
|
|
118
|
+
# one is attached. The one before is let go of only once this one is held, so that an
|
|
119
|
+
# exception along the way leaves the calls reading the one before, still held.
|
|
111
120
|
def attach(input)
|
|
112
|
-
|
|
113
|
-
@base =
|
|
121
|
+
pin = Runtime.pin(input)
|
|
122
|
+
@base = pin.to_i
|
|
123
|
+
@pin = pin
|
|
114
124
|
end
|
|
115
125
|
|
|
116
126
|
# Reads the next record of the attached input, +length+ bytes from +start+, as a
|
|
@@ -125,8 +135,8 @@ module HyperTabular
|
|
|
125
135
|
case code
|
|
126
136
|
when OK, ERR_STRUCTURE then return code
|
|
127
137
|
when ERR_CELLS
|
|
138
|
+
@names = Runtime.buffer(SPAN_BYTES * needed)
|
|
128
139
|
@names_cap = needed
|
|
129
|
-
@names = Runtime.buffer(SPAN_BYTES * @names_cap)
|
|
130
140
|
when ERR_ARENA then grow_arena(needed)
|
|
131
141
|
else raise CONTRACT
|
|
132
142
|
end
|
|
@@ -162,9 +172,7 @@ module HyperTabular
|
|
|
162
172
|
return code
|
|
163
173
|
when ERR_STRUCTURE then return code
|
|
164
174
|
when ERR_CELLS
|
|
165
|
-
|
|
166
|
-
@cells_cap = @per_row * Delimited.cell_rows(@per_row, @batch_rows)
|
|
167
|
-
@cells = Runtime.buffer(SPAN_BYTES * @cells_cap)
|
|
175
|
+
grow_cells([@per_row, needed].max, @batch_rows)
|
|
168
176
|
when ERR_ARENA then grow_arena(needed)
|
|
169
177
|
else raise CONTRACT
|
|
170
178
|
end
|
|
@@ -203,17 +211,25 @@ module HyperTabular
|
|
|
203
211
|
|
|
204
212
|
private
|
|
205
213
|
|
|
206
|
-
def init(
|
|
207
|
-
raw = Runtime.buffer(
|
|
208
|
-
raw[0,
|
|
209
|
-
|
|
214
|
+
def init(options, tokens)
|
|
215
|
+
raw = Runtime.buffer(options.bytesize)
|
|
216
|
+
raw[0, options.bytesize] = options
|
|
217
|
+
unless tokens.nil?
|
|
218
|
+
@tokens_len = tokens.bytesize
|
|
219
|
+
@tokens = Runtime.buffer(@tokens_len)
|
|
220
|
+
@tokens[0, @tokens_len] = tokens
|
|
221
|
+
end
|
|
222
|
+
Runtime.function(:hypertabular_delimited_init_with).call(@state, raw) == OK
|
|
210
223
|
end
|
|
211
224
|
|
|
212
|
-
# The arena
|
|
213
|
-
# for every call, since either may have been grown
|
|
225
|
+
# The arena, the cell table, the tokens and the row widths as the BuffersWith block a fill
|
|
226
|
+
# takes them — packed again for every call, since either of the first two may have been grown
|
|
227
|
+
# since the last.
|
|
214
228
|
def buffers
|
|
215
|
-
@buffers[0,
|
|
216
|
-
|
|
229
|
+
@buffers[0, Columns::BUFFERS_WITH_BYTES] = Columns.word(Columns::BUFFERS_WITH_BYTES) +
|
|
230
|
+
[0, 0, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, 0, 0,
|
|
231
|
+
0, 0, 0, 0, 0, 0, @tokens.to_i, @tokens_len.to_i,
|
|
232
|
+
@columns.widths_address].pack("J*")
|
|
217
233
|
@buffers
|
|
218
234
|
end
|
|
219
235
|
|
|
@@ -223,9 +239,19 @@ module HyperTabular
|
|
|
223
239
|
needed
|
|
224
240
|
end
|
|
225
241
|
|
|
242
|
+
# Each capacity moves only with the block behind it: the allocation can raise and the
|
|
243
|
+
# caller can read on, and the next call hands the core this capacity with this block.
|
|
244
|
+
def grow_cells(per_row, batch_rows)
|
|
245
|
+
capacity = per_row * Delimited.cell_rows(per_row, batch_rows)
|
|
246
|
+
@cells = Runtime.buffer(SPAN_BYTES * capacity)
|
|
247
|
+
@cells_cap = capacity
|
|
248
|
+
@per_row = per_row
|
|
249
|
+
end
|
|
250
|
+
|
|
226
251
|
def grow_arena(needed)
|
|
227
|
-
|
|
228
|
-
@arena = Runtime.buffer(
|
|
252
|
+
capacity = [needed, @arena_cap * 2].max
|
|
253
|
+
@arena = Runtime.buffer(capacity)
|
|
254
|
+
@arena_cap = capacity
|
|
229
255
|
end
|
|
230
256
|
end
|
|
231
257
|
end
|
|
@@ -5,8 +5,9 @@ module HyperTabular
|
|
|
5
5
|
# which never see a pointer.
|
|
6
6
|
#
|
|
7
7
|
# A workbook call works in buffers the caller hands it (a Buffers block of seven pointer
|
|
8
|
-
# and size pairs
|
|
9
|
-
#
|
|
8
|
+
# and size pairs, or a BuffersWith with tokens and row widths for a fill), and when one is
|
|
9
|
+
# too small it says which and how large, having undone nothing: the buffer is grown with
|
|
10
|
+
# what it held kept, and the same call made again.
|
|
10
11
|
module Book
|
|
11
12
|
class << self
|
|
12
13
|
# For the specs: every buffer a workbook call works in starts with room for one
|
|
@@ -78,6 +79,18 @@ module HyperTabular
|
|
|
78
79
|
@buffers
|
|
79
80
|
end
|
|
80
81
|
|
|
82
|
+
# As #buffers, for a fill: with the sheet's +tokens+ (a Pointer, or nil) and their length, and
|
|
83
|
+
# the address the row widths go, or 0. Packed again for every call, since any of them may
|
|
84
|
+
# have been grown since the last.
|
|
85
|
+
def buffers_with(tables, tokens, tokens_len, widths)
|
|
86
|
+
@buffers_with ||= Runtime.buffer(Columns::BUFFERS_WITH_BYTES)
|
|
87
|
+
@buffers_with[0, Columns::BUFFERS_WITH_BYTES] =
|
|
88
|
+
Columns.word(Columns::BUFFERS_WITH_BYTES) +
|
|
89
|
+
[@window.to_i, @window_cap, @arena.to_i, @arena_cap, @cells.to_i, @cells_cap, @row.to_i, @row_cap,
|
|
90
|
+
*tables.to_a, tokens.to_i, tokens_len, widths].pack("J*")
|
|
91
|
+
@buffers_with
|
|
92
|
+
end
|
|
93
|
+
|
|
81
94
|
# Makes the window at least +needed+ bytes, what it held kept.
|
|
82
95
|
def grow_window(needed)
|
|
83
96
|
@window, @window_cap = grown(@window, @window_cap, needed, 1)
|
|
@@ -108,7 +121,8 @@ module HyperTabular
|
|
|
108
121
|
|
|
109
122
|
# Makes +call+ (a block taking the Buffers block and the out block) until it stops
|
|
110
123
|
# asking for room, growing the buffer it names each time. Returns the code it ended
|
|
111
|
-
# on and the Filled fields: rows, consumed, arena_used, needed, then the failure.
|
|
124
|
+
# on and the Filled fields: rows, consumed, arena_used, needed, then the failure. A fill's
|
|
125
|
+
# block makes its own BuffersWith, from the buffers as they are.
|
|
112
126
|
def drive(out, tables = nil)
|
|
113
127
|
loop do
|
|
114
128
|
code = yield buffers(tables)
|
|
@@ -234,28 +248,37 @@ module HyperTabular
|
|
|
234
248
|
# and one for the row's number. Nil until #bind.
|
|
235
249
|
attr_reader :columns, :per_row
|
|
236
250
|
|
|
237
|
-
# Positions a copy of the workbook's state on +sheet+ (an entry of Opened#sheets)
|
|
238
|
-
|
|
251
|
+
# Positions a copy of the workbook's state on +sheet+ (an entry of Opened#sheets), as
|
|
252
|
+
# +options+, the bytes of a RawSheetOptions, say. +tokens+ is the sheet's tokens as the core
|
|
253
|
+
# takes them, or nil.
|
|
254
|
+
def initialize(book, sheet, options, batch_rows, tokens = nil)
|
|
239
255
|
@book = book
|
|
240
256
|
@state = book.copy_of_state
|
|
241
257
|
@batch_rows = batch_rows
|
|
242
258
|
@scratch = Book.stingy ? Scratch.new(1, 1, 1, 0) : Scratch.new(0, 4096, 64, 0)
|
|
243
259
|
@out = Runtime.buffer(FILLED_BYTES)
|
|
260
|
+
unless tokens.nil?
|
|
261
|
+
@tokens_len = tokens.bytesize
|
|
262
|
+
@tokens = Runtime.buffer(@tokens_len)
|
|
263
|
+
@tokens[0, @tokens_len] = tokens
|
|
264
|
+
end
|
|
244
265
|
_name, _hidden, part, index = sheet
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
266
|
+
raw = Runtime.buffer(options.bytesize)
|
|
267
|
+
raw[0, options.bytesize] = options
|
|
268
|
+
code = Runtime.function(:hypertabular_workbook_sheet_with).call(
|
|
269
|
+
@state, book.container, book.length, Runtime.pin(part), part.bytesize, index, raw, @out
|
|
248
270
|
)
|
|
249
271
|
Book.settle(code, @out[0, FILLED_BYTES].unpack(FILLED))
|
|
250
272
|
end
|
|
251
273
|
|
|
252
|
-
# Takes the plan: +specs+ is one
|
|
253
|
-
# one value of each takes,
|
|
254
|
-
# column the plan reads
|
|
274
|
+
# Takes the plan: +specs+ is one Column.specs entry per plan column, +sizes+ the bytes
|
|
275
|
+
# one value of each takes, +width+ the slots a row needs to reach every source
|
|
276
|
+
# column the plan reads, +notes+ whether each column keeps its notes and +widths+ whether
|
|
277
|
+
# each row's width is kept. Once, before the first #fill. The core keeps a header row's
|
|
255
278
|
# cells in the row's slots — a row the sheet repeats (an ODS number-rows-repeated) is
|
|
256
279
|
# delivered again from them — so the slots are only ever grown here, never replaced.
|
|
257
|
-
def bind(specs, sizes, width)
|
|
258
|
-
@columns = Columns.new(specs, sizes, @batch_rows)
|
|
280
|
+
def bind(specs, sizes, width, notes = true, widths = false)
|
|
281
|
+
@columns = Columns.new(specs, sizes, @batch_rows, notes, widths)
|
|
259
282
|
@per_row = specs.size + 1
|
|
260
283
|
# A stingy scratch is left to grow into the cell table the hard way, mid-read.
|
|
261
284
|
@scratch.reserve_cells(@batch_rows * @per_row) unless Book.stingy
|
|
@@ -267,8 +290,9 @@ module HyperTabular
|
|
|
267
290
|
# record, byte, expected, found] — that came with them, if one did: the caller raises
|
|
268
291
|
# it after them. The table and the arena are copies.
|
|
269
292
|
def fill
|
|
270
|
-
call = Runtime.function(:
|
|
271
|
-
code, filled = @scratch.drive(@out, @book.tables) do
|
|
293
|
+
call = Runtime.function(:hypertabular_workbook_fill_with)
|
|
294
|
+
code, filled = @scratch.drive(@out, @book.tables) do
|
|
295
|
+
buffers = @scratch.buffers_with(@book.tables, @tokens, @tokens_len.to_i, @columns.widths_address)
|
|
272
296
|
call.call(@state, @book.container, @book.length, @columns.specs, @columns.table,
|
|
273
297
|
@columns.count, @columns.batch_rows, buffers, @out)
|
|
274
298
|
end
|
data/lib/hypertabular/runtime.rb
CHANGED
|
@@ -16,16 +16,30 @@ module HyperTabular
|
|
|
16
16
|
EXPORTS = {
|
|
17
17
|
hypertabular_version: [[], :uint32],
|
|
18
18
|
hypertabular_delimited_state_size: [[], :size],
|
|
19
|
-
# (state, dialect)
|
|
19
|
+
# (state, dialect): 1.0's, which no read takes; a spec holds it to what it always read.
|
|
20
20
|
hypertabular_delimited_init: [%i[pointer pointer], :int32],
|
|
21
|
+
# (state, options), the RawDelimitedOptions every read starts from.
|
|
22
|
+
hypertabular_delimited_init_with: [%i[pointer pointer], :int32],
|
|
23
|
+
# (encoding, input, input_len, last, out, cap, progress)
|
|
24
|
+
hypertabular_transcode: [%i[uint32 pointer size uint32 pointer size pointer], :int32],
|
|
21
25
|
# (state, input, input_len, last, names, names_cap, arena, arena_cap, out)
|
|
22
26
|
hypertabular_delimited_header:
|
|
23
27
|
[%i[pointer pointer size uint32 pointer size pointer size pointer], :int32],
|
|
24
|
-
# (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out)
|
|
25
|
-
#
|
|
26
|
-
# only the arena and the cell table.
|
|
28
|
+
# (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out):
|
|
29
|
+
# 1.0's, which no read takes, reading every format Western; a spec holds it to that.
|
|
27
30
|
hypertabular_delimited_fill:
|
|
28
31
|
[%i[pointer pointer size uint32 pointer pointer size size pointer pointer], :int32],
|
|
32
|
+
# (state, input, input_len, last, specs, columns, column_count, max_rows, buffers, out) —
|
|
33
|
+
# buffers is the BuffersWith block, of which a delimited fill reads the arena, the cell
|
|
34
|
+
# table, the tokens and the row widths.
|
|
35
|
+
hypertabular_delimited_fill_with:
|
|
36
|
+
[%i[pointer pointer size uint32 pointer pointer size size pointer pointer], :int32],
|
|
37
|
+
# (columns, column_count, rows, row_widths, expected_width, long_rows, cursor, out, cap, written)
|
|
38
|
+
hypertabular_faults:
|
|
39
|
+
[%i[pointer size size pointer uint32 uint32 pointer pointer size pointer], :int32],
|
|
40
|
+
# (input, input_len, arena, arena_len, cells, typed, rows, columns, formats, out)
|
|
41
|
+
hypertabular_suggest:
|
|
42
|
+
[%i[pointer size pointer size pointer pointer size size pointer pointer], :int32],
|
|
29
43
|
# (cell, len, out, cap)
|
|
30
44
|
hypertabular_delimited_unescape: [%i[pointer size pointer size], :size],
|
|
31
45
|
hypertabular_workbook_state_size: [[], :size],
|
|
@@ -34,17 +48,180 @@ module HyperTabular
|
|
|
34
48
|
hypertabular_workbook_sheets: [%i[pointer pointer size pointer pointer], :int32],
|
|
35
49
|
hypertabular_workbook_strings: [%i[pointer pointer size pointer pointer], :int32],
|
|
36
50
|
hypertabular_workbook_styles: [%i[pointer pointer size pointer pointer], :int32],
|
|
37
|
-
# (state, container, container_len, part, part_len, index, has_header, skip_empty_rows, out)
|
|
51
|
+
# (state, container, container_len, part, part_len, index, has_header, skip_empty_rows, out):
|
|
52
|
+
# 1.0's, which no read takes.
|
|
38
53
|
hypertabular_workbook_sheet:
|
|
39
54
|
[%i[pointer pointer size pointer size uint32 uint32 uint32 pointer], :int32],
|
|
55
|
+
# (state, container, container_len, part, part_len, index, options, out)
|
|
56
|
+
hypertabular_workbook_sheet_with:
|
|
57
|
+
[%i[pointer pointer size pointer size uint32 pointer pointer], :int32],
|
|
40
58
|
hypertabular_workbook_header: [%i[pointer pointer size pointer pointer], :int32],
|
|
41
|
-
# (state, container, container_len, specs, columns, column_count, max_rows, buffers, out)
|
|
42
|
-
|
|
59
|
+
# (state, container, container_len, specs, columns, column_count, max_rows, buffers, out):
|
|
60
|
+
# 1.0's, which no read takes.
|
|
61
|
+
hypertabular_workbook_fill: [%i[pointer pointer size pointer pointer size size pointer pointer], :int32],
|
|
62
|
+
# (state, container, container_len, specs, columns, column_count, max_rows, buffers, out) —
|
|
63
|
+
# buffers is the BuffersWith block.
|
|
64
|
+
hypertabular_workbook_fill_with:
|
|
65
|
+
[%i[pointer pointer size pointer pointer size size pointer pointer], :int32],
|
|
66
|
+
# (text, len, out)
|
|
67
|
+
hypertabular_range_parse: [%i[pointer size pointer], :int32],
|
|
68
|
+
hypertabular_batch_check: [%i[pointer size uint64 uint64 uint32 pointer], :int32]
|
|
43
69
|
}.freeze
|
|
44
70
|
|
|
45
71
|
@mutex = Mutex.new
|
|
46
72
|
@functions = nil
|
|
47
73
|
|
|
74
|
+
# The entry points the Magnus extension also defines, the Fiddle way. They live in a module
|
|
75
|
+
# Runtime extends rather than on its singleton class, so the extension's
|
|
76
|
+
# define_singleton_method puts its definition in front of each: an override through
|
|
77
|
+
# the ancestor chain, not a redefinition, and `ruby -w` has nothing to say about it (1.0.0
|
|
78
|
+
# printed "method redefined" for it, and for Delimited.start and .version, on every
|
|
79
|
+
# require under -w). With the extension loaded no call ever reaches it.
|
|
80
|
+
module FiddleBackend
|
|
81
|
+
# A quoted delimited cell — the bytes a flagged cell-table entry names — with its
|
|
82
|
+
# quotes resolved, as the core cast it.
|
|
83
|
+
def unescape(quoted)
|
|
84
|
+
input = pin(quoted)
|
|
85
|
+
out = buffer([quoted.bytesize, 1].max)
|
|
86
|
+
written = function(:hypertabular_delimited_unescape).call(input, quoted.bytesize, out, quoted.bytesize)
|
|
87
|
+
out[0, written]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The index of the first of +specs+ (Column.specs' form) the core refuses, or nil. A fill of
|
|
91
|
+
# no rows judges a column's door, what it declares, its format, date options, vocabulary and
|
|
92
|
+
# tokens and constraints, and writes nothing. It answers on a state of its own, so a sheet's
|
|
93
|
+
# plan is asked the same way.
|
|
94
|
+
def refused(specs)
|
|
95
|
+
state = buffer(function(:hypertabular_delimited_state_size).call)
|
|
96
|
+
options = buffer(Dialect::OPTIONS_BYTES)
|
|
97
|
+
options[0, Dialect::OPTIONS_BYTES] = Dialect::CSV.packed_with
|
|
98
|
+
raise Delimited::CONTRACT unless
|
|
99
|
+
function(:hypertabular_delimited_init_with).call(state, options) == Delimited::OK
|
|
100
|
+
|
|
101
|
+
value = buffer(32)
|
|
102
|
+
verdict = buffer(Columns::VERDICT_BYTES)
|
|
103
|
+
# A constrained column without checks is refused, so each is asked with them.
|
|
104
|
+
check = buffer(Columns::CHECK_BYTES)
|
|
105
|
+
table = buffer(Columns::COLUMN_BUFFER_BYTES)
|
|
106
|
+
table[0, Columns::COLUMN_BUFFER_BYTES] = Columns.column_buffer(value, verdict, nil, check)
|
|
107
|
+
memory = buffer(Columns::BUFFERS_WITH_BYTES)
|
|
108
|
+
memory[0, Columns::WORD] = Columns.word(Columns::BUFFERS_WITH_BYTES)
|
|
109
|
+
out = buffer(Delimited::FILLED_BYTES)
|
|
110
|
+
fill = function(:hypertabular_delimited_fill_with)
|
|
111
|
+
specs.index do |spec|
|
|
112
|
+
lists = []
|
|
113
|
+
bytes = Columns.spec_with(spec, lists)
|
|
114
|
+
held = buffer(bytes.bytesize)
|
|
115
|
+
held[0, bytes.bytesize] = bytes
|
|
116
|
+
fill.call(state, nil, 0, 0, held, table, 1, 0, memory, out) == Delimited::ERR_CONTRACT
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# The next step of a declared encoding's decode: +length+ bytes of +input+ from +start+, with
|
|
121
|
+
# +last+ saying nothing follows them, into at most +capacity+ bytes of UTF-8. Answers
|
|
122
|
+
# [consumed, written, failed, failure_at, out], +out+ a binary String of its own.
|
|
123
|
+
def transcode(encoding, input, start, length, last, capacity)
|
|
124
|
+
out = buffer(capacity)
|
|
125
|
+
progress = buffer(TRANSCODED_BYTES)
|
|
126
|
+
base = pin(input)
|
|
127
|
+
raise Delimited::CONTRACT unless
|
|
128
|
+
function(:hypertabular_transcode).call(encoding, base.to_i + start, length, last ? 1 : 0, out, capacity,
|
|
129
|
+
progress) == Delimited::OK
|
|
130
|
+
|
|
131
|
+
consumed, written, failure_at, failed = progress[0, TRANSCODED_BYTES].unpack("Q<3L<")
|
|
132
|
+
[consumed, written, failed, failure_at, out[0, written]]
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Every RawProblem of a batch, as one binary String: +verdicts+ is each column's array as the
|
|
136
|
+
# core wrote it, +notes+ the same or nil, +checks+ each column's RawCheck array or nil (and
|
|
137
|
+
# nil for every column of a plan that declares no constraints), and +widths+ the row widths
|
|
138
|
+
# or nil.
|
|
139
|
+
def faults(verdicts, notes, checks, rows, widths, expected, long_rows)
|
|
140
|
+
pinned = verdicts.map { |column| pin(column) }
|
|
141
|
+
noted = notes&.map { |column| pin(column) }
|
|
142
|
+
checked = checks&.map { |column| column && pin(column) }
|
|
143
|
+
table = buffer([Columns::COLUMN_BUFFER_BYTES * verdicts.size, 1].max)
|
|
144
|
+
verdicts.each_index do |column|
|
|
145
|
+
table[Columns::COLUMN_BUFFER_BYTES * column, Columns::COLUMN_BUFFER_BYTES] =
|
|
146
|
+
Columns.column_buffer(nil, pinned[column], noted && noted[column], checked && checked[column])
|
|
147
|
+
end
|
|
148
|
+
held = widths && pin(widths)
|
|
149
|
+
cursor = buffer(8)
|
|
150
|
+
written = buffer(Columns::WORD)
|
|
151
|
+
out = buffer(FAULTS_BYTES)
|
|
152
|
+
call = function(:hypertabular_faults)
|
|
153
|
+
listed = "".b
|
|
154
|
+
loop do
|
|
155
|
+
code = call.call(table, verdicts.size, rows, held || 0, expected, long_rows, cursor, out,
|
|
156
|
+
FAULTS_BYTES / PROBLEM_BYTES, written)
|
|
157
|
+
raise Delimited::CONTRACT unless code == Delimited::OK
|
|
158
|
+
|
|
159
|
+
entries = written[0, Columns::WORD].unpack1("J")
|
|
160
|
+
listed << out[0, PROBLEM_BYTES * entries]
|
|
161
|
+
return listed if entries < FAULTS_BYTES / PROBLEM_BYTES
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# What the core makes of a sample: door, param, status, matched and sampled for each of
|
|
166
|
+
# +columns+, flat, or nil if it refuses. +input+ is the sample's bytes, +cells+ its spans a
|
|
167
|
+
# column after another, +typed+ a byte for each or nil, +formats+ a packed format for each.
|
|
168
|
+
def suggest(input, cells, typed, rows, columns, formats)
|
|
169
|
+
out = buffer([SUGGESTION_BYTES * columns, 1].max)
|
|
170
|
+
pins = [input, cells, typed, formats].map { |text| text && pin(text) }
|
|
171
|
+
code = function(:hypertabular_suggest).call(pins[0], input.bytesize, nil, 0, pins[1], pins[2] || 0, rows,
|
|
172
|
+
columns, pins[3], out)
|
|
173
|
+
code == Delimited::OK ? out[0, SUGGESTION_BYTES * columns].unpack("L<*") : nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The core's hypertabular_batch_check: [code, needed, limit, buffer_bytes, refused,
|
|
177
|
+
# sample_rows, width]. The Magnus extension defines its own, as it does range_parse.
|
|
178
|
+
def batch_check(specs, batch_rows, buffer_bytes, flags)
|
|
179
|
+
lists = []
|
|
180
|
+
packed = specs.map { |spec| Columns.spec_with(spec, lists) }.join
|
|
181
|
+
held = buffer(packed.bytesize)
|
|
182
|
+
held[0, packed.bytesize] = packed unless packed.empty?
|
|
183
|
+
out = buffer(BATCH_CHECK_BYTES)
|
|
184
|
+
code = function(:hypertabular_batch_check).call(held, specs.size, batch_rows, buffer_bytes, flags, out)
|
|
185
|
+
needed, limit, start, sample_rows, width, refused = out[0, BATCH_CHECK_BYTES].unpack("Q<5L<")
|
|
186
|
+
[code, needed, limit, start, refused, sample_rows, width]
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# A range's four corners as the core reads them from +text+, or nil if it isn't one.
|
|
190
|
+
def range_parse(text)
|
|
191
|
+
out = buffer(RANGE_BYTES)
|
|
192
|
+
return nil unless function(:hypertabular_range_parse).call(pin(text), text.bytesize, out) == Delimited::OK
|
|
193
|
+
|
|
194
|
+
out[0, RANGE_BYTES].unpack("L<4")
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Flags to hypertabular_batch_check (RawBatchCheck in the core).
|
|
199
|
+
BATCH_NOTES = 1
|
|
200
|
+
BATCH_WIDTHS = 2
|
|
201
|
+
BATCH_SHEET = 4
|
|
202
|
+
# RawBatchCheck's refused codes.
|
|
203
|
+
BATCH_ROWS_ZERO = 1
|
|
204
|
+
BUFFER_BYTES_ZERO = 2
|
|
205
|
+
ROW_TOO_WIDE = 4
|
|
206
|
+
# Bytes in one RawBatchCheck.
|
|
207
|
+
BATCH_CHECK_BYTES = 48
|
|
208
|
+
# Bytes in one RawTranscoded.
|
|
209
|
+
TRANSCODED_BYTES = 32
|
|
210
|
+
# Bytes in one RawProblem.
|
|
211
|
+
PROBLEM_BYTES = 36
|
|
212
|
+
# The room one hypertabular_faults call lists into.
|
|
213
|
+
FAULTS_BYTES = PROBLEM_BYTES * 256
|
|
214
|
+
# Bytes in one RawSuggestion.
|
|
215
|
+
SUGGESTION_BYTES = 20
|
|
216
|
+
# Bytes in one RawRange.
|
|
217
|
+
RANGE_BYTES = 16
|
|
218
|
+
# The allocator behind every buffer, which has to be the one RUBY_FREE frees: Ruby's own in
|
|
219
|
+
# CRuby, and libc's in Fiddle's FFI backend, the one every other Ruby takes, whose RUBY_FREE is
|
|
220
|
+
# libc's free.
|
|
221
|
+
ZEROING = RUBY_ENGINE == "ruby" ? "ruby_xcalloc" : "calloc"
|
|
222
|
+
|
|
223
|
+
extend FiddleBackend
|
|
224
|
+
|
|
48
225
|
class << self
|
|
49
226
|
# The export's Fiddle::Function, for the caller to invoke directly.
|
|
50
227
|
def function(symbol)
|
|
@@ -54,9 +231,63 @@ module HyperTabular
|
|
|
54
231
|
# Every native allocation goes through here, and loads the library first: that is what
|
|
55
232
|
# keeps a missing library reported as missing_library_message instead of as whatever
|
|
56
233
|
# touching Fiddle raises first. Zeroed, freed with the object that holds it.
|
|
234
|
+
#
|
|
235
|
+
# Many of these are sized by a caller, so nothing here may touch the memory: pages nobody
|
|
236
|
+
# reads or writes stay the OS's. Fiddle::Pointer.malloc writes every byte. ruby_xcalloc asks
|
|
237
|
+
# calloc, counts the bytes towards the next collection as Pointer.malloc does, pairs with
|
|
238
|
+
# the RUBY_FREE it has always been freed with, and raises NoMemoryError for what it can't get.
|
|
239
|
+
#
|
|
240
|
+
# Fiddle::Pointer takes its size as a C long, 32 bits on 64-bit Windows. A size past that is
|
|
241
|
+
# refused here, before the allocation it would otherwise strand.
|
|
57
242
|
def buffer(size)
|
|
58
243
|
functions
|
|
59
|
-
|
|
244
|
+
raise NoMemoryError, "failed to allocate memory" if size >= 1 << ((Fiddle::SIZEOF_LONG * 8) - 1)
|
|
245
|
+
|
|
246
|
+
# A zero-sized request still answers an address of its own. TruffleRuby and JRuby's
|
|
247
|
+
# allocator is calloc, which answers null where ruby_xcalloc raises.
|
|
248
|
+
address = zeroing.call(1, [size, 1].max).to_i
|
|
249
|
+
raise NoMemoryError, "failed to allocate memory" if address.zero?
|
|
250
|
+
|
|
251
|
+
Fiddle::Pointer.new(address, size, Fiddle::RUBY_FREE)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# Asks the core whether a read may ask for this much, before anything is allocated for it:
|
|
255
|
+
# the size to start the input buffer at, or an ArgumentError in the core's terms. +specs+ is
|
|
256
|
+
# Column.specs' answer, [] where no plan is known yet, and +flags+ is batch_flags' answer. What
|
|
257
|
+
# is here is only what a count has to be for the core to be asked at all.
|
|
258
|
+
def check_sizes(specs, batch_rows, buffer_bytes, flags)
|
|
259
|
+
counted("batch_rows", batch_rows)
|
|
260
|
+
counted("buffer_bytes", buffer_bytes)
|
|
261
|
+
code, needed, limit, start, refused, _, width = batch_check(specs, batch_rows, buffer_bytes, flags)
|
|
262
|
+
return start if code == Delimited::OK
|
|
263
|
+
raise Delimited::CONTRACT unless code == Delimited::ERR_BATCH
|
|
264
|
+
|
|
265
|
+
raise ArgumentError, case refused
|
|
266
|
+
when BATCH_ROWS_ZERO then "batch_rows must be at least 1"
|
|
267
|
+
when BUFFER_BYTES_ZERO then "buffer_bytes must be at least 1"
|
|
268
|
+
when ROW_TOO_WIDE
|
|
269
|
+
"a row of #{width} columns asks for #{needed} bytes, past the limit of #{limit}"
|
|
270
|
+
else "batch_rows asks for #{needed} bytes, past the limit of #{limit}"
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# The rows of a sampler's batch for a sampler wanting +wanted+ rows: the core's answer, however
|
|
275
|
+
# many batches that takes. Only a row past the limit is refused, in its words.
|
|
276
|
+
def sample_rows(specs, wanted, flags)
|
|
277
|
+
_, _, _, _, _, rows, = batch_check(specs, wanted, 1, flags)
|
|
278
|
+
check_sizes(specs, wanted, 1, flags) if rows.zero?
|
|
279
|
+
rows
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# The flags batch_check takes, for a read that keeps +notes+ and +widths+.
|
|
283
|
+
def batch_flags(notes, widths, sheet: false)
|
|
284
|
+
(notes ? BATCH_NOTES : 0) | (widths ? BATCH_WIDTHS : 0) | (sheet ? BATCH_SHEET : 0)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
private def counted(name, count)
|
|
288
|
+
raise ArgumentError, "#{name} must be an Integer; got #{count.inspect}" unless count.is_a?(Integer)
|
|
289
|
+
raise ArgumentError, "#{name} must not be negative; got #{count}" if count.negative?
|
|
290
|
+
raise ArgumentError, "#{name} must fit in 64 bits; got #{count}" if count >= 1 << 64
|
|
60
291
|
end
|
|
61
292
|
|
|
62
293
|
# A pointer to a String's own bytes — nothing is copied. While the returned object is
|
|
@@ -68,16 +299,22 @@ module HyperTabular
|
|
|
68
299
|
Fiddle::Pointer[string]
|
|
69
300
|
end
|
|
70
301
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
302
|
+
private
|
|
303
|
+
|
|
304
|
+
# The allocator, resolved beside the exports, so a Ruby that doesn't have it is a LoadError
|
|
305
|
+
# that available? answers for, not a failure at the first read. ruby_xcalloc needs the lock
|
|
306
|
+
# Fiddle would otherwise let go of around a call.
|
|
307
|
+
def zeroing
|
|
308
|
+
functions
|
|
309
|
+
@zeroing
|
|
78
310
|
end
|
|
79
311
|
|
|
80
|
-
|
|
312
|
+
def load_zeroing
|
|
313
|
+
Fiddle::Function.new(Fiddle.dlopen(nil)[ZEROING], [Fiddle::TYPE_SIZE_T, Fiddle::TYPE_SIZE_T],
|
|
314
|
+
Fiddle::TYPE_VOIDP, need_gvl: true)
|
|
315
|
+
rescue Fiddle::DLError => e
|
|
316
|
+
raise LoadError, "HyperTabular's Fiddle backend can't find #{ZEROING} in this Ruby: #{e.message}"
|
|
317
|
+
end
|
|
81
318
|
|
|
82
319
|
# The shared library to dlopen: this install's native/{rid}/{lib}, or — the
|
|
83
320
|
# development loop — the in-repo cargo build, exactly what the other bindings' local
|
|
@@ -127,9 +364,10 @@ module HyperTabular
|
|
|
127
364
|
raise LoadError, missing_library_message if path.nil?
|
|
128
365
|
|
|
129
366
|
handle = Fiddle.dlopen(path)
|
|
367
|
+
@zeroing = load_zeroing
|
|
130
368
|
types = {
|
|
131
369
|
pointer: Fiddle::TYPE_VOIDP, size: Fiddle::TYPE_SIZE_T,
|
|
132
|
-
uint32: Fiddle::TYPE_UINT32_T, int32: Fiddle::TYPE_INT32_T
|
|
370
|
+
uint32: Fiddle::TYPE_UINT32_T, int32: Fiddle::TYPE_INT32_T, uint64: Fiddle::TYPE_UINT64_T
|
|
133
371
|
}
|
|
134
372
|
EXPORTS.to_h do |name, (arguments, result)|
|
|
135
373
|
[name, Fiddle::Function.new(handle[name.to_s], arguments.map { |type|
|