hypertabular-wasm 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE-UNICODE +39 -0
- data/README.md +278 -34
- data/ext/hypertabular_native/3.4/hypertabular_native.a.gz +0 -0
- data/ext/hypertabular_native/4.0/hypertabular_native.a.gz +0 -0
- data/lib/hypertabular/batch.rb +219 -24
- data/lib/hypertabular/column.rb +166 -21
- data/lib/hypertabular/delimited_reader.rb +319 -94
- data/lib/hypertabular/dialect.rb +92 -10
- data/lib/hypertabular/guarded.rb +116 -0
- data/lib/hypertabular/header.rb +59 -22
- data/lib/hypertabular/row.rb +17 -4
- data/lib/hypertabular/runtime/columns.rb +78 -12
- data/lib/hypertabular/runtime/delimited.rb +63 -37
- data/lib/hypertabular/runtime/workbook.rb +39 -15
- data/lib/hypertabular/runtime.rb +255 -17
- data/lib/hypertabular/suggest.rb +283 -0
- data/lib/hypertabular/tabular_error.rb +9 -3
- data/lib/hypertabular/workbook.rb +185 -45
- data/lib/hypertabular.rb +14 -8
- metadata +7 -3
|
@@ -39,8 +39,18 @@ module HyperTabular
|
|
|
39
39
|
#
|
|
40
40
|
# Each #read returns a Batch, which owns what it shows: it stays good after the next
|
|
41
41
|
# #read. Not thread-safe.
|
|
42
|
+
#
|
|
43
|
+
# A reader is called into once at a time: a call back into it from the IO it reads or from a
|
|
44
|
+
# plan's #each, while it is reading or binding, raises RuntimeError naming it busy, and the call
|
|
45
|
+
# it came from goes on as it was. An exception that lands in a read (Timeout.timeout,
|
|
46
|
+
# Thread#raise, Ctrl-C's Interrupt) waits until the core's step and the reader's own state agree,
|
|
47
|
+
# and a batch it lands on is handed over by the next #read, numbered the same. Only an IO read
|
|
48
|
+
# that blocks is cut part-way, and since Ruby may drop what that read had taken from the IO, every
|
|
49
|
+
# later read then raises RuntimeError naming the exception. Once the IO has answered, the
|
|
50
|
+
# exception waits until its answer is held.
|
|
42
51
|
class DelimitedReader
|
|
43
52
|
include Enumerable
|
|
53
|
+
include Guarded
|
|
44
54
|
|
|
45
55
|
# The row ceiling: a single record larger than this is a :row_too_long TabularError.
|
|
46
56
|
MAX_ROW_BYTES = 1 << 30
|
|
@@ -54,14 +64,17 @@ module HyperTabular
|
|
|
54
64
|
# Encodings whose bytes already are the UTF-8 (or byte-identical) form the core reads.
|
|
55
65
|
BYTE_COMPATIBLE = HyperCast::Interop::BYTE_COMPATIBLE
|
|
56
66
|
|
|
57
|
-
# Opens a file of
|
|
58
|
-
# for #bind. With a block, yields the reader, closes it — and the file — when
|
|
59
|
-
# ends, and returns the block's value; without one, returns the reader, whose
|
|
60
|
-
# closes the file.
|
|
61
|
-
def self.open(path, dialect, plan = nil, batch_rows: DEFAULT_BATCH_ROWS, buffer_bytes: DEFAULT_BUFFER_BYTES
|
|
67
|
+
# Opens a file of delimited text, in the dialect's encoding, through +plan+ — or, with none,
|
|
68
|
+
# header first, for #bind. With a block, yields the reader, closes it — and the file — when
|
|
69
|
+
# the block ends, and returns the block's value; without one, returns the reader, whose
|
|
70
|
+
# #close closes the file.
|
|
71
|
+
def self.open(path, dialect, plan = nil, batch_rows: DEFAULT_BATCH_ROWS, buffer_bytes: DEFAULT_BUFFER_BYTES,
|
|
72
|
+
notes: true)
|
|
73
|
+
judged(dialect, plan, batch_rows, buffer_bytes, notes)
|
|
62
74
|
file = File.open(path, "rb")
|
|
63
75
|
begin
|
|
64
|
-
reader = new(file, dialect, plan, batch_rows: batch_rows, buffer_bytes: buffer_bytes, close_source: true
|
|
76
|
+
reader = new(file, dialect, plan, batch_rows: batch_rows, buffer_bytes: buffer_bytes, close_source: true,
|
|
77
|
+
notes: notes)
|
|
65
78
|
ensure
|
|
66
79
|
# The file is ours to close when no reader came of it, whatever went wrong.
|
|
67
80
|
file.close if reader.nil?
|
|
@@ -75,43 +88,81 @@ module HyperTabular
|
|
|
75
88
|
end
|
|
76
89
|
end
|
|
77
90
|
|
|
78
|
-
#
|
|
79
|
-
#
|
|
80
|
-
|
|
81
|
-
|
|
91
|
+
# The core's answer on these sizes, ahead of touching a file. A plan that can't be honoured says
|
|
92
|
+
# so where it always has, so its size is judged without it.
|
|
93
|
+
private_class_method def self.judged(dialect, plan, batch_rows, buffer_bytes, notes)
|
|
94
|
+
raise ArgumentError, "dialect must be a HyperTabular::Dialect; got #{dialect.inspect}" unless
|
|
95
|
+
dialect.is_a?(Dialect)
|
|
96
|
+
|
|
97
|
+
specs = begin
|
|
98
|
+
plan.nil? ? [] : Column.specs(Column.plan(plan))
|
|
99
|
+
rescue ArgumentError
|
|
100
|
+
[]
|
|
101
|
+
end
|
|
102
|
+
Runtime.check_sizes(specs, batch_rows, buffer_bytes, Runtime.batch_flags(notes, dialect.long_rows == :report))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Offers a plan for the delimited file at +path+ from a sample, and applies nothing: a
|
|
106
|
+
# Suggestion to inspect, change and hand to a read. Declaring the locale (a format and a date
|
|
107
|
+
# order in +options+, a SuggestOptions) settles what a sample can't: `1.234` alone can't say
|
|
108
|
+
# which it is. A format the core would refuse is an ArgumentError before the file is opened,
|
|
109
|
+
# and a structural failure among the sampled rows a TabularError.
|
|
110
|
+
def self.suggest(path, dialect, options = SuggestOptions::DEFAULT)
|
|
111
|
+
SuggestOptions.judged(options)
|
|
112
|
+
Sample.delimited(dialect, options) { |rows| self.open(path, dialect, batch_rows: rows, notes: false) }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# DelimitedReader.suggest, over a String.
|
|
116
|
+
def self.suggest_string(text, dialect, options = SuggestOptions::DEFAULT)
|
|
117
|
+
SuggestOptions.judged(options)
|
|
118
|
+
Sample.delimited(dialect, options) { |rows| new(text, dialect, batch_rows: rows, notes: false) }
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Reads delimited text from +source+: a String, read in place when it is UTF-8 — nothing
|
|
122
|
+
# is copied — or an IO (anything with #readpartial or #read), read forward only through a
|
|
123
|
+
# buffer of +buffer_bytes+ that grows when a record does not fit it, one read of the IO each
|
|
124
|
+
# time the core wants more, so finished rows reach the caller while a stream goes quiet. An
|
|
125
|
+
# IO is read as bytes; one opened in text mode on Windows should be in binmode first. A
|
|
126
|
+
# Dialect that declares an encoding has its bytes decoded ahead of the read, which a String
|
|
127
|
+
# is copied for. An exception the IO raises goes to the caller and is final: every later
|
|
128
|
+
# #read raises RuntimeError naming it.
|
|
82
129
|
#
|
|
83
130
|
# +dialect+ is the declared Dialect and +plan+ the output Columns, in output order — or
|
|
84
131
|
# nil, for a reader that reads its header first and is given its plan by #bind.
|
|
85
132
|
# +batch_rows+ is the most rows one #read delivers. +close_source+ says whether #close
|
|
86
|
-
# also closes the IO.
|
|
133
|
+
# also closes the IO. +notes+, on by default, is whether each cell's detail and origin are
|
|
134
|
+
# kept: off, a Fault has no detail, Batch#origin says :cell, and Batch#faults lists no uncached
|
|
135
|
+
# formula, for a read a little faster on a file full of bad cells.
|
|
87
136
|
#
|
|
88
137
|
# A dialect the scanner cannot honour, or a plan that is not Columns, is an
|
|
89
138
|
# ArgumentError. With a header declared the header record is read here, so a header
|
|
90
139
|
# that is structurally broken is a TabularError here.
|
|
91
140
|
def initialize(source, dialect, plan = nil, batch_rows: DEFAULT_BATCH_ROWS, buffer_bytes: DEFAULT_BUFFER_BYTES,
|
|
92
|
-
close_source: false)
|
|
141
|
+
close_source: false, notes: true)
|
|
93
142
|
raise ArgumentError, "dialect must be a HyperTabular::Dialect; got #{dialect.inspect}" unless
|
|
94
143
|
dialect.is_a?(Dialect)
|
|
95
|
-
raise ArgumentError, "batch_rows must be a positive Integer; got #{batch_rows.inspect}" unless
|
|
96
|
-
batch_rows.is_a?(Integer) && batch_rows.positive?
|
|
97
|
-
raise ArgumentError, "buffer_bytes must be a positive Integer; got #{buffer_bytes.inspect}" unless
|
|
98
|
-
buffer_bytes.is_a?(Integer) && buffer_bytes.positive?
|
|
99
144
|
|
|
145
|
+
flags = Runtime.batch_flags(notes, dialect.long_rows == :report)
|
|
146
|
+
start = Runtime.check_sizes([], batch_rows, buffer_bytes, flags)
|
|
100
147
|
planned = Column.plan(plan) unless plan.nil?
|
|
101
148
|
@dialect = dialect
|
|
102
149
|
@batch_rows = batch_rows
|
|
103
|
-
@buffer_bytes =
|
|
150
|
+
@buffer_bytes = start
|
|
104
151
|
@close_source = close_source
|
|
105
|
-
@
|
|
152
|
+
@notes = notes ? true : false
|
|
153
|
+
@kernel = Runtime::Delimited.start(dialect.packed_with, dialect.packed_tokens)
|
|
106
154
|
raise ArgumentError, "separator #{dialect.separator.inspect} is not tab or printable ASCII other than '\"'" if
|
|
107
155
|
@kernel.nil?
|
|
108
156
|
|
|
109
157
|
@plan = UNBOUND
|
|
110
158
|
# A plan handed over here is bound before the header is read, as #bind would bind it.
|
|
111
|
-
bind_kernel(planned) if planned
|
|
159
|
+
bind_kernel(planned, Column.specs(planned)) if planned
|
|
112
160
|
@start = 0
|
|
113
161
|
@closed = false
|
|
114
162
|
@failure = nil
|
|
163
|
+
# What the IO answered and the decoder made of it, each held until the buffer it goes into is
|
|
164
|
+
# whole; and the batch made, until it is handed over.
|
|
165
|
+
@incoming = @undelivered = @handing = @in_flight = nil
|
|
115
166
|
take(source)
|
|
116
167
|
@header = dialect.has_header ? read_header : nil
|
|
117
168
|
# A plan bound before the header was read is held to it as soon as it is.
|
|
@@ -136,13 +187,16 @@ module HyperTabular
|
|
|
136
187
|
# width the header gives every record, which leaves the reader unbound; RuntimeError if
|
|
137
188
|
# the reader already has a plan; IOError on a closed reader.
|
|
138
189
|
def bind(plan)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
190
|
+
exclusive do
|
|
191
|
+
raise IOError, "closed reader" if @closed
|
|
192
|
+
raise "the reader already has a plan: a plan is bound once" if bound?
|
|
193
|
+
|
|
194
|
+
plan = Column.plan(plan)
|
|
195
|
+
specs = Column.specs(plan)
|
|
196
|
+
within_header(plan)
|
|
197
|
+
atomically { bind_kernel(plan, specs) }
|
|
198
|
+
self
|
|
199
|
+
end
|
|
146
200
|
end
|
|
147
201
|
|
|
148
202
|
# Whether a plan has been bound: always, for a reader opened with one.
|
|
@@ -163,6 +217,9 @@ module HyperTabular
|
|
|
163
217
|
# The most rows one #read delivers.
|
|
164
218
|
attr_reader :batch_rows
|
|
165
219
|
|
|
220
|
+
# Whether each cell's detail and origin are kept.
|
|
221
|
+
attr_reader :notes
|
|
222
|
+
|
|
166
223
|
# Records finished so far — the header and skipped blank lines included.
|
|
167
224
|
def records
|
|
168
225
|
@kernel.records
|
|
@@ -171,40 +228,30 @@ module HyperTabular
|
|
|
171
228
|
# Reads the next batch: a Batch of up to #batch_rows rows, or nil once the input is
|
|
172
229
|
# exhausted. A TabularError when the input is structurally broken — raised after every
|
|
173
230
|
# intact row before the break has been delivered, and the same error again on every
|
|
174
|
-
# later call.
|
|
175
|
-
# #bind then
|
|
231
|
+
# later call. Whatever the IO raises, unchanged, and then RuntimeError naming it on every later
|
|
232
|
+
# call. IOError on a closed reader; RuntimeError before a plan is bound, which #bind then
|
|
233
|
+
# cures, and from inside one of this reader's own calls (busy).
|
|
176
234
|
def read
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
length, last = window
|
|
183
|
-
raise structural unless @kernel.fill(@start, length, last) == Runtime::Delimited::OK
|
|
235
|
+
handing_out do
|
|
236
|
+
raise IOError, "closed reader" if @closed
|
|
237
|
+
next hand_over if @handing
|
|
238
|
+
raise @failure if @failure
|
|
239
|
+
raise UNBOUND_MESSAGE unless bound?
|
|
184
240
|
|
|
185
|
-
|
|
186
|
-
if @kernel.rows.positive?
|
|
187
|
-
batch = made(@kernel.rows)
|
|
188
|
-
@start += consumed
|
|
189
|
-
return batch
|
|
190
|
-
end
|
|
191
|
-
@start += consumed
|
|
192
|
-
return nil if last && (consumed == length || consumed.zero?)
|
|
193
|
-
|
|
194
|
-
refill if consumed.zero?
|
|
241
|
+
advance(false) ? hand_over : nil
|
|
195
242
|
end
|
|
196
243
|
end
|
|
197
244
|
|
|
198
245
|
# Every remaining row, batch after batch: yields one frozen Array per row, a verdict
|
|
199
|
-
# per plan column. Without a block, an
|
|
246
|
+
# per plan column (a constrained column's as Batch#get_checked says it). Without a block, an
|
|
247
|
+
# Enumerator. Raises what #read raises.
|
|
200
248
|
def each_row
|
|
201
249
|
return to_enum(:each_row) unless block_given?
|
|
202
250
|
|
|
203
|
-
|
|
204
|
-
columns = Array.new(@plan.size) { |column| batch.
|
|
251
|
+
batches do |batch|
|
|
252
|
+
columns = Array.new(@plan.size) { |column| batch.send(:verdicts_of, column) }
|
|
205
253
|
batch.rows.times { |row| yield columns.map { |column| column[row] }.freeze }
|
|
206
254
|
end
|
|
207
|
-
self
|
|
208
255
|
end
|
|
209
256
|
|
|
210
257
|
# Every remaining row, batch after batch, as a Row — which, its batch being a copy,
|
|
@@ -213,19 +260,19 @@ module HyperTabular
|
|
|
213
260
|
def each(&block)
|
|
214
261
|
return to_enum(:each) unless block
|
|
215
262
|
|
|
216
|
-
|
|
217
|
-
batch.each(&block)
|
|
218
|
-
end
|
|
219
|
-
self
|
|
263
|
+
batches { |batch| batch.each(&block) }
|
|
220
264
|
end
|
|
221
265
|
|
|
222
266
|
# Closes the IO if this reader was told to (+close_source+, or DelimitedReader.open).
|
|
223
267
|
# Batches already read stay good. Safe to call twice.
|
|
224
268
|
def close
|
|
225
|
-
|
|
269
|
+
exclusive do
|
|
270
|
+
next if @closed
|
|
226
271
|
|
|
227
|
-
|
|
228
|
-
|
|
272
|
+
@closed = true
|
|
273
|
+
@in_flight = nil
|
|
274
|
+
@io.close if @close_source && @io
|
|
275
|
+
end
|
|
229
276
|
nil
|
|
230
277
|
end
|
|
231
278
|
|
|
@@ -248,6 +295,55 @@ module HyperTabular
|
|
|
248
295
|
UNBOUND_MESSAGE = "the reader has no plan yet: bind one before reading".freeze
|
|
249
296
|
private_constant :UNBOUND, :UNBOUND_MESSAGE
|
|
250
297
|
|
|
298
|
+
# The step ahead of the fill, for a dialect that declares an encoding: the original bytes, and
|
|
299
|
+
# what the core's transcoder has made of them. A failure it meets is held until the fill has
|
|
300
|
+
# had all the text ahead of it.
|
|
301
|
+
class Decoder
|
|
302
|
+
# The original offset of a unit with no character, once one has been met.
|
|
303
|
+
attr_reader :failed_at
|
|
304
|
+
|
|
305
|
+
# Decodes +encoding+ (the core's code). A String is all of the +input+; otherwise #feed gives
|
|
306
|
+
# it each read of the source.
|
|
307
|
+
def initialize(encoding, input)
|
|
308
|
+
@encoding = encoding
|
|
309
|
+
@raw = input || "".b
|
|
310
|
+
@at = 0
|
|
311
|
+
@eof = input ? true : false
|
|
312
|
+
@offset = 0
|
|
313
|
+
@failed_at = nil
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# At most +room+ more bytes of decoded text, which is empty when there is no more until the
|
|
317
|
+
# failure is raised or the source is over; nil when it needs more of the source first (#feed).
|
|
318
|
+
# +room+ is at least the 4 one character takes.
|
|
319
|
+
def step(room)
|
|
320
|
+
consumed, written, failed, failure_at, out =
|
|
321
|
+
Runtime.transcode(@encoding, @raw, @at, @raw.bytesize - @at, @eof, room)
|
|
322
|
+
@failed_at = @offset + failure_at unless failed.zero?
|
|
323
|
+
@at += consumed
|
|
324
|
+
@offset += consumed
|
|
325
|
+
out if written.positive? || @failed_at || @eof
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# Keeps what is left, less than a character, and +chunk+ behind it: a read of the source, empty
|
|
329
|
+
# at its end.
|
|
330
|
+
def feed(chunk)
|
|
331
|
+
if chunk.empty?
|
|
332
|
+
@eof = true
|
|
333
|
+
return
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
@raw = @raw.byteslice(@at, @raw.bytesize - @at).b << chunk.b
|
|
337
|
+
@at = 0
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Whether the whole source has been decoded.
|
|
341
|
+
def finished?
|
|
342
|
+
@eof && @at == @raw.bytesize
|
|
343
|
+
end
|
|
344
|
+
end
|
|
345
|
+
private_constant :Decoder
|
|
346
|
+
|
|
251
347
|
# ArgumentError for the first column of +plan+ at or past the width the header has given
|
|
252
348
|
# every record: such a column could only ever be empty.
|
|
253
349
|
def within_header(plan)
|
|
@@ -261,19 +357,24 @@ module HyperTabular
|
|
|
261
357
|
"past the #{width} columns the header gives every record"
|
|
262
358
|
end
|
|
263
359
|
|
|
264
|
-
# Sizes the kernel's arrays for +plan+ (already checked to be Columns) and takes it.
|
|
265
|
-
|
|
360
|
+
# Sizes the kernel's arrays for +plan+ (already checked to be Columns) and takes it. Only a
|
|
361
|
+
# dialect that reports long rows keeps each row's width.
|
|
362
|
+
def bind_kernel(plan, specs)
|
|
266
363
|
# Cell-table entries one row takes: the widest ordinal the plan reads, plus two.
|
|
267
364
|
per_row = (plan.map(&:ordinal).max || -1) + 2
|
|
268
|
-
@
|
|
365
|
+
widths = @dialect.long_rows == :report
|
|
366
|
+
Runtime.check_sizes(specs, @batch_rows, @buffer_bytes, Runtime.batch_flags(@notes, widths))
|
|
367
|
+
@kernel.bind(specs, plan.map(&:value_bytes), @batch_rows, per_row, @notes, widths)
|
|
269
368
|
@plan = plan
|
|
270
369
|
end
|
|
271
370
|
|
|
272
371
|
# Takes the source: a String becomes the whole input, an IO the thing to refill from.
|
|
273
372
|
def take(source)
|
|
373
|
+
encoding = Dialect::ENCODINGS.fetch(@dialect.encoding)
|
|
274
374
|
if source.is_a?(String)
|
|
275
|
-
@buffer = utf8(source)
|
|
276
|
-
@
|
|
375
|
+
@buffer = encoding.zero? ? utf8(source) : String.new(encoding: Encoding::UTF_8)
|
|
376
|
+
@decoder = Decoder.new(encoding, source.b.freeze) unless encoding.zero?
|
|
377
|
+
@eof = encoding.zero?
|
|
277
378
|
elsif defined?(::Pathname) && source.is_a?(::Pathname)
|
|
278
379
|
# It has a #read, and one that starts the file over on every call.
|
|
279
380
|
raise ArgumentError, "source is a Pathname; DelimitedReader.open reads a path"
|
|
@@ -282,6 +383,7 @@ module HyperTabular
|
|
|
282
383
|
@partial = source.respond_to?(:readpartial)
|
|
283
384
|
@buffer = String.new(encoding: Encoding::UTF_8)
|
|
284
385
|
@eof = false
|
|
386
|
+
@decoder = Decoder.new(encoding, nil) unless encoding.zero?
|
|
285
387
|
else
|
|
286
388
|
raise ArgumentError, "source must be a String or an IO; got #{source.class}"
|
|
287
389
|
end
|
|
@@ -306,60 +408,168 @@ module HyperTabular
|
|
|
306
408
|
length > MAX_ROW_BYTES ? [MAX_ROW_BYTES, false] : [length, @eof]
|
|
307
409
|
end
|
|
308
410
|
|
|
309
|
-
#
|
|
310
|
-
#
|
|
311
|
-
#
|
|
411
|
+
# Runs the core over the buffer until it has a batch (held in @handing, for #hand_over) or, for
|
|
412
|
+
# the +header+, the names: answers it, or false once the input is over. Only the IO's own read is
|
|
413
|
+
# outside the wait for asynchronous exceptions, and only while it blocks (#fetch).
|
|
414
|
+
def advance(header)
|
|
415
|
+
loop do
|
|
416
|
+
found = atomically { step(header) }
|
|
417
|
+
return found unless found.is_a?(Integer)
|
|
418
|
+
|
|
419
|
+
fetch(found)
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
# One step of #advance, inside the wait: what it answers, or how many bytes to ask the IO for
|
|
424
|
+
# before the next.
|
|
425
|
+
def step(header)
|
|
426
|
+
loop do
|
|
427
|
+
length, last = window
|
|
428
|
+
offset = @kernel.offset
|
|
429
|
+
code = moving(offset) { header ? @kernel.header(@start, length, last) : @kernel.fill(@start, length, last) }
|
|
430
|
+
raise structural unless code == Runtime::Delimited::OK
|
|
431
|
+
|
|
432
|
+
found = moving(offset) do
|
|
433
|
+
consumed = @kernel.consumed
|
|
434
|
+
held = header ? header_names : made(@kernel.rows) if @kernel.rows.positive?
|
|
435
|
+
@handing = held unless header || held.nil?
|
|
436
|
+
@start += consumed
|
|
437
|
+
next held unless held.nil?
|
|
438
|
+
next false if last && (consumed == length || consumed.zero?)
|
|
439
|
+
|
|
440
|
+
refill if consumed.zero?
|
|
441
|
+
end
|
|
442
|
+
return found unless found.nil?
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
# Runs the block, which follows the core's call made at +offset+: an exception it raises once the
|
|
447
|
+
# core has read on from there is kept as the reader's failure, since what the core read is lost.
|
|
448
|
+
def moving(offset)
|
|
449
|
+
yield
|
|
450
|
+
rescue Exception => e
|
|
451
|
+
@failure ||= spoiled(e) unless e.equal?(@failure) || @kernel.offset == offset
|
|
452
|
+
raise
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
# The noun the busy and spoiled messages name.
|
|
456
|
+
def noun = "reader"
|
|
457
|
+
|
|
458
|
+
# Puts the unfinished record at the front of a new buffer, and what the IO answered behind it; or,
|
|
459
|
+
# with nothing answered yet, how much to ask it for — at least as much again as is pending, so a
|
|
460
|
+
# record that does not fit doubles the buffer rather than creeping up on it.
|
|
312
461
|
def refill
|
|
313
462
|
pending = @buffer.bytesize - @start
|
|
463
|
+
return refill_decoded(pending) if @decoder
|
|
464
|
+
|
|
314
465
|
raise too_long if @io.nil? || pending >= MAX_ROW_BYTES
|
|
466
|
+
return [@buffer_bytes, pending].max if @incoming.nil?
|
|
315
467
|
|
|
316
|
-
|
|
317
|
-
|
|
468
|
+
if @incoming.empty?
|
|
469
|
+
@incoming = nil
|
|
318
470
|
@eof = true
|
|
319
471
|
return
|
|
320
472
|
end
|
|
321
|
-
|
|
322
|
-
@
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
473
|
+
rebuild(@buffer.byteslice(@start, pending) << @incoming.dup.force_encoding(Encoding::UTF_8))
|
|
474
|
+
@incoming = nil
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
# As #refill, through the decoder. A failure it holds is raised once the fill has had all the
|
|
478
|
+
# text ahead of it, and wants more: only a `\r` the fill is holding can still be finished, and
|
|
479
|
+
# nothing is coming to say whether a `\n` is part of it, so this says so instead.
|
|
480
|
+
def refill_decoded(pending)
|
|
481
|
+
if @undelivered.nil? && (byte = @decoder.failed_at)
|
|
482
|
+
unfinished = @buffer.byteslice(@start, pending)
|
|
483
|
+
unless unfinished.end_with?("\r")
|
|
484
|
+
raise @failure = TabularError.new(:encoding, @kernel.records, @kernel.line, byte)
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
return rebuild(unfinished << "\n")
|
|
488
|
+
end
|
|
489
|
+
raise too_long if pending >= MAX_ROW_BYTES
|
|
490
|
+
|
|
491
|
+
# Decoded bytes are gone from the decoder, so they are held until the buffer they go into is
|
|
492
|
+
# whole: an allocation refused along the way leaves a read that can be asked again.
|
|
493
|
+
@undelivered ||= decoded([@buffer_bytes, pending, 4].max)
|
|
494
|
+
return @buffer_bytes if @undelivered.nil?
|
|
495
|
+
|
|
496
|
+
rebuild(@buffer.byteslice(@start, pending) << @undelivered.force_encoding(Encoding::UTF_8))
|
|
497
|
+
@undelivered = nil
|
|
498
|
+
@eof = @decoder.finished? && @decoder.failed_at.nil?
|
|
499
|
+
nil
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
# The decoder's next text, given what the IO answered when it needs it; nil when it needs more.
|
|
503
|
+
def decoded(room)
|
|
504
|
+
loop do
|
|
505
|
+
text = @decoder.step(room)
|
|
506
|
+
return text unless text.nil? && @incoming
|
|
507
|
+
|
|
508
|
+
@decoder.feed(@incoming)
|
|
509
|
+
@incoming = nil
|
|
510
|
+
end
|
|
326
511
|
end
|
|
327
512
|
|
|
328
|
-
#
|
|
329
|
-
#
|
|
330
|
-
def
|
|
331
|
-
@
|
|
332
|
-
|
|
513
|
+
# Attaches +grown+, a String no batch and no core call has seen, and makes it the buffer: the core
|
|
514
|
+
# is never handed an address that a String it was reading no longer owns. Answers nil.
|
|
515
|
+
def rebuild(grown)
|
|
516
|
+
@kernel.attach(grown)
|
|
517
|
+
@buffer = grown
|
|
518
|
+
@start = 0
|
|
333
519
|
nil
|
|
334
520
|
end
|
|
335
521
|
|
|
522
|
+
# Up to +bytes+ more of the IO, held in @incoming, empty at its end: #readpartial where there is
|
|
523
|
+
# one, so a pipe or a socket hands over what it has rather than waiting to fill the request.
|
|
524
|
+
#
|
|
525
|
+
# An asynchronous exception lands only while the IO's read blocks, so a Timeout still breaks a
|
|
526
|
+
# read waiting on a pipe; once the IO has answered it waits until the answer is held. One that
|
|
527
|
+
# lands in the blocked read may take with it bytes Ruby had already read from the IO, so it
|
|
528
|
+
# leaves the reader refusing every later read. So does anything the IO's read raises itself,
|
|
529
|
+
# an IOError or a SystemCallError included: Ruby can't tell one from an exception that landed
|
|
530
|
+
# there (Timeout.timeout's own class can be Errno::ETIMEDOUT), so none is taken as resumable.
|
|
531
|
+
def fetch(bytes)
|
|
532
|
+
answered = false
|
|
533
|
+
while_blocked do
|
|
534
|
+
@incoming = (@partial ? @io.readpartial(bytes) : @io.read(bytes)) || EMPTY
|
|
535
|
+
answered = true
|
|
536
|
+
rescue EOFError
|
|
537
|
+
@incoming = EMPTY
|
|
538
|
+
answered = true
|
|
539
|
+
end
|
|
540
|
+
rescue Exception => e
|
|
541
|
+
Thread.handle_interrupt(Object => :never) { @failure ||= cut_short(e) } unless answered
|
|
542
|
+
raise
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
# The failure that stands for +error+, raised while the IO was being read.
|
|
546
|
+
def cut_short(error)
|
|
547
|
+
RuntimeError.new("the reader can't read on: #{error.class} (#{error.message}) was raised while it was " \
|
|
548
|
+
"reading its IO, and what that read had taken from the IO may be lost")
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
# The end of the IO, held.
|
|
552
|
+
EMPTY = "".b.freeze
|
|
553
|
+
private_constant :EMPTY
|
|
554
|
+
|
|
336
555
|
# The batch the core just wrote, copied out of the kernel's arrays: its spans index the
|
|
337
556
|
# input from @start, where the core was handed it.
|
|
338
557
|
def made(rows)
|
|
339
558
|
columns = @kernel.columns
|
|
559
|
+
reported = @dialect.long_rows == :report
|
|
560
|
+
notes = @notes ? Array.new(@plan.size) { |column| columns.notes(column, rows) } : nil
|
|
340
561
|
Batch.new(@plan, rows, Array.new(@plan.size) { |column| columns.values(column, rows) },
|
|
341
562
|
Array.new(@plan.size) { |column| columns.verdicts(column, rows) },
|
|
342
|
-
@kernel.cells, @kernel.per_row, @buffer, @start, @kernel.arena,
|
|
563
|
+
@kernel.cells, @kernel.per_row, @buffer, @start, @kernel.arena,
|
|
564
|
+
workbook: false, notes: notes, checks: Batch.checks_of(@plan, columns, rows),
|
|
565
|
+
widths: reported ? columns.widths(rows) : nil,
|
|
566
|
+
expected: @kernel.expected, long_rows: Dialect::LONG_ROWS.fetch(@dialect.long_rows))
|
|
343
567
|
end
|
|
344
568
|
|
|
345
|
-
# Reads the header record: its names, or none for an input with no record at all
|
|
569
|
+
# Reads the header record: its names, or none for an input with no record at all, whose width
|
|
570
|
+
# is unknown.
|
|
346
571
|
def read_header
|
|
347
|
-
|
|
348
|
-
length, last = window
|
|
349
|
-
raise structural unless @kernel.header(@start, length, last) == Runtime::Delimited::OK
|
|
350
|
-
|
|
351
|
-
consumed = @kernel.consumed
|
|
352
|
-
if @kernel.rows.positive?
|
|
353
|
-
names = header_names
|
|
354
|
-
@start += consumed
|
|
355
|
-
return names
|
|
356
|
-
end
|
|
357
|
-
@start += consumed
|
|
358
|
-
# An empty input has no header and no rows; the width is unknown.
|
|
359
|
-
return Header.new.freeze if last && (consumed == length || consumed.zero?)
|
|
360
|
-
|
|
361
|
-
refill if consumed.zero?
|
|
362
|
-
end
|
|
572
|
+
advance(true) || Header.new.freeze
|
|
363
573
|
end
|
|
364
574
|
|
|
365
575
|
# The names the core just located: in the input as written, or — flagged — unescaped
|
|
@@ -387,6 +597,21 @@ module HyperTabular
|
|
|
387
597
|
@failure = TabularError.from(@kernel.failure)
|
|
388
598
|
end
|
|
389
599
|
|
|
600
|
+
# What a sampler reads this reader in: batches of as many rows as the core says fit +wanted+.
|
|
601
|
+
protected def sample_in(plan, wanted)
|
|
602
|
+
@batch_rows = Runtime.sample_rows(Column.specs(plan), wanted,
|
|
603
|
+
Runtime.batch_flags(@notes, @dialect.long_rows == :report))
|
|
604
|
+
bind(plan)
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
# Binds a first read's one text column, one row a batch, and answers how many cells a record
|
|
608
|
+
# has: how a suggestion learns a headerless source's width.
|
|
609
|
+
protected def first_width
|
|
610
|
+
bind([Column.text(0)])
|
|
611
|
+
read
|
|
612
|
+
column_count || 0
|
|
613
|
+
end
|
|
614
|
+
|
|
390
615
|
# The failure of a record that outgrew the ceiling, kept: it is final too.
|
|
391
616
|
def too_long
|
|
392
617
|
@failure = TabularError.new(:row_too_long, @kernel.records, @kernel.line, @kernel.offset)
|