csv_preview 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CsvPreview
4
+ # Draws a {Layout} and encodes it (rendering.md section 7): a canvas with the header fill, the rules
5
+ # and the frame, then one n-ary composite of every rendered string (each cut to its cell) and the
6
+ # right-edge fade, flattened on the background and encoded with its metadata stripped.
7
+ #
8
+ # The painter draws no text itself. Every string was rendered, and measured with the same call, by
9
+ # {Layout}; the painter only places those images. Colours and sizes come from the layout's {Theme}.
10
+ #
11
+ # ruby-vips is already loaded when a {Layout} exists (building one draws its text), so loading this
12
+ # file does not load it.
13
+ # @api private
14
+ class Painter
15
+ # Output resolution at scale 1, written to the PNG pHYs chunk: 72 dpi, so a scale-2 image says 144
16
+ # dpi and viewers that honour it show it at its logical size.
17
+ DPI = 72.0
18
+ MM_PER_INCH = 25.4
19
+
20
+ # Saver settings for each output format (rendering.md 7.5). Metadata stripping is added per call.
21
+ ENCODINGS = {
22
+ png: [".png", { compression: 6 }.freeze],
23
+ webp: [".webp", { lossless: true }.freeze],
24
+ jpeg: [".jpg", { Q: 90, optimize_coding: true, subsample_mode: :off, interlace: false }.freeze]
25
+ }.freeze
26
+
27
+ # VIPS_FOREIGN_KEEP_NONE, the saver `keep` flags value that strips all metadata (libvips 8.15 or
28
+ # newer). Passed as the Integer because ruby-vips before 2.2.5 cannot convert a Symbol such as
29
+ # `:none` for a flags argument and raises TypeError; every supported ruby-vips accepts the Integer.
30
+ KEEP_NONE = 0
31
+
32
+ # One image to composite and the position of its top-left corner on the canvas.
33
+ Layer = Data.define(:image, :x, :y)
34
+ private_constant :Layer
35
+
36
+ # @param layout [Layout]
37
+ # @param format [Symbol] :png, :jpeg or :webp
38
+ # @param deadline [#check!] checked before each cell is placed, before compositing and before encoding
39
+ # @return [String] the encoded image (binary)
40
+ # @raise [ArgumentError] for an unknown format
41
+ # @raise [RenderFailed] when libvips is missing, fails, lacks the saver for `format`, or the image
42
+ # would not match the layout's size and caps
43
+ # @raise [TimeoutExceeded]
44
+ def self.call(layout, format:, deadline:)
45
+ raise ArgumentError, "format must be one of #{ENCODINGS.keys.inspect}" unless ENCODINGS.key?(format)
46
+ unless Text.supported?
47
+ raise RenderFailed, "csv_preview needs libvips 8.12 or newer with text (Pango) support. Install libvips " \
48
+ "with Pango, #{Text::INSTALL_HINT}"
49
+ end
50
+
51
+ new(layout, format, deadline).paint
52
+ end
53
+
54
+ private_class_method :new
55
+
56
+ # @api private
57
+ def initialize(layout, format, deadline)
58
+ @layout = layout
59
+ @format = format
60
+ @deadline = deadline
61
+ @theme = layout.theme
62
+ @geometry = layout.geometry
63
+ @width = layout.width_px
64
+ @height = layout.height_px
65
+ @resolution = DPI * layout.scale / MM_PER_INCH
66
+ end
67
+
68
+ # @api private
69
+ def paint
70
+ layers = text_layers
71
+ layers << fade_layer if @geometry.fade
72
+ check!
73
+ image = composite(canvas, layers)
74
+ check_size!(image)
75
+ check!
76
+ encode(image)
77
+ rescue Vips::Error
78
+ # The libvips message stays on the cause; this message never includes input content.
79
+ raise RenderFailed, "libvips failed to paint the preview; check the libvips installation, #{Text::INSTALL_HINT}"
80
+ end
81
+
82
+ private
83
+
84
+ def check! = @deadline.check!(:paint)
85
+
86
+ def rgb(token) = @theme.rgb(token)
87
+
88
+ # 7.1: one image with the header fill, the rules and the frame. The frame can be part of it because
89
+ # nothing composited later reaches it: text is cut inside it and the fade stops at it.
90
+ #
91
+ # Nothing is drawn in place. Every band but the two side frames spans the full width, so the
92
+ # canvas is a 1 px column of row colours (later bands overwrite earlier ones), stretched to the
93
+ # width, with the side frames inserted on top. In-place drawing (`mutate` with `draw_rect!`) is
94
+ # not thread-safe: each draw makes libvips signal the earlier draw operations on the image without
95
+ # holding a reference to them, and Ruby's GC on another thread can free one mid-signal, which
96
+ # crashes the process.
97
+ def canvas
98
+ g = @geometry
99
+ rows = Array.new(@height, pixel(:background))
100
+ band = lambda do |token, y, height|
101
+ top = y.clamp(0, @height)
102
+ bottom = (y + height).clamp(0, @height)
103
+ rows.fill(pixel(token), top...bottom) if bottom > top
104
+ end
105
+ band.call(:header_fill, g.frame, g.header_height)
106
+ band.call(:header_rule_color, g.frame + g.header_height - g.header_rule, g.header_rule)
107
+ (1...@layout.rows_shown).each { |i| band.call(:row_rule_color, row_top(i), g.row_rule) }
108
+ band.call(:footer_rule_color, g.footer_top, g.footer_rule) if g.footer_top
109
+ band.call(:frame_color, 0, g.frame)
110
+ band.call(:frame_color, @height - g.frame, g.frame)
111
+ image = Vips::Image.new_from_memory_copy(rows.join, 1, @height, 3, :uchar)
112
+ .embed(0, 0, @width, @height, extend: :copy)
113
+ if g.frame.positive?
114
+ side = Vips::Image.black(g.frame, @height, bands: 3).new_from_image(rgb(:frame_color))
115
+ image = image.insert(side, 0, 0).insert(side, @width - g.frame, 0)
116
+ end
117
+ image.copy(interpretation: :srgb, xres: @resolution, yres: @resolution)
118
+ end
119
+
120
+ # One sRGB pixel of a colour token as 3 bytes.
121
+ def pixel(token) = rgb(token).pack("C3")
122
+
123
+ def row_top(i) = @geometry.frame + @geometry.header_height + (i * @geometry.row_height)
124
+
125
+ # 7.2: every string at its placement, cut to its cell box.
126
+ def text_layers
127
+ @layout.placements.filter_map do |placement|
128
+ check!
129
+ clip(placement)
130
+ end
131
+ end
132
+
133
+ # Cuts a string's image to the intersection of its box and the area inside the frame, so tall
134
+ # marks and fallback glyphs never paint over a neighbouring cell, a rule or the frame. nil when
135
+ # nothing is left.
136
+ def clip(placement)
137
+ image = placement.run.image
138
+ box = placement.box
139
+ frame = @geometry.frame
140
+ left = [placement.x, box.x, frame].max
141
+ top = [placement.y, box.y, frame].max
142
+ right = [placement.x + image.width, box.x + box.width, @width - frame].min
143
+ bottom = [placement.y + image.height, box.y + box.height, @height - frame].min
144
+ return nil if right <= left || bottom <= top
145
+
146
+ width = right - left
147
+ height = bottom - top
148
+ unless width == image.width && height == image.height
149
+ image = image.crop(left - placement.x, top - placement.y, width, height)
150
+ end
151
+ Layer.new(image: image, x: left, y: top)
152
+ end
153
+
154
+ # 7.3: the background colour with an alpha ramp of 255 * t ** fade_exponent, from 0 at the fade's
155
+ # left edge to 255 at the frame, over the header band and the rows.
156
+ def fade_layer
157
+ box = @geometry.fade
158
+ ramp = ((Vips::Image.grey(box.width, 1)**@theme.fade_exponent) * 255).cast(:uchar)
159
+ ramp = ramp.embed(0, 0, box.width, box.height, extend: :copy)
160
+ colour = Vips::Image.black(box.width, box.height, bands: 3).new_from_image(rgb(:background))
161
+ Layer.new(image: colour.bandjoin(ramp).copy(interpretation: :srgb), x: box.x, y: box.y)
162
+ end
163
+
164
+ # 7.4: one n-ary composite of every layer, then the alpha is flattened away: 8-bit, 3-band sRGB.
165
+ def composite(base, layers)
166
+ return base if layers.empty?
167
+
168
+ base.composite(layers.map(&:image), Array.new(layers.size, :over), x: layers.map(&:x), y: layers.map(&:y))
169
+ .flatten(background: rgb(:background))
170
+ .cast(:uchar)
171
+ .copy(interpretation: :srgb, xres: @resolution, yres: @resolution)
172
+ end
173
+
174
+ # 7.6: the image is exactly the layout's size, within the caps, before anything is encoded.
175
+ def check_size!(image)
176
+ return if image.width == @width && image.height == @height && image.bands == 3 &&
177
+ @width <= @layout.max_width_px && @height <= @layout.max_height_px
178
+
179
+ raise RenderFailed, "painted #{image.width} by #{image.height} px for a #{@width} by #{@height} px layout " \
180
+ "with caps of #{@layout.max_width_px} by #{@layout.max_height_px} px. This is a bug in " \
181
+ "csv_preview; please report it at #{Help::ISSUES}"
182
+ end
183
+
184
+ # 7.5: metadata is always stripped; libvips otherwise writes an eXIf chunk into PNGs.
185
+ def encode(image)
186
+ suffix, options = ENCODINGS.fetch(@format)
187
+ strip = Vips.at_least_libvips?(8, 15) ? { keep: KEEP_NONE } : { strip: true }
188
+ image.write_to_buffer(suffix, **options, **strip)
189
+ rescue Vips::Error
190
+ raise RenderFailed, "libvips could not encode #{@format} (it may have been built without a #{@format} saver). " \
191
+ "Use format: :png or a libvips with #{@format} support, see #{Help::OPTIONS}"
192
+ end
193
+ end
194
+ end
@@ -0,0 +1,456 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "csv"
4
+ require "stringio"
5
+
6
+ module CsvPreview
7
+ # Turns the Reader's decoded, bounded text into a {Sample} with Ruby's csv library, wrapped so that it
8
+ # stays bounded and never aborts on malformed input. Internal API.
9
+ #
10
+ # See docs/spec/parsing-and-safety.md section 4 (and appendix A for the reference core).
11
+ # @api private
12
+ class Parser
13
+ CANDIDATES = [",", ";", "\t", "|"].freeze
14
+ DEFAULT_DELIMITER = ","
15
+ SNIFF_BYTES = 65_536
16
+ SNIFF_ROWS = 30
17
+ SNIFF_COLUMNS = 64
18
+ CONSISTENCY_SLACK = Rational(1, 10)
19
+ # Excel's delimiter directive, honoured only as the very first line and only with delimiter: :auto.
20
+ SEP_DIRECTIVE = /\Asep=(.)\r?\n/
21
+ NOT_SPACE = /[^ ]/
22
+
23
+ # @param result [Reader::Result] its text is rewritten in place (it belongs to the gem)
24
+ # @param config [Configuration] a resolved snapshot
25
+ # @param deadline [Deadline]
26
+ # @return [Sample]
27
+ # @raise [TimeoutExceeded]
28
+ def self.call(result, config:, deadline:) = new(config, deadline).call(result)
29
+
30
+ # Rewrites line breaks in place: CRLF and lone CR become LF, then runs of LF collapse to one.
31
+ # Display-equivalent (section 4.2). Two C passes, no copy.
32
+ #
33
+ # @param text [String] mutable UTF-8 text
34
+ # @return [String] the same object
35
+ def self.normalize!(text)
36
+ text.tr!("\r", "\n")
37
+ text.squeeze!("\n")
38
+ text
39
+ end
40
+
41
+ # Byte cap for one kept raw cell (section 4.8): 4 bytes per code point, twice over, plus headroom.
42
+ # @param max_cell_chars [Integer]
43
+ # @return [Integer]
44
+ def self.raw_cap(max_cell_chars) = (8 * (max_cell_chars + 1)) + 64
45
+
46
+ def initialize(config, deadline)
47
+ @config = config
48
+ @deadline = deadline
49
+ @raw_cap = self.class.raw_cap(config.max_cell_chars)
50
+ end
51
+
52
+ # Section 4.11.
53
+ def call(result)
54
+ text = +result.text
55
+ delimiter = @config.delimiter
56
+ if delimiter == :auto
57
+ directive, text = excel_directive(text)
58
+ self.class.normalize!(text)
59
+ delimiter = directive || sniff(text, result.complete)
60
+ else
61
+ self.class.normalize!(text)
62
+ end
63
+
64
+ max_rows = @config.max_rows
65
+ records = Records.new(text, complete: result.complete, delimiter: delimiter, keep_rows: max_rows + 1,
66
+ keep_columns: @config.max_columns, raw_cap: @raw_cap, deadline: @deadline).call
67
+ cleaned = records.kept.map do |cells|
68
+ @deadline.check!(:sanitize)
69
+ cells.map { |cell| Sanitizer.clean(cell.text, max_cell_chars: @config.max_cell_chars, capped: cell.capped) }
70
+ end
71
+
72
+ header = records.count.positive? &&
73
+ (@config.header_row == :auto ? Header.header?(cleaned, records.widths.max) : @config.header_row)
74
+ data = header ? cleaned[1, max_rows] || [] : cleaned.first(max_rows)
75
+ used = header ? records.widths.first(max_rows + 1) : records.widths.first(max_rows)
76
+ column_count = used.max || 0
77
+ rows = data.map { |row| fit(row, column_count) }
78
+ rows_read = records.count - (header ? 1 : 0)
79
+ complete = records.complete
80
+
81
+ Sample.new(
82
+ headers: header ? fit(cleaned[0], column_count) : nil,
83
+ rows: rows,
84
+ column_count: column_count,
85
+ total_columns: records.max_width,
86
+ total_rows: complete ? rows_read : nil,
87
+ rows_read: rows_read,
88
+ rows_truncated: rows_read > rows.size || !complete,
89
+ columns_truncated: records.max_width > column_count,
90
+ column_kinds: Array.new(column_count) { |c| CellKind.column(rows.map { |row| row[c] }) },
91
+ delimiter: delimiter,
92
+ encoding: result.encoding,
93
+ empty: records.count.zero?
94
+ )
95
+ end
96
+
97
+ private
98
+
99
+ # Removes a leading "sep=X" line in place when X is a valid delimiter. The directive is neither shown
100
+ # nor counted.
101
+ #
102
+ # @return [Array(String, String), Array(nil, String)] the directive's delimiter (or nil) and the text
103
+ def excel_directive(text)
104
+ return [nil, text] unless text.start_with?("sep=")
105
+
106
+ match = SEP_DIRECTIVE.match(text)
107
+ delimiter = match && match[1]
108
+ return [nil, text] unless delimiter && valid_delimiter?(delimiter)
109
+
110
+ text.bytesplice(0, match.byteoffset(0)[1], "")
111
+ [delimiter, text]
112
+ end
113
+
114
+ def valid_delimiter?(char)
115
+ char.ascii_only? && !Configuration::FORBIDDEN_DELIMITERS.include?(char)
116
+ end
117
+
118
+ # Section 4.9.
119
+ def sniff(text, complete)
120
+ sample = text.byteslice(0, SNIFF_BYTES).scrub("")
121
+ whole = sample.bytesize == text.bytesize
122
+ unless whole
123
+ lf = sample.byterindex("\n")
124
+ sample = sample.byteslice(0, lf + 1) if lf
125
+ end
126
+
127
+ scores = CANDIDATES.each_with_index.filter_map do |candidate, index|
128
+ @deadline.check!(:sniff)
129
+ score(sample, candidate, index, complete && whole)
130
+ end
131
+ return DEFAULT_DELIMITER if scores.empty?
132
+
133
+ best = scores.map { |s| s[:consistency] }.max
134
+ scores.select { |s| s[:consistency] >= best - CONSISTENCY_SLACK }
135
+ .max_by { |s| [s[:clean], s[:mode], -s[:index]] }[:delimiter]
136
+ end
137
+
138
+ # @return [Hash, nil] nil when the candidate does not split the sample (mode below 2)
139
+ def score(sample, candidate, index, complete)
140
+ records = Records.new(sample, complete: complete, delimiter: candidate, keep_rows: SNIFF_ROWS,
141
+ keep_columns: SNIFF_COLUMNS, raw_cap: @raw_cap, deadline: @deadline,
142
+ kept_only: true).call
143
+ tally = records.widths.tally
144
+ return nil if tally.empty?
145
+
146
+ mode, mode_count = tally.max_by { |width, count| [count, width] }
147
+ return nil if mode < 2
148
+
149
+ others = CANDIDATES - [candidate]
150
+ non_empty = clean = 0
151
+ records.kept.each do |cells|
152
+ cells.each do |cell|
153
+ stripped = strip_spaces(cell.text)
154
+ next if stripped.empty?
155
+
156
+ non_empty += 1
157
+ clean += 1 if others.none? { |o| stripped.include?(o) } || CellKind.cell(stripped) == :number
158
+ end
159
+ end
160
+ {
161
+ delimiter: candidate, index: index, mode: mode,
162
+ consistency: Rational(mode_count, records.widths.size),
163
+ clean: non_empty.zero? ? 0 : Rational(clean, non_empty)
164
+ }
165
+ end
166
+
167
+ # Removes leading and trailing ASCII spaces with linear searches (the cell is at most raw_cap bytes).
168
+ def strip_spaces(text)
169
+ first = text.index(NOT_SPACE) or return ""
170
+ text[first..text.rindex(NOT_SPACE)]
171
+ end
172
+
173
+ # Pads a row with "" or cuts it to n cells.
174
+ def fit(row, n) = row.size >= n ? row.first(n) : row + Array.new(n - row.size, "")
175
+
176
+ # Header detection for header_row: :auto (section 4.10).
177
+ module Header
178
+ DATE = %r{\A(?:\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})
179
+ (?:[ T]\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:\s?(?:[AaPp][Mm]|Z|[+-]\d{2}:?\d{2}))?)?\z}x
180
+ MAX_DATE_LENGTH = 40
181
+
182
+ module_function
183
+
184
+ # @param cleaned [Array<Array<String>>] the sanitized kept records, the first being the candidate
185
+ # @param columns [Integer] the columns to consider (the widest kept record)
186
+ # @return [Boolean]
187
+ def header?(cleaned, columns)
188
+ first = Array.new(columns) { |c| cleaned[0][c] || "" }
189
+ head = first.map { |cell| klass(cell) }
190
+ data = cleaned.drop(1)
191
+ return head.all?(:text) if data.empty?
192
+
193
+ score = 0
194
+ columns.times do |c|
195
+ cells = data.filter_map { |row| row[c] unless row[c].nil? || row[c].empty? }
196
+ next if cells.empty?
197
+
198
+ score += column_score(first[c], head[c], cells)
199
+ end
200
+ values = first.reject(&:empty?)
201
+ score -= 1 if values.uniq.size != values.size
202
+ score.positive? || (score.zero? && head.all?(:text))
203
+ end
204
+
205
+ def column_score(top, top_kind, cells)
206
+ kinds = cells.map { |cell| klass(cell) }
207
+ dtype = if kinds.count(:number) * 5 >= cells.size * 4 then :number
208
+ elsif kinds.count(:date) * 5 >= cells.size * 4 then :date
209
+ else :text
210
+ end
211
+ if dtype == :text
212
+ return -1 if top_kind == :empty
213
+ return 0 unless top_kind == :text && cells.size >= 2
214
+
215
+ lengths = cells.map(&:length).uniq
216
+ return 0 unless lengths.size == 1
217
+
218
+ top.length == lengths[0] ? -1 : 1
219
+ elsif top_kind == :text then 2
220
+ elsif top_kind == dtype then -2
221
+ else 0
222
+ end
223
+ end
224
+
225
+ def klass(cell)
226
+ return :empty if cell.empty?
227
+ return :number if CellKind.cell(cell) == :number
228
+ return :date if cell.length <= MAX_DATE_LENGTH && DATE.match?(cell)
229
+
230
+ :text
231
+ end
232
+ end
233
+
234
+ # Runs Ruby's csv library over normalised text inside the gem's bounds: chunked reads through
235
+ # {Feed}, the field guard, and the recovery rules of section 4.6. See appendix A.
236
+ class Records
237
+ MAX_RECORD_FIELDS = 65_536 # field guard, section 4.5
238
+ CHECK_EVERY = 1_024 # records between deadline checks (every chunk csv reads is checked too)
239
+ QUOTE = '"'
240
+ LF = 0x0A
241
+ NOT_LF = /[^\n]/
242
+ BLANK = " \t\n"
243
+
244
+ # @!attribute text [String] the field, cut to the raw cap
245
+ # @!attribute capped [Boolean] the field was longer than the raw cap
246
+ RawCell = Data.define(:text, :capped)
247
+ Result = Data.define(:kept, :widths, :count, :max_width, :complete, :too_wide, :recovered)
248
+
249
+ # The IO that csv reads. Deliberately not a StringIO: csv reads a StringIO whole and runs its regexes
250
+ # over one long string, but reads any other IO in chunks of at most 8 KiB (section 4.3).
251
+ class Feed
252
+ CHUNK = 8_192
253
+
254
+ # @param text [String] the text this pass parses
255
+ # @param base [Integer] offset of text[0] in the parser's text
256
+ # @param owner [Records] receives the deadline checks and the field-guard counts
257
+ # @param guard [Boolean] whether the field guard applies to this pass
258
+ # @param chunk_limit [Integer, nil] tests only: forces smaller chunks
259
+ def initialize(text, base, owner, guard:, chunk_limit: nil)
260
+ @io = StringIO.new(text)
261
+ @base = base
262
+ @owner = owner
263
+ @guard = guard
264
+ @chunk_limit = chunk_limit
265
+ @stopped = false
266
+ end
267
+
268
+ # csv calls gets("\n", 8192). Up to `limit` bytes are returned whatever the separator: csv's
269
+ # chunked scanner treats chunks as a byte stream. StringIO#gets never splits a character.
270
+ def gets(_separator = nil, limit = CHUNK)
271
+ return nil if @stopped
272
+
273
+ @owner.checkpoint
274
+ limit = [limit, @chunk_limit].min if @chunk_limit
275
+ chunk = @io.gets(nil, limit) or return nil
276
+ @stopped = @guard && @owner.too_wide?(chunk, @base + @io.pos)
277
+ chunk
278
+ end
279
+
280
+ def eof? = @stopped || @io.eof?
281
+ alias eof eof?
282
+ def external_encoding = Encoding::UTF_8
283
+ def internal_encoding = nil
284
+ end
285
+
286
+ # @param text [String] normalised UTF-8 text (not modified)
287
+ # @param complete [Boolean] the Reader read the whole source
288
+ # @param delimiter [String] one ASCII character
289
+ # @param keep_rows [Integer] non-blank records to keep (max_rows + 1)
290
+ # @param keep_columns [Integer] fields to keep per record
291
+ # @param raw_cap [Integer] byte cap per kept field
292
+ # @param deadline [Deadline]
293
+ # @param kept_only [Boolean] stop once keep_rows records are kept (the sniffer)
294
+ # @param chunk_limit [Integer, nil] tests only: forces smaller chunks
295
+ def initialize(text, complete:, delimiter:, keep_rows:, keep_columns:, raw_cap:, deadline:,
296
+ kept_only: false, chunk_limit: nil)
297
+ @text = text
298
+ @complete = complete
299
+ @delimiter = delimiter
300
+ @keep_rows = keep_rows
301
+ @keep_columns = keep_columns
302
+ @raw_cap = raw_cap
303
+ @deadline = deadline
304
+ @kept_only = kept_only
305
+ @chunk_limit = chunk_limit
306
+ end
307
+
308
+ # @return [Result]
309
+ # @raise [TimeoutExceeded]
310
+ def call
311
+ @end = @text.bytesize
312
+ @last_quote = last_quote_offset
313
+ @kept = []
314
+ @widths = []
315
+ @count = @max_width = @rows_seen = @fields = 0
316
+ @too_wide = @recovered = false
317
+ catch(:enough) { parse }
318
+ Result.new(kept: @kept, widths: @widths, count: @count, max_width: @max_width, complete: @complete,
319
+ too_wide: @too_wide, recovered: @recovered)
320
+ end
321
+
322
+ # Feed callback: a deadline checkpoint before every chunk.
323
+ def checkpoint = @deadline.check!(:parse)
324
+
325
+ # Feed callback. Counts delimiters since the last record csv returned (quoted ones too, so it
326
+ # over-counts). Past the guard, csv's input ends here.
327
+ def too_wide?(chunk, pos)
328
+ @fields += chunk.count(@delimiter)
329
+ return false if @fields <= field_limit
330
+
331
+ @stop ||= [@end, @complete, @too_wide]
332
+ @end = pos if pos < @end
333
+ @complete = false
334
+ @too_wide = true
335
+ true
336
+ end
337
+
338
+ private
339
+
340
+ def field_limit = MAX_RECORD_FIELDS
341
+ def last_quote_offset = @text.byterindex(QUOTE)
342
+
343
+ def csv_options(quoting)
344
+ { col_sep: @delimiter, row_sep: "\n", quote_char: (QUOTE if quoting),
345
+ liberal_parsing: true, skip_blanks: true }
346
+ end
347
+
348
+ # Section 4.6: at most one stray-quote recovery, then one terminal pass.
349
+ def parse
350
+ start = 0
351
+ stray_done = false
352
+ while start < @end
353
+ @stop = nil
354
+ x = parse_from(start) or return
355
+
356
+ # csv raised CSV::MalformedCSVError: the record at offset x has a quote that is still open
357
+ # where csv's input ended.
358
+ @recovered = true
359
+ if @stop && !stray_done
360
+ @end, @complete, @too_wide = @stop # rule 1: a runaway quote, not a wide record
361
+ elsif @stop || (!@complete && cut_tail?(x))
362
+ parse_string(@text.byteslice(x, @end - x) + QUOTE, x, quoting: true, guard: false) # rule 2
363
+ return
364
+ end
365
+ if stray_done
366
+ parse_string(@text.byteslice(x, @end - x), x, quoting: false) # rule 4
367
+ return
368
+ end
369
+
370
+ stray_done = true # rule 3
371
+ lf = @text.byteindex("\n", x)
372
+ stop = lf && lf < @end ? lf + 1 : @end
373
+ parse_string(@text.byteslice(x, stop - x), x, quoting: false)
374
+ start = stop
375
+ end
376
+ end
377
+
378
+ # The byte limit cut record x, as far as the gem can tell: x is past the kept records, or x
379
+ # starts on the last line read.
380
+ def cut_tail?(x)
381
+ lf = @text.byteindex("\n", x)
382
+ @count >= @keep_rows || lf.nil? || lf >= @end - 1
383
+ end
384
+
385
+ # Parses @text[start...@end] with quoting on. Returns nil when csv finished, or the offset of the
386
+ # record csv rejected.
387
+ def parse_from(start)
388
+ @fields = 0
389
+ pos = start
390
+ feed = Feed.new(@text.byteslice(start, @end - start), start, self, guard: true, chunk_limit: @chunk_limit)
391
+ csv = CSV.new(feed, **csv_options(true))
392
+ while (row = csv.shift)
393
+ pos = record_end(pos, csv.line)
394
+ add(row)
395
+ next unless @last_quote.nil? || pos > @last_quote
396
+
397
+ # No quote character is left: quote-free mode gives the same rows, faster.
398
+ parse_string(@text.byteslice(pos, @end - pos), pos, quoting: false) if pos < @end
399
+ return nil
400
+ end
401
+ nil
402
+ rescue CSV::MalformedCSVError
403
+ skip_blank_lines(pos)
404
+ end
405
+
406
+ # A recovery or quote-free pass over a separate string. csv has nothing to reject here (no quotes,
407
+ # or the open quote was closed); if it ever does, the rest is dropped and the counts become lower
408
+ # bounds.
409
+ def parse_string(str, base, quoting:, guard: true)
410
+ @fields = 0
411
+ feed = Feed.new(str, base, self, guard: guard, chunk_limit: @chunk_limit)
412
+ csv = CSV.new(feed, **csv_options(quoting))
413
+ while (row = csv.shift)
414
+ add(row)
415
+ end
416
+ rescue CSV::MalformedCSVError
417
+ @complete = false
418
+ end
419
+
420
+ # Byte offset just past the record csv returned (csv.line is its raw text, line break included).
421
+ def record_end(pos, line) = skip_blank_lines(pos) + line.bytesize
422
+
423
+ # csv skips empty lines (skip_blanks) and leaves them out of csv.line. After normalisation each
424
+ # is a single LF.
425
+ def skip_blank_lines(pos)
426
+ return pos unless @text.getbyte(pos) == LF
427
+
428
+ [@text.byteindex(NOT_LF, pos) || @end, @end].min
429
+ end
430
+
431
+ def add(row)
432
+ @fields = 0
433
+ @rows_seen += 1
434
+ checkpoint if (@rows_seen % CHECK_EVERY).zero?
435
+ last = row.rindex { |f| f && f.count(BLANK) != f.bytesize }
436
+ return unless last # blank record: not kept, not counted
437
+
438
+ width = last + 1
439
+ if @count < @keep_rows
440
+ @kept << row.first(@keep_columns).map { |f| cell(f) }
441
+ @widths << [width, @keep_columns].min
442
+ end
443
+ @max_width = width if width > @max_width
444
+ @count += 1
445
+ throw :enough if @kept_only && @count >= @keep_rows
446
+ end
447
+
448
+ def cell(field)
449
+ return RawCell.new("", false) if field.nil?
450
+ return RawCell.new(field, false) if field.bytesize <= @raw_cap
451
+
452
+ RawCell.new(field.byteslice(0, @raw_cap).scrub(""), true)
453
+ end
454
+ end
455
+ end
456
+ end