csv_preview 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +25 -0
- data/LICENSE +21 -0
- data/README.md +587 -0
- data/SECURITY.md +51 -0
- data/fonts/Inter-Regular.ttf +0 -0
- data/fonts/Inter-SemiBold.ttf +0 -0
- data/fonts/OFL.txt +92 -0
- data/lib/csv_preview/active_storage_previewer.rb +97 -0
- data/lib/csv_preview/cell_kind.rb +75 -0
- data/lib/csv_preview/configuration.rb +183 -0
- data/lib/csv_preview/deadline.rb +31 -0
- data/lib/csv_preview/errors.rb +30 -0
- data/lib/csv_preview/layout.rb +590 -0
- data/lib/csv_preview/painter.rb +194 -0
- data/lib/csv_preview/parser.rb +456 -0
- data/lib/csv_preview/railtie.rb +54 -0
- data/lib/csv_preview/reader.rb +259 -0
- data/lib/csv_preview/sample.rb +85 -0
- data/lib/csv_preview/sanitizer.rb +31 -0
- data/lib/csv_preview/text.rb +621 -0
- data/lib/csv_preview/theme.rb +207 -0
- data/lib/csv_preview/version.rb +6 -0
- data/lib/csv_preview.rb +149 -0
- data/skills/csv-preview/SKILL.md +171 -0
- metadata +121 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CsvPreview
|
|
4
|
+
# Draws a {Layout} and encodes it (rendering.md section 7): a canvas with the header fill, the rules
|
|
5
|
+
# and the frame, then one n-ary composite of every rendered string (each cut to its cell) and the
|
|
6
|
+
# right-edge fade, flattened on the background and encoded with its metadata stripped.
|
|
7
|
+
#
|
|
8
|
+
# The painter draws no text itself. Every string was rendered, and measured with the same call, by
|
|
9
|
+
# {Layout}; the painter only places those images. Colours and sizes come from the layout's {Theme}.
|
|
10
|
+
#
|
|
11
|
+
# ruby-vips is already loaded when a {Layout} exists (building one draws its text), so loading this
|
|
12
|
+
# file does not load it.
|
|
13
|
+
# @api private
|
|
14
|
+
class Painter
|
|
15
|
+
# Output resolution at scale 1, written to the PNG pHYs chunk: 72 dpi, so a scale-2 image says 144
|
|
16
|
+
# dpi and viewers that honour it show it at its logical size.
|
|
17
|
+
DPI = 72.0
|
|
18
|
+
MM_PER_INCH = 25.4
|
|
19
|
+
|
|
20
|
+
# Saver settings for each output format (rendering.md 7.5). Metadata stripping is added per call.
|
|
21
|
+
ENCODINGS = {
|
|
22
|
+
png: [".png", { compression: 6 }.freeze],
|
|
23
|
+
webp: [".webp", { lossless: true }.freeze],
|
|
24
|
+
jpeg: [".jpg", { Q: 90, optimize_coding: true, subsample_mode: :off, interlace: false }.freeze]
|
|
25
|
+
}.freeze
|
|
26
|
+
|
|
27
|
+
# VIPS_FOREIGN_KEEP_NONE, the saver `keep` flags value that strips all metadata (libvips 8.15 or
|
|
28
|
+
# newer). Passed as the Integer because ruby-vips before 2.2.5 cannot convert a Symbol such as
|
|
29
|
+
# `:none` for a flags argument and raises TypeError; every supported ruby-vips accepts the Integer.
|
|
30
|
+
KEEP_NONE = 0
|
|
31
|
+
|
|
32
|
+
# One image to composite and the position of its top-left corner on the canvas.
|
|
33
|
+
Layer = Data.define(:image, :x, :y)
|
|
34
|
+
private_constant :Layer
|
|
35
|
+
|
|
36
|
+
# @param layout [Layout]
|
|
37
|
+
# @param format [Symbol] :png, :jpeg or :webp
|
|
38
|
+
# @param deadline [#check!] checked before each cell is placed, before compositing and before encoding
|
|
39
|
+
# @return [String] the encoded image (binary)
|
|
40
|
+
# @raise [ArgumentError] for an unknown format
|
|
41
|
+
# @raise [RenderFailed] when libvips is missing, fails, lacks the saver for `format`, or the image
|
|
42
|
+
# would not match the layout's size and caps
|
|
43
|
+
# @raise [TimeoutExceeded]
|
|
44
|
+
def self.call(layout, format:, deadline:)
|
|
45
|
+
raise ArgumentError, "format must be one of #{ENCODINGS.keys.inspect}" unless ENCODINGS.key?(format)
|
|
46
|
+
unless Text.supported?
|
|
47
|
+
raise RenderFailed, "csv_preview needs libvips 8.12 or newer with text (Pango) support. Install libvips " \
|
|
48
|
+
"with Pango, #{Text::INSTALL_HINT}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
new(layout, format, deadline).paint
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private_class_method :new
|
|
55
|
+
|
|
56
|
+
# @api private
|
|
57
|
+
def initialize(layout, format, deadline)
|
|
58
|
+
@layout = layout
|
|
59
|
+
@format = format
|
|
60
|
+
@deadline = deadline
|
|
61
|
+
@theme = layout.theme
|
|
62
|
+
@geometry = layout.geometry
|
|
63
|
+
@width = layout.width_px
|
|
64
|
+
@height = layout.height_px
|
|
65
|
+
@resolution = DPI * layout.scale / MM_PER_INCH
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @api private
|
|
69
|
+
def paint
|
|
70
|
+
layers = text_layers
|
|
71
|
+
layers << fade_layer if @geometry.fade
|
|
72
|
+
check!
|
|
73
|
+
image = composite(canvas, layers)
|
|
74
|
+
check_size!(image)
|
|
75
|
+
check!
|
|
76
|
+
encode(image)
|
|
77
|
+
rescue Vips::Error
|
|
78
|
+
# The libvips message stays on the cause; this message never includes input content.
|
|
79
|
+
raise RenderFailed, "libvips failed to paint the preview; check the libvips installation, #{Text::INSTALL_HINT}"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
def check! = @deadline.check!(:paint)
|
|
85
|
+
|
|
86
|
+
def rgb(token) = @theme.rgb(token)
|
|
87
|
+
|
|
88
|
+
# 7.1: one image with the header fill, the rules and the frame. The frame can be part of it because
|
|
89
|
+
# nothing composited later reaches it: text is cut inside it and the fade stops at it.
|
|
90
|
+
#
|
|
91
|
+
# Nothing is drawn in place. Every band but the two side frames spans the full width, so the
|
|
92
|
+
# canvas is a 1 px column of row colours (later bands overwrite earlier ones), stretched to the
|
|
93
|
+
# width, with the side frames inserted on top. In-place drawing (`mutate` with `draw_rect!`) is
|
|
94
|
+
# not thread-safe: each draw makes libvips signal the earlier draw operations on the image without
|
|
95
|
+
# holding a reference to them, and Ruby's GC on another thread can free one mid-signal, which
|
|
96
|
+
# crashes the process.
|
|
97
|
+
def canvas
|
|
98
|
+
g = @geometry
|
|
99
|
+
rows = Array.new(@height, pixel(:background))
|
|
100
|
+
band = lambda do |token, y, height|
|
|
101
|
+
top = y.clamp(0, @height)
|
|
102
|
+
bottom = (y + height).clamp(0, @height)
|
|
103
|
+
rows.fill(pixel(token), top...bottom) if bottom > top
|
|
104
|
+
end
|
|
105
|
+
band.call(:header_fill, g.frame, g.header_height)
|
|
106
|
+
band.call(:header_rule_color, g.frame + g.header_height - g.header_rule, g.header_rule)
|
|
107
|
+
(1...@layout.rows_shown).each { |i| band.call(:row_rule_color, row_top(i), g.row_rule) }
|
|
108
|
+
band.call(:footer_rule_color, g.footer_top, g.footer_rule) if g.footer_top
|
|
109
|
+
band.call(:frame_color, 0, g.frame)
|
|
110
|
+
band.call(:frame_color, @height - g.frame, g.frame)
|
|
111
|
+
image = Vips::Image.new_from_memory_copy(rows.join, 1, @height, 3, :uchar)
|
|
112
|
+
.embed(0, 0, @width, @height, extend: :copy)
|
|
113
|
+
if g.frame.positive?
|
|
114
|
+
side = Vips::Image.black(g.frame, @height, bands: 3).new_from_image(rgb(:frame_color))
|
|
115
|
+
image = image.insert(side, 0, 0).insert(side, @width - g.frame, 0)
|
|
116
|
+
end
|
|
117
|
+
image.copy(interpretation: :srgb, xres: @resolution, yres: @resolution)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# One sRGB pixel of a colour token as 3 bytes.
|
|
121
|
+
def pixel(token) = rgb(token).pack("C3")
|
|
122
|
+
|
|
123
|
+
def row_top(i) = @geometry.frame + @geometry.header_height + (i * @geometry.row_height)
|
|
124
|
+
|
|
125
|
+
# 7.2: every string at its placement, cut to its cell box.
|
|
126
|
+
def text_layers
|
|
127
|
+
@layout.placements.filter_map do |placement|
|
|
128
|
+
check!
|
|
129
|
+
clip(placement)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Cuts a string's image to the intersection of its box and the area inside the frame, so tall
|
|
134
|
+
# marks and fallback glyphs never paint over a neighbouring cell, a rule or the frame. nil when
|
|
135
|
+
# nothing is left.
|
|
136
|
+
def clip(placement)
|
|
137
|
+
image = placement.run.image
|
|
138
|
+
box = placement.box
|
|
139
|
+
frame = @geometry.frame
|
|
140
|
+
left = [placement.x, box.x, frame].max
|
|
141
|
+
top = [placement.y, box.y, frame].max
|
|
142
|
+
right = [placement.x + image.width, box.x + box.width, @width - frame].min
|
|
143
|
+
bottom = [placement.y + image.height, box.y + box.height, @height - frame].min
|
|
144
|
+
return nil if right <= left || bottom <= top
|
|
145
|
+
|
|
146
|
+
width = right - left
|
|
147
|
+
height = bottom - top
|
|
148
|
+
unless width == image.width && height == image.height
|
|
149
|
+
image = image.crop(left - placement.x, top - placement.y, width, height)
|
|
150
|
+
end
|
|
151
|
+
Layer.new(image: image, x: left, y: top)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# 7.3: the background colour with an alpha ramp of 255 * t ** fade_exponent, from 0 at the fade's
|
|
155
|
+
# left edge to 255 at the frame, over the header band and the rows.
|
|
156
|
+
def fade_layer
|
|
157
|
+
box = @geometry.fade
|
|
158
|
+
ramp = ((Vips::Image.grey(box.width, 1)**@theme.fade_exponent) * 255).cast(:uchar)
|
|
159
|
+
ramp = ramp.embed(0, 0, box.width, box.height, extend: :copy)
|
|
160
|
+
colour = Vips::Image.black(box.width, box.height, bands: 3).new_from_image(rgb(:background))
|
|
161
|
+
Layer.new(image: colour.bandjoin(ramp).copy(interpretation: :srgb), x: box.x, y: box.y)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# 7.4: one n-ary composite of every layer, then the alpha is flattened away: 8-bit, 3-band sRGB.
|
|
165
|
+
def composite(base, layers)
|
|
166
|
+
return base if layers.empty?
|
|
167
|
+
|
|
168
|
+
base.composite(layers.map(&:image), Array.new(layers.size, :over), x: layers.map(&:x), y: layers.map(&:y))
|
|
169
|
+
.flatten(background: rgb(:background))
|
|
170
|
+
.cast(:uchar)
|
|
171
|
+
.copy(interpretation: :srgb, xres: @resolution, yres: @resolution)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# 7.6: the image is exactly the layout's size, within the caps, before anything is encoded.
|
|
175
|
+
def check_size!(image)
|
|
176
|
+
return if image.width == @width && image.height == @height && image.bands == 3 &&
|
|
177
|
+
@width <= @layout.max_width_px && @height <= @layout.max_height_px
|
|
178
|
+
|
|
179
|
+
raise RenderFailed, "painted #{image.width} by #{image.height} px for a #{@width} by #{@height} px layout " \
|
|
180
|
+
"with caps of #{@layout.max_width_px} by #{@layout.max_height_px} px. This is a bug in " \
|
|
181
|
+
"csv_preview; please report it at #{Help::ISSUES}"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# 7.5: metadata is always stripped; libvips otherwise writes an eXIf chunk into PNGs.
|
|
185
|
+
def encode(image)
|
|
186
|
+
suffix, options = ENCODINGS.fetch(@format)
|
|
187
|
+
strip = Vips.at_least_libvips?(8, 15) ? { keep: KEEP_NONE } : { strip: true }
|
|
188
|
+
image.write_to_buffer(suffix, **options, **strip)
|
|
189
|
+
rescue Vips::Error
|
|
190
|
+
raise RenderFailed, "libvips could not encode #{@format} (it may have been built without a #{@format} saver). " \
|
|
191
|
+
"Use format: :png or a libvips with #{@format} support, see #{Help::OPTIONS}"
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "csv"
|
|
4
|
+
require "stringio"
|
|
5
|
+
|
|
6
|
+
module CsvPreview
|
|
7
|
+
# Turns the Reader's decoded, bounded text into a {Sample} with Ruby's csv library, wrapped so that it
|
|
8
|
+
# stays bounded and never aborts on malformed input. Internal API.
|
|
9
|
+
#
|
|
10
|
+
# See docs/spec/parsing-and-safety.md section 4 (and appendix A for the reference core).
|
|
11
|
+
# @api private
|
|
12
|
+
class Parser
|
|
13
|
+
CANDIDATES = [",", ";", "\t", "|"].freeze
|
|
14
|
+
DEFAULT_DELIMITER = ","
|
|
15
|
+
SNIFF_BYTES = 65_536
|
|
16
|
+
SNIFF_ROWS = 30
|
|
17
|
+
SNIFF_COLUMNS = 64
|
|
18
|
+
CONSISTENCY_SLACK = Rational(1, 10)
|
|
19
|
+
# Excel's delimiter directive, honoured only as the very first line and only with delimiter: :auto.
|
|
20
|
+
SEP_DIRECTIVE = /\Asep=(.)\r?\n/
|
|
21
|
+
NOT_SPACE = /[^ ]/
|
|
22
|
+
|
|
23
|
+
# @param result [Reader::Result] its text is rewritten in place (it belongs to the gem)
|
|
24
|
+
# @param config [Configuration] a resolved snapshot
|
|
25
|
+
# @param deadline [Deadline]
|
|
26
|
+
# @return [Sample]
|
|
27
|
+
# @raise [TimeoutExceeded]
|
|
28
|
+
def self.call(result, config:, deadline:) = new(config, deadline).call(result)
|
|
29
|
+
|
|
30
|
+
# Rewrites line breaks in place: CRLF and lone CR become LF, then runs of LF collapse to one.
|
|
31
|
+
# Display-equivalent (section 4.2). Two C passes, no copy.
|
|
32
|
+
#
|
|
33
|
+
# @param text [String] mutable UTF-8 text
|
|
34
|
+
# @return [String] the same object
|
|
35
|
+
def self.normalize!(text)
|
|
36
|
+
text.tr!("\r", "\n")
|
|
37
|
+
text.squeeze!("\n")
|
|
38
|
+
text
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Byte cap for one kept raw cell (section 4.8): 4 bytes per code point, twice over, plus headroom.
|
|
42
|
+
# @param max_cell_chars [Integer]
|
|
43
|
+
# @return [Integer]
|
|
44
|
+
def self.raw_cap(max_cell_chars) = (8 * (max_cell_chars + 1)) + 64
|
|
45
|
+
|
|
46
|
+
def initialize(config, deadline)
|
|
47
|
+
@config = config
|
|
48
|
+
@deadline = deadline
|
|
49
|
+
@raw_cap = self.class.raw_cap(config.max_cell_chars)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Section 4.11.
|
|
53
|
+
def call(result)
|
|
54
|
+
text = +result.text
|
|
55
|
+
delimiter = @config.delimiter
|
|
56
|
+
if delimiter == :auto
|
|
57
|
+
directive, text = excel_directive(text)
|
|
58
|
+
self.class.normalize!(text)
|
|
59
|
+
delimiter = directive || sniff(text, result.complete)
|
|
60
|
+
else
|
|
61
|
+
self.class.normalize!(text)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
max_rows = @config.max_rows
|
|
65
|
+
records = Records.new(text, complete: result.complete, delimiter: delimiter, keep_rows: max_rows + 1,
|
|
66
|
+
keep_columns: @config.max_columns, raw_cap: @raw_cap, deadline: @deadline).call
|
|
67
|
+
cleaned = records.kept.map do |cells|
|
|
68
|
+
@deadline.check!(:sanitize)
|
|
69
|
+
cells.map { |cell| Sanitizer.clean(cell.text, max_cell_chars: @config.max_cell_chars, capped: cell.capped) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
header = records.count.positive? &&
|
|
73
|
+
(@config.header_row == :auto ? Header.header?(cleaned, records.widths.max) : @config.header_row)
|
|
74
|
+
data = header ? cleaned[1, max_rows] || [] : cleaned.first(max_rows)
|
|
75
|
+
used = header ? records.widths.first(max_rows + 1) : records.widths.first(max_rows)
|
|
76
|
+
column_count = used.max || 0
|
|
77
|
+
rows = data.map { |row| fit(row, column_count) }
|
|
78
|
+
rows_read = records.count - (header ? 1 : 0)
|
|
79
|
+
complete = records.complete
|
|
80
|
+
|
|
81
|
+
Sample.new(
|
|
82
|
+
headers: header ? fit(cleaned[0], column_count) : nil,
|
|
83
|
+
rows: rows,
|
|
84
|
+
column_count: column_count,
|
|
85
|
+
total_columns: records.max_width,
|
|
86
|
+
total_rows: complete ? rows_read : nil,
|
|
87
|
+
rows_read: rows_read,
|
|
88
|
+
rows_truncated: rows_read > rows.size || !complete,
|
|
89
|
+
columns_truncated: records.max_width > column_count,
|
|
90
|
+
column_kinds: Array.new(column_count) { |c| CellKind.column(rows.map { |row| row[c] }) },
|
|
91
|
+
delimiter: delimiter,
|
|
92
|
+
encoding: result.encoding,
|
|
93
|
+
empty: records.count.zero?
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
private
|
|
98
|
+
|
|
99
|
+
# Removes a leading "sep=X" line in place when X is a valid delimiter. The directive is neither shown
|
|
100
|
+
# nor counted.
|
|
101
|
+
#
|
|
102
|
+
# @return [Array(String, String), Array(nil, String)] the directive's delimiter (or nil) and the text
|
|
103
|
+
def excel_directive(text)
|
|
104
|
+
return [nil, text] unless text.start_with?("sep=")
|
|
105
|
+
|
|
106
|
+
match = SEP_DIRECTIVE.match(text)
|
|
107
|
+
delimiter = match && match[1]
|
|
108
|
+
return [nil, text] unless delimiter && valid_delimiter?(delimiter)
|
|
109
|
+
|
|
110
|
+
text.bytesplice(0, match.byteoffset(0)[1], "")
|
|
111
|
+
[delimiter, text]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def valid_delimiter?(char)
|
|
115
|
+
char.ascii_only? && !Configuration::FORBIDDEN_DELIMITERS.include?(char)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Section 4.9.
|
|
119
|
+
def sniff(text, complete)
|
|
120
|
+
sample = text.byteslice(0, SNIFF_BYTES).scrub("")
|
|
121
|
+
whole = sample.bytesize == text.bytesize
|
|
122
|
+
unless whole
|
|
123
|
+
lf = sample.byterindex("\n")
|
|
124
|
+
sample = sample.byteslice(0, lf + 1) if lf
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
scores = CANDIDATES.each_with_index.filter_map do |candidate, index|
|
|
128
|
+
@deadline.check!(:sniff)
|
|
129
|
+
score(sample, candidate, index, complete && whole)
|
|
130
|
+
end
|
|
131
|
+
return DEFAULT_DELIMITER if scores.empty?
|
|
132
|
+
|
|
133
|
+
best = scores.map { |s| s[:consistency] }.max
|
|
134
|
+
scores.select { |s| s[:consistency] >= best - CONSISTENCY_SLACK }
|
|
135
|
+
.max_by { |s| [s[:clean], s[:mode], -s[:index]] }[:delimiter]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# @return [Hash, nil] nil when the candidate does not split the sample (mode below 2)
|
|
139
|
+
def score(sample, candidate, index, complete)
|
|
140
|
+
records = Records.new(sample, complete: complete, delimiter: candidate, keep_rows: SNIFF_ROWS,
|
|
141
|
+
keep_columns: SNIFF_COLUMNS, raw_cap: @raw_cap, deadline: @deadline,
|
|
142
|
+
kept_only: true).call
|
|
143
|
+
tally = records.widths.tally
|
|
144
|
+
return nil if tally.empty?
|
|
145
|
+
|
|
146
|
+
mode, mode_count = tally.max_by { |width, count| [count, width] }
|
|
147
|
+
return nil if mode < 2
|
|
148
|
+
|
|
149
|
+
others = CANDIDATES - [candidate]
|
|
150
|
+
non_empty = clean = 0
|
|
151
|
+
records.kept.each do |cells|
|
|
152
|
+
cells.each do |cell|
|
|
153
|
+
stripped = strip_spaces(cell.text)
|
|
154
|
+
next if stripped.empty?
|
|
155
|
+
|
|
156
|
+
non_empty += 1
|
|
157
|
+
clean += 1 if others.none? { |o| stripped.include?(o) } || CellKind.cell(stripped) == :number
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
{
|
|
161
|
+
delimiter: candidate, index: index, mode: mode,
|
|
162
|
+
consistency: Rational(mode_count, records.widths.size),
|
|
163
|
+
clean: non_empty.zero? ? 0 : Rational(clean, non_empty)
|
|
164
|
+
}
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Removes leading and trailing ASCII spaces with linear searches (the cell is at most raw_cap bytes).
|
|
168
|
+
def strip_spaces(text)
|
|
169
|
+
first = text.index(NOT_SPACE) or return ""
|
|
170
|
+
text[first..text.rindex(NOT_SPACE)]
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Pads a row with "" or cuts it to n cells.
|
|
174
|
+
def fit(row, n) = row.size >= n ? row.first(n) : row + Array.new(n - row.size, "")
|
|
175
|
+
|
|
176
|
+
# Header detection for header_row: :auto (section 4.10).
|
|
177
|
+
module Header
|
|
178
|
+
DATE = %r{\A(?:\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})
|
|
179
|
+
(?:[ T]\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:\s?(?:[AaPp][Mm]|Z|[+-]\d{2}:?\d{2}))?)?\z}x
|
|
180
|
+
MAX_DATE_LENGTH = 40
|
|
181
|
+
|
|
182
|
+
module_function
|
|
183
|
+
|
|
184
|
+
# @param cleaned [Array<Array<String>>] the sanitized kept records, the first being the candidate
|
|
185
|
+
# @param columns [Integer] the columns to consider (the widest kept record)
|
|
186
|
+
# @return [Boolean]
|
|
187
|
+
def header?(cleaned, columns)
|
|
188
|
+
first = Array.new(columns) { |c| cleaned[0][c] || "" }
|
|
189
|
+
head = first.map { |cell| klass(cell) }
|
|
190
|
+
data = cleaned.drop(1)
|
|
191
|
+
return head.all?(:text) if data.empty?
|
|
192
|
+
|
|
193
|
+
score = 0
|
|
194
|
+
columns.times do |c|
|
|
195
|
+
cells = data.filter_map { |row| row[c] unless row[c].nil? || row[c].empty? }
|
|
196
|
+
next if cells.empty?
|
|
197
|
+
|
|
198
|
+
score += column_score(first[c], head[c], cells)
|
|
199
|
+
end
|
|
200
|
+
values = first.reject(&:empty?)
|
|
201
|
+
score -= 1 if values.uniq.size != values.size
|
|
202
|
+
score.positive? || (score.zero? && head.all?(:text))
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def column_score(top, top_kind, cells)
|
|
206
|
+
kinds = cells.map { |cell| klass(cell) }
|
|
207
|
+
dtype = if kinds.count(:number) * 5 >= cells.size * 4 then :number
|
|
208
|
+
elsif kinds.count(:date) * 5 >= cells.size * 4 then :date
|
|
209
|
+
else :text
|
|
210
|
+
end
|
|
211
|
+
if dtype == :text
|
|
212
|
+
return -1 if top_kind == :empty
|
|
213
|
+
return 0 unless top_kind == :text && cells.size >= 2
|
|
214
|
+
|
|
215
|
+
lengths = cells.map(&:length).uniq
|
|
216
|
+
return 0 unless lengths.size == 1
|
|
217
|
+
|
|
218
|
+
top.length == lengths[0] ? -1 : 1
|
|
219
|
+
elsif top_kind == :text then 2
|
|
220
|
+
elsif top_kind == dtype then -2
|
|
221
|
+
else 0
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def klass(cell)
|
|
226
|
+
return :empty if cell.empty?
|
|
227
|
+
return :number if CellKind.cell(cell) == :number
|
|
228
|
+
return :date if cell.length <= MAX_DATE_LENGTH && DATE.match?(cell)
|
|
229
|
+
|
|
230
|
+
:text
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Runs Ruby's csv library over normalised text inside the gem's bounds: chunked reads through
|
|
235
|
+
# {Feed}, the field guard, and the recovery rules of section 4.6. See appendix A.
|
|
236
|
+
class Records
|
|
237
|
+
MAX_RECORD_FIELDS = 65_536 # field guard, section 4.5
|
|
238
|
+
CHECK_EVERY = 1_024 # records between deadline checks (every chunk csv reads is checked too)
|
|
239
|
+
QUOTE = '"'
|
|
240
|
+
LF = 0x0A
|
|
241
|
+
NOT_LF = /[^\n]/
|
|
242
|
+
BLANK = " \t\n"
|
|
243
|
+
|
|
244
|
+
# @!attribute text [String] the field, cut to the raw cap
|
|
245
|
+
# @!attribute capped [Boolean] the field was longer than the raw cap
|
|
246
|
+
RawCell = Data.define(:text, :capped)
|
|
247
|
+
Result = Data.define(:kept, :widths, :count, :max_width, :complete, :too_wide, :recovered)
|
|
248
|
+
|
|
249
|
+
# The IO that csv reads. Deliberately not a StringIO: csv reads a StringIO whole and runs its regexes
|
|
250
|
+
# over one long string, but reads any other IO in chunks of at most 8 KiB (section 4.3).
|
|
251
|
+
class Feed
|
|
252
|
+
CHUNK = 8_192
|
|
253
|
+
|
|
254
|
+
# @param text [String] the text this pass parses
|
|
255
|
+
# @param base [Integer] offset of text[0] in the parser's text
|
|
256
|
+
# @param owner [Records] receives the deadline checks and the field-guard counts
|
|
257
|
+
# @param guard [Boolean] whether the field guard applies to this pass
|
|
258
|
+
# @param chunk_limit [Integer, nil] tests only: forces smaller chunks
|
|
259
|
+
def initialize(text, base, owner, guard:, chunk_limit: nil)
|
|
260
|
+
@io = StringIO.new(text)
|
|
261
|
+
@base = base
|
|
262
|
+
@owner = owner
|
|
263
|
+
@guard = guard
|
|
264
|
+
@chunk_limit = chunk_limit
|
|
265
|
+
@stopped = false
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# csv calls gets("\n", 8192). Up to `limit` bytes are returned whatever the separator: csv's
|
|
269
|
+
# chunked scanner treats chunks as a byte stream. StringIO#gets never splits a character.
|
|
270
|
+
def gets(_separator = nil, limit = CHUNK)
|
|
271
|
+
return nil if @stopped
|
|
272
|
+
|
|
273
|
+
@owner.checkpoint
|
|
274
|
+
limit = [limit, @chunk_limit].min if @chunk_limit
|
|
275
|
+
chunk = @io.gets(nil, limit) or return nil
|
|
276
|
+
@stopped = @guard && @owner.too_wide?(chunk, @base + @io.pos)
|
|
277
|
+
chunk
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def eof? = @stopped || @io.eof?
|
|
281
|
+
alias eof eof?
|
|
282
|
+
def external_encoding = Encoding::UTF_8
|
|
283
|
+
def internal_encoding = nil
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# @param text [String] normalised UTF-8 text (not modified)
|
|
287
|
+
# @param complete [Boolean] the Reader read the whole source
|
|
288
|
+
# @param delimiter [String] one ASCII character
|
|
289
|
+
# @param keep_rows [Integer] non-blank records to keep (max_rows + 1)
|
|
290
|
+
# @param keep_columns [Integer] fields to keep per record
|
|
291
|
+
# @param raw_cap [Integer] byte cap per kept field
|
|
292
|
+
# @param deadline [Deadline]
|
|
293
|
+
# @param kept_only [Boolean] stop once keep_rows records are kept (the sniffer)
|
|
294
|
+
# @param chunk_limit [Integer, nil] tests only: forces smaller chunks
|
|
295
|
+
def initialize(text, complete:, delimiter:, keep_rows:, keep_columns:, raw_cap:, deadline:,
|
|
296
|
+
kept_only: false, chunk_limit: nil)
|
|
297
|
+
@text = text
|
|
298
|
+
@complete = complete
|
|
299
|
+
@delimiter = delimiter
|
|
300
|
+
@keep_rows = keep_rows
|
|
301
|
+
@keep_columns = keep_columns
|
|
302
|
+
@raw_cap = raw_cap
|
|
303
|
+
@deadline = deadline
|
|
304
|
+
@kept_only = kept_only
|
|
305
|
+
@chunk_limit = chunk_limit
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# @return [Result]
|
|
309
|
+
# @raise [TimeoutExceeded]
|
|
310
|
+
def call
|
|
311
|
+
@end = @text.bytesize
|
|
312
|
+
@last_quote = last_quote_offset
|
|
313
|
+
@kept = []
|
|
314
|
+
@widths = []
|
|
315
|
+
@count = @max_width = @rows_seen = @fields = 0
|
|
316
|
+
@too_wide = @recovered = false
|
|
317
|
+
catch(:enough) { parse }
|
|
318
|
+
Result.new(kept: @kept, widths: @widths, count: @count, max_width: @max_width, complete: @complete,
|
|
319
|
+
too_wide: @too_wide, recovered: @recovered)
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Feed callback: a deadline checkpoint before every chunk.
|
|
323
|
+
def checkpoint = @deadline.check!(:parse)
|
|
324
|
+
|
|
325
|
+
# Feed callback. Counts delimiters since the last record csv returned (quoted ones too, so it
|
|
326
|
+
# over-counts). Past the guard, csv's input ends here.
|
|
327
|
+
def too_wide?(chunk, pos)
|
|
328
|
+
@fields += chunk.count(@delimiter)
|
|
329
|
+
return false if @fields <= field_limit
|
|
330
|
+
|
|
331
|
+
@stop ||= [@end, @complete, @too_wide]
|
|
332
|
+
@end = pos if pos < @end
|
|
333
|
+
@complete = false
|
|
334
|
+
@too_wide = true
|
|
335
|
+
true
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
private
|
|
339
|
+
|
|
340
|
+
def field_limit = MAX_RECORD_FIELDS
|
|
341
|
+
def last_quote_offset = @text.byterindex(QUOTE)
|
|
342
|
+
|
|
343
|
+
def csv_options(quoting)
|
|
344
|
+
{ col_sep: @delimiter, row_sep: "\n", quote_char: (QUOTE if quoting),
|
|
345
|
+
liberal_parsing: true, skip_blanks: true }
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Section 4.6: at most one stray-quote recovery, then one terminal pass.
|
|
349
|
+
def parse
|
|
350
|
+
start = 0
|
|
351
|
+
stray_done = false
|
|
352
|
+
while start < @end
|
|
353
|
+
@stop = nil
|
|
354
|
+
x = parse_from(start) or return
|
|
355
|
+
|
|
356
|
+
# csv raised CSV::MalformedCSVError: the record at offset x has a quote that is still open
|
|
357
|
+
# where csv's input ended.
|
|
358
|
+
@recovered = true
|
|
359
|
+
if @stop && !stray_done
|
|
360
|
+
@end, @complete, @too_wide = @stop # rule 1: a runaway quote, not a wide record
|
|
361
|
+
elsif @stop || (!@complete && cut_tail?(x))
|
|
362
|
+
parse_string(@text.byteslice(x, @end - x) + QUOTE, x, quoting: true, guard: false) # rule 2
|
|
363
|
+
return
|
|
364
|
+
end
|
|
365
|
+
if stray_done
|
|
366
|
+
parse_string(@text.byteslice(x, @end - x), x, quoting: false) # rule 4
|
|
367
|
+
return
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
stray_done = true # rule 3
|
|
371
|
+
lf = @text.byteindex("\n", x)
|
|
372
|
+
stop = lf && lf < @end ? lf + 1 : @end
|
|
373
|
+
parse_string(@text.byteslice(x, stop - x), x, quoting: false)
|
|
374
|
+
start = stop
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# The byte limit cut record x, as far as the gem can tell: x is past the kept records, or x
|
|
379
|
+
# starts on the last line read.
|
|
380
|
+
def cut_tail?(x)
|
|
381
|
+
lf = @text.byteindex("\n", x)
|
|
382
|
+
@count >= @keep_rows || lf.nil? || lf >= @end - 1
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Parses @text[start...@end] with quoting on. Returns nil when csv finished, or the offset of the
|
|
386
|
+
# record csv rejected.
|
|
387
|
+
def parse_from(start)
|
|
388
|
+
@fields = 0
|
|
389
|
+
pos = start
|
|
390
|
+
feed = Feed.new(@text.byteslice(start, @end - start), start, self, guard: true, chunk_limit: @chunk_limit)
|
|
391
|
+
csv = CSV.new(feed, **csv_options(true))
|
|
392
|
+
while (row = csv.shift)
|
|
393
|
+
pos = record_end(pos, csv.line)
|
|
394
|
+
add(row)
|
|
395
|
+
next unless @last_quote.nil? || pos > @last_quote
|
|
396
|
+
|
|
397
|
+
# No quote character is left: quote-free mode gives the same rows, faster.
|
|
398
|
+
parse_string(@text.byteslice(pos, @end - pos), pos, quoting: false) if pos < @end
|
|
399
|
+
return nil
|
|
400
|
+
end
|
|
401
|
+
nil
|
|
402
|
+
rescue CSV::MalformedCSVError
|
|
403
|
+
skip_blank_lines(pos)
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# A recovery or quote-free pass over a separate string. csv has nothing to reject here (no quotes,
|
|
407
|
+
# or the open quote was closed); if it ever does, the rest is dropped and the counts become lower
|
|
408
|
+
# bounds.
|
|
409
|
+
def parse_string(str, base, quoting:, guard: true)
|
|
410
|
+
@fields = 0
|
|
411
|
+
feed = Feed.new(str, base, self, guard: guard, chunk_limit: @chunk_limit)
|
|
412
|
+
csv = CSV.new(feed, **csv_options(quoting))
|
|
413
|
+
while (row = csv.shift)
|
|
414
|
+
add(row)
|
|
415
|
+
end
|
|
416
|
+
rescue CSV::MalformedCSVError
|
|
417
|
+
@complete = false
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# Byte offset just past the record csv returned (csv.line is its raw text, line break included).
|
|
421
|
+
def record_end(pos, line) = skip_blank_lines(pos) + line.bytesize
|
|
422
|
+
|
|
423
|
+
# csv skips empty lines (skip_blanks) and leaves them out of csv.line. After normalisation each
|
|
424
|
+
# is a single LF.
|
|
425
|
+
def skip_blank_lines(pos)
|
|
426
|
+
return pos unless @text.getbyte(pos) == LF
|
|
427
|
+
|
|
428
|
+
[@text.byteindex(NOT_LF, pos) || @end, @end].min
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
def add(row)
|
|
432
|
+
@fields = 0
|
|
433
|
+
@rows_seen += 1
|
|
434
|
+
checkpoint if (@rows_seen % CHECK_EVERY).zero?
|
|
435
|
+
last = row.rindex { |f| f && f.count(BLANK) != f.bytesize }
|
|
436
|
+
return unless last # blank record: not kept, not counted
|
|
437
|
+
|
|
438
|
+
width = last + 1
|
|
439
|
+
if @count < @keep_rows
|
|
440
|
+
@kept << row.first(@keep_columns).map { |f| cell(f) }
|
|
441
|
+
@widths << [width, @keep_columns].min
|
|
442
|
+
end
|
|
443
|
+
@max_width = width if width > @max_width
|
|
444
|
+
@count += 1
|
|
445
|
+
throw :enough if @kept_only && @count >= @keep_rows
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
def cell(field)
|
|
449
|
+
return RawCell.new("", false) if field.nil?
|
|
450
|
+
return RawCell.new(field, false) if field.bytesize <= @raw_cap
|
|
451
|
+
|
|
452
|
+
RawCell.new(field.byteslice(0, @raw_cap).scrub(""), true)
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
end
|
|
456
|
+
end
|