csv_preview 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +25 -0
- data/LICENSE +21 -0
- data/README.md +587 -0
- data/SECURITY.md +51 -0
- data/fonts/Inter-Regular.ttf +0 -0
- data/fonts/Inter-SemiBold.ttf +0 -0
- data/fonts/OFL.txt +92 -0
- data/lib/csv_preview/active_storage_previewer.rb +97 -0
- data/lib/csv_preview/cell_kind.rb +75 -0
- data/lib/csv_preview/configuration.rb +183 -0
- data/lib/csv_preview/deadline.rb +31 -0
- data/lib/csv_preview/errors.rb +30 -0
- data/lib/csv_preview/layout.rb +590 -0
- data/lib/csv_preview/painter.rb +194 -0
- data/lib/csv_preview/parser.rb +456 -0
- data/lib/csv_preview/railtie.rb +54 -0
- data/lib/csv_preview/reader.rb +259 -0
- data/lib/csv_preview/sample.rb +85 -0
- data/lib/csv_preview/sanitizer.rb +31 -0
- data/lib/csv_preview/text.rb +621 -0
- data/lib/csv_preview/theme.rb +207 -0
- data/lib/csv_preview/version.rb +6 -0
- data/lib/csv_preview.rb +149 -0
- data/skills/csv-preview/SKILL.md +171 -0
- metadata +121 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/railtie"
|
|
4
|
+
|
|
5
|
+
module CsvPreview
|
|
6
|
+
# Opt-in automatic registration of {ActiveStoragePreviewer}. Loaded by `lib/csv_preview.rb` only when
|
|
7
|
+
# Rails is present. With `config.csv_preview.register_previewer = true` the previewer is put first in
|
|
8
|
+
# `config.active_storage.previewers`; by default (false) nothing changes.
|
|
9
|
+
class Railtie < Rails::Railtie
|
|
10
|
+
config.csv_preview = ActiveSupport::OrderedOptions.new
|
|
11
|
+
config.csv_preview.register_previewer = false
|
|
12
|
+
|
|
13
|
+
# The flag is read after initializer files have run, hence the after_initialize hook. Active Storage
|
|
14
|
+
# copies the previewer list in a hook registered by its "active_storage.configs" initializer, so
|
|
15
|
+
# declaring this one before it registers this hook first. The list is changed in place, which keeps
|
|
16
|
+
# `ActiveStorage.previewers` and the config value the same Array.
|
|
17
|
+
initializer "csv_preview.register_previewer", before: "active_storage.configs" do |app|
|
|
18
|
+
config.after_initialize do
|
|
19
|
+
if app.config.csv_preview.register_previewer
|
|
20
|
+
if defined?(ActiveStorage::Engine)
|
|
21
|
+
previewers = (app.config.active_storage.previewers ||= [])
|
|
22
|
+
unless previewers.include?(CsvPreview::ActiveStoragePreviewer)
|
|
23
|
+
previewers.unshift(CsvPreview::ActiveStoragePreviewer)
|
|
24
|
+
end
|
|
25
|
+
else
|
|
26
|
+
Rails.logger&.warn("csv_preview: config.csv_preview.register_previewer is true, but Active Storage is " \
|
|
27
|
+
"not loaded; see #{CsvPreview::Help::RAILS}")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
Railtie.warm_up(app)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
class << self
|
|
36
|
+
# Pays the one-time font and Pango set-up cost at boot instead of in the first request, but only
|
|
37
|
+
# when the previewer is registered (by either route) and the app has not chosen its own font (an
|
|
38
|
+
# app that always passes `font:` never needs the bundled-font self-check, rendering.md 5.4).
|
|
39
|
+
# @api private
|
|
40
|
+
def warm_up(app)
|
|
41
|
+
return unless defined?(ActiveStorage::Engine)
|
|
42
|
+
# A pending autoload means nothing has referenced the previewer, so it cannot be in the list.
|
|
43
|
+
# Checking that first keeps this gate from loading the previewer in apps that never opted in.
|
|
44
|
+
return if CsvPreview.autoload?(:ActiveStoragePreviewer)
|
|
45
|
+
return unless app.config.active_storage.previewers&.include?(CsvPreview::ActiveStoragePreviewer)
|
|
46
|
+
return unless CsvPreview.config.font.nil? && CsvPreview::Text.supported?
|
|
47
|
+
|
|
48
|
+
CsvPreview::Text.warm_up!
|
|
49
|
+
rescue CsvPreview::Error => e
|
|
50
|
+
Rails.logger&.warn("csv_preview: font warm-up failed (#{e.class}: #{e.message})")
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
module CsvPreview
|
|
6
|
+
# Turns a source (path or IO-like) into bounded, decoded UTF-8 text. Internal API.
|
|
7
|
+
#
|
|
8
|
+
# See docs/spec/parsing-and-safety.md section 3. Messages of raised errors never include input bytes.
|
|
9
|
+
# @api private
|
|
10
|
+
module Reader
|
|
11
|
+
# @!attribute text [String] valid UTF-8, BOM removed
|
|
12
|
+
# @!attribute encoding [String] canonical Ruby name of the source encoding
|
|
13
|
+
# @!attribute complete [Boolean] the whole source was read within max_input_bytes
|
|
14
|
+
# @!attribute bytes_read [Integer] never above max_input_bytes
|
|
15
|
+
Result = Data.define(:text, :encoding, :complete, :bytes_read)
|
|
16
|
+
|
|
17
|
+
CHUNK = 65_536
|
|
18
|
+
# Open flag for paths (see read_path); 0 where the platform has no O_NONBLOCK.
|
|
19
|
+
OPEN_NONBLOCK = defined?(File::NONBLOCK) ? File::NONBLOCK : 0
|
|
20
|
+
BINARY_WINDOW_BYTES = 8_192
|
|
21
|
+
BINARY_WINDOW_CHARS = 4_096
|
|
22
|
+
PARITY_WINDOW_BYTES = 4_096
|
|
23
|
+
SUSPICIOUS = "\x01-\x08\x0B\x0C\x0E-\x1F\x7F"
|
|
24
|
+
REPLACEMENT = "�"
|
|
25
|
+
# Appended to every UnreadableInput message.
|
|
26
|
+
NOT_TEXT_HINT = "Only CSV and TSV text can be previewed; show a fallback for other files, see #{Help::ERRORS}"
|
|
27
|
+
|
|
28
|
+
MAGIC = [
|
|
29
|
+
["a PNG image", ->(b) { b.start_with?("\x89PNG\r\n\x1A\n".b) }],
|
|
30
|
+
["a JPEG image", ->(b) { b.start_with?("\xFF\xD8\xFF".b) }],
|
|
31
|
+
["a GIF image", ->(b) { b.start_with?("GIF87a".b, "GIF89a".b) }],
|
|
32
|
+
["a PDF document", ->(b) { b.start_with?("%PDF-".b) }],
|
|
33
|
+
["a ZIP archive (XLSX, ODS and similar)", ->(b) { b.start_with?("PK\x03\x04".b, "PK\x05\x06".b, "PK\x07\x08".b) }],
|
|
34
|
+
["an OLE2 file (XLS and similar)", ->(b) { b.start_with?("\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b) }],
|
|
35
|
+
["gzip data", ->(b) { b.start_with?("\x1F\x8B".b) }],
|
|
36
|
+
["bzip2 data", ->(b) { b.match?(/\ABZh[1-9]\x31\x41\x59\x26\x53\x59/n) }],
|
|
37
|
+
["xz data", ->(b) { b.start_with?("\xFD7zXZ\x00".b) }],
|
|
38
|
+
["zstd data", ->(b) { b.start_with?("\x28\xB5\x2F\xFD".b) }],
|
|
39
|
+
["a 7z archive", ->(b) { b.start_with?("7z\xBC\xAF\x27\x1C".b) }],
|
|
40
|
+
["a RAR archive", ->(b) { b.start_with?("Rar!\x1A\x07".b) }],
|
|
41
|
+
["a RIFF container", ->(b) { b.start_with?("RIFF".b) }],
|
|
42
|
+
["a TIFF image", ->(b) { b.start_with?("II*\x00".b, "MM\x00*".b) }],
|
|
43
|
+
["an ISO media file", ->(b) { b.byteslice(4, 4) == "ftyp".b }],
|
|
44
|
+
["an ELF executable", ->(b) { b.start_with?("\x7FELF".b) }],
|
|
45
|
+
["a SQLite database", ->(b) { b.start_with?("SQLite format 3\x00".b) }]
|
|
46
|
+
].freeze
|
|
47
|
+
|
|
48
|
+
# Detection BOMs, UTF-32LE before UTF-16LE because they share FF FE.
|
|
49
|
+
BOMS = [
|
|
50
|
+
["\xFF\xFE\x00\x00".b, Encoding::UTF_32LE],
|
|
51
|
+
["\x00\x00\xFE\xFF".b, Encoding::UTF_32BE],
|
|
52
|
+
["\xEF\xBB\xBF".b, Encoding::UTF_8],
|
|
53
|
+
["\xFF\xFE".b, Encoding::UTF_16LE],
|
|
54
|
+
["\xFE\xFF".b, Encoding::UTF_16BE]
|
|
55
|
+
].freeze
|
|
56
|
+
|
|
57
|
+
module_function
|
|
58
|
+
|
|
59
|
+
# @param source [String, Pathname, #read] a path, or an object responding to `read(length)`
|
|
60
|
+
# @param config [Configuration] a resolved snapshot
|
|
61
|
+
# @param deadline [Deadline]
|
|
62
|
+
# @return [Result]
|
|
63
|
+
# @raise [UnreadableInput] binary or non-text content
|
|
64
|
+
# @raise [ArgumentError] unsupported source type or a non-regular file
|
|
65
|
+
# @raise [SystemCallError] opening a path failed
|
|
66
|
+
def call(source, config:, deadline:)
|
|
67
|
+
limit = config.max_input_bytes
|
|
68
|
+
if source.is_a?(String) || source.is_a?(Pathname)
|
|
69
|
+
buffer, complete = read_path(source.to_s, limit, deadline)
|
|
70
|
+
elsif source.respond_to?(:read)
|
|
71
|
+
buffer, complete = read_bounded(source, limit, deadline)
|
|
72
|
+
else
|
|
73
|
+
raise ArgumentError, "source must be a path or respond to #read"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
decode(buffer, complete, config.encoding, deadline)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Opened non-blocking so the open cannot wait for a writer (a FIFO), which no timeout could bound;
|
|
80
|
+
# the stat then refuses anything but a regular file, where non-blocking changes nothing.
|
|
81
|
+
def read_path(path, limit, deadline)
|
|
82
|
+
File.open(path, File::RDONLY | OPEN_NONBLOCK, binmode: true) do |file|
|
|
83
|
+
raise ArgumentError, "source path is not a regular file" unless file.stat.file?
|
|
84
|
+
|
|
85
|
+
buffer, eof = read_bounded(file, limit, deadline)
|
|
86
|
+
[buffer, eof || file.size <= limit]
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# @return [Array(String, Boolean)] the binary buffer and whether end of file was seen
|
|
91
|
+
def read_bounded(io, limit, deadline)
|
|
92
|
+
buffer = String.new(capacity: [limit, CHUNK].min, encoding: Encoding::BINARY)
|
|
93
|
+
eof = false
|
|
94
|
+
while buffer.bytesize < limit
|
|
95
|
+
deadline.check!(:read)
|
|
96
|
+
want = [CHUNK, limit - buffer.bytesize].min
|
|
97
|
+
chunk = io.read(want)
|
|
98
|
+
unless chunk.nil? || chunk.is_a?(String)
|
|
99
|
+
raise ArgumentError, "source#read returned #{chunk.class}"
|
|
100
|
+
end
|
|
101
|
+
if chunk.nil? || chunk.empty?
|
|
102
|
+
eof = true
|
|
103
|
+
break
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
chunk = chunk.byteslice(0, want) if chunk.bytesize > want
|
|
107
|
+
buffer << chunk.b
|
|
108
|
+
end
|
|
109
|
+
[buffer, eof]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def decode(buffer, complete, option, deadline)
|
|
113
|
+
bytes_read = buffer.bytesize
|
|
114
|
+
reject_magic!(buffer)
|
|
115
|
+
|
|
116
|
+
if option == :auto
|
|
117
|
+
encoding, body = detect(buffer, complete)
|
|
118
|
+
else
|
|
119
|
+
encoding = Encoding.find(option)
|
|
120
|
+
body = trim_cut(strip_bom(buffer, encoding), encoding, complete)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# ASCII-compatible encodings are checked on the raw bytes before paying for a decode.
|
|
124
|
+
reject_binary_bytes!(buffer) if encoding.ascii_compatible?
|
|
125
|
+
text = to_utf8(body, encoding)
|
|
126
|
+
reject_binary_text!(text) unless encoding.ascii_compatible?
|
|
127
|
+
text = text.byteslice(3, text.bytesize - 3) if text.start_with?("\uFEFF")
|
|
128
|
+
deadline.check!(:decode)
|
|
129
|
+
Result.new(text: text, encoding: encoding.name, complete: complete, bytes_read: bytes_read)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Step 1 of binary detection: magic numbers at offset 0.
|
|
133
|
+
def reject_magic!(buffer)
|
|
134
|
+
MAGIC.each do |label, matcher|
|
|
135
|
+
raise UnreadableInput, "input looks like #{label}, not CSV text. #{NOT_TEXT_HINT}" if matcher.call(buffer)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Step 3 for ASCII-compatible encodings, on the raw bytes.
|
|
140
|
+
def reject_binary_bytes!(buffer)
|
|
141
|
+
window = buffer.byteslice(0, BINARY_WINDOW_BYTES).b
|
|
142
|
+
raise UnreadableInput, "input contains NUL bytes, so it is not CSV text. #{NOT_TEXT_HINT}" if window.include?("\0")
|
|
143
|
+
|
|
144
|
+
reject_control_ratio!(window, window.bytesize)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Step 3 for UTF-16 and UTF-32, on the decoded text (NUL bytes are normal there).
|
|
148
|
+
def reject_binary_text!(text)
|
|
149
|
+
window = text[0, BINARY_WINDOW_CHARS]
|
|
150
|
+
raise UnreadableInput, "input contains NUL characters, so it is not CSV text. #{NOT_TEXT_HINT}" if window.include?("\0")
|
|
151
|
+
|
|
152
|
+
reject_control_ratio!(window, window.length)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def reject_control_ratio!(window, size)
|
|
156
|
+
return if window.count(SUSPICIOUS) <= [8, size / 20].max
|
|
157
|
+
|
|
158
|
+
raise UnreadableInput, "input contains too many control characters to be CSV text. #{NOT_TEXT_HINT}"
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# @return [Array(Encoding, String)] the encoding and the BOM-stripped, cut-trimmed bytes to decode
|
|
162
|
+
def detect(buffer, complete)
|
|
163
|
+
BOMS.each do |bom, encoding|
|
|
164
|
+
return [encoding, trim_cut(buffer.byteslice(bom.bytesize..), encoding, complete)] if buffer.start_with?(bom)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
parity = utf16_by_parity(buffer)
|
|
168
|
+
return [parity, trim_cut(buffer, parity, complete)] if parity
|
|
169
|
+
|
|
170
|
+
detect_utf8_or_windows1252(buffer, complete)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def utf16_by_parity(buffer)
|
|
174
|
+
n = [buffer.bytesize, PARITY_WINDOW_BYTES].min & ~1
|
|
175
|
+
return nil if n.zero?
|
|
176
|
+
|
|
177
|
+
even = odd = 0
|
|
178
|
+
(0...n).step(2) do |i|
|
|
179
|
+
even += 1 if buffer.getbyte(i).zero?
|
|
180
|
+
odd += 1 if buffer.getbyte(i + 1).zero?
|
|
181
|
+
end
|
|
182
|
+
units = n / 2.0
|
|
183
|
+
even_ratio = even / units
|
|
184
|
+
odd_ratio = odd / units
|
|
185
|
+
return Encoding::UTF_16LE if odd_ratio >= 0.30 && even_ratio <= 0.05
|
|
186
|
+
return Encoding::UTF_16BE if even_ratio >= 0.30 && odd_ratio <= 0.05
|
|
187
|
+
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def detect_utf8_or_windows1252(buffer, complete)
|
|
192
|
+
candidate = trim_cut(buffer, Encoding::UTF_8, complete)
|
|
193
|
+
candidate.force_encoding(Encoding::UTF_8)
|
|
194
|
+
return [Encoding::UTF_8, candidate] if candidate.valid_encoding?
|
|
195
|
+
|
|
196
|
+
clean = candidate.scrub("")
|
|
197
|
+
invalid = candidate.bytesize - clean.bytesize
|
|
198
|
+
multibyte = clean.length - clean.count("\x00-\x7F")
|
|
199
|
+
return [Encoding::UTF_8, candidate] if multibyte >= 1 && multibyte >= invalid
|
|
200
|
+
|
|
201
|
+
[Encoding::Windows_1252, buffer.force_encoding(Encoding::BINARY)]
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def strip_bom(buffer, encoding)
|
|
205
|
+
bom = BOMS.find { |_, enc| enc == encoding }&.first
|
|
206
|
+
bom && buffer.start_with?(bom) ? buffer.byteslice(bom.bytesize..) : buffer
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Drops an incomplete trailing character when the byte limit cut the input.
|
|
210
|
+
def trim_cut(body, encoding, complete)
|
|
211
|
+
return body if complete
|
|
212
|
+
|
|
213
|
+
case encoding
|
|
214
|
+
when Encoding::UTF_8 then trim_utf8(body)
|
|
215
|
+
when Encoding::UTF_16LE, Encoding::UTF_16BE then trim_utf16(body, encoding)
|
|
216
|
+
when Encoding::UTF_32LE, Encoding::UTF_32BE then body.byteslice(0, body.bytesize - (body.bytesize % 4))
|
|
217
|
+
else body
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def trim_utf8(body)
|
|
222
|
+
n = body.bytesize
|
|
223
|
+
j = n - 1
|
|
224
|
+
steps = 0
|
|
225
|
+
while steps < 3 && j >= 0 && (body.getbyte(j) & 0xC0) == 0x80
|
|
226
|
+
j -= 1
|
|
227
|
+
steps += 1
|
|
228
|
+
end
|
|
229
|
+
return body if j.negative?
|
|
230
|
+
|
|
231
|
+
lead = body.getbyte(j)
|
|
232
|
+
need = if lead >= 0xF0 && lead <= 0xF7 then 4
|
|
233
|
+
elsif lead >= 0xE0 && lead < 0xF0 then 3
|
|
234
|
+
elsif lead >= 0xC0 && lead < 0xE0 then 2
|
|
235
|
+
else 1
|
|
236
|
+
end
|
|
237
|
+
need > n - j ? body.byteslice(0, j) : body
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def trim_utf16(body, encoding)
|
|
241
|
+
body = body.byteslice(0, body.bytesize - 1) if body.bytesize.odd?
|
|
242
|
+
return body if body.bytesize < 2
|
|
243
|
+
|
|
244
|
+
hi_index = encoding == Encoding::UTF_16LE ? body.bytesize - 1 : body.bytesize - 2
|
|
245
|
+
high = body.getbyte(hi_index)
|
|
246
|
+
high >= 0xD8 && high <= 0xDB ? body.byteslice(0, body.bytesize - 2) : body
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Never raises: invalid and undefined sequences become U+FFFD.
|
|
250
|
+
def to_utf8(body, encoding)
|
|
251
|
+
if encoding == Encoding::UTF_8
|
|
252
|
+
text = body.force_encoding(Encoding::UTF_8)
|
|
253
|
+
text.valid_encoding? ? text : text.scrub(REPLACEMENT)
|
|
254
|
+
else
|
|
255
|
+
body.force_encoding(encoding).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CsvPreview
|
|
4
|
+
# What was read from a source: cleaned header and data cells plus counts. An immutable value object;
|
|
5
|
+
# two samples are equal when every member is equal. Built by {CsvPreview.sample}.
|
|
6
|
+
#
|
|
7
|
+
# Strings are decoded, sanitised and cut to at most `max_cell_chars` grapheme clusters (the last
|
|
8
|
+
# being "…" when the cell was longer), but not yet fitted to column widths. Short rows are padded
|
|
9
|
+
# with "". See docs/spec/README.md section 9.3.
|
|
10
|
+
class Sample
|
|
11
|
+
# @return [Array<String>, nil] header cells, nil when no header row was detected or requested
|
|
12
|
+
attr_reader :headers
|
|
13
|
+
# @return [Array<Array<String>>] up to max_rows data rows, each exactly {#column_count} cells
|
|
14
|
+
attr_reader :rows
|
|
15
|
+
# @return [Integer] the widest of the header and the shown rows, at most max_columns
|
|
16
|
+
attr_reader :column_count
|
|
17
|
+
# @return [Integer] the widest row seen in the bytes read, ignoring trailing blank fields
|
|
18
|
+
attr_reader :total_columns
|
|
19
|
+
# @return [Integer, nil] data rows in the whole file, or nil when it was not fully read
|
|
20
|
+
attr_reader :total_rows
|
|
21
|
+
# @return [Integer] data rows seen within the bytes read (a lower bound when total_rows is nil)
|
|
22
|
+
attr_reader :rows_read
|
|
23
|
+
# @return [Array<Symbol>] :number or :text per kept column
|
|
24
|
+
attr_reader :column_kinds
|
|
25
|
+
# @return [String] the delimiter used
|
|
26
|
+
attr_reader :delimiter
|
|
27
|
+
# @return [String] the source encoding detected or used
|
|
28
|
+
attr_reader :encoding
|
|
29
|
+
|
|
30
|
+
# @api private
|
|
31
|
+
def initialize(headers:, rows:, column_count:, total_columns:, total_rows:, rows_read:, rows_truncated:,
|
|
32
|
+
columns_truncated:, column_kinds:, delimiter:, encoding:, empty:)
|
|
33
|
+
@headers = headers && freeze_strings(headers)
|
|
34
|
+
@rows = rows.map { |row| freeze_strings(row) }.freeze
|
|
35
|
+
@column_count = column_count
|
|
36
|
+
@total_columns = total_columns
|
|
37
|
+
@total_rows = total_rows
|
|
38
|
+
@rows_read = rows_read
|
|
39
|
+
@rows_truncated = rows_truncated ? true : false
|
|
40
|
+
@columns_truncated = columns_truncated ? true : false
|
|
41
|
+
@column_kinds = column_kinds.dup.freeze
|
|
42
|
+
@delimiter = delimiter.dup.freeze
|
|
43
|
+
@encoding = encoding.dup.freeze
|
|
44
|
+
@empty = empty ? true : false
|
|
45
|
+
freeze
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @return [Boolean] whether data rows were left out (or the file was not fully read)
|
|
49
|
+
def rows_truncated? = @rows_truncated
|
|
50
|
+
|
|
51
|
+
# @return [Boolean] whether columns were left out
|
|
52
|
+
def columns_truncated? = @columns_truncated
|
|
53
|
+
|
|
54
|
+
# @return [Boolean] true when the bytes read have no non-blank cells
|
|
55
|
+
def empty? = @empty
|
|
56
|
+
|
|
57
|
+
# @param other [Object]
|
|
58
|
+
# @return [Boolean] whether `other` is a Sample with equal members
|
|
59
|
+
def ==(other)
|
|
60
|
+
other.is_a?(Sample) && state == other.state
|
|
61
|
+
end
|
|
62
|
+
alias eql? ==
|
|
63
|
+
|
|
64
|
+
# @return [Integer] consistent with {#==}, so equal samples can be Hash keys
|
|
65
|
+
def hash = [Sample, *state].hash
|
|
66
|
+
|
|
67
|
+
# @return [String] the header, counts, delimiter and encoding (not the data rows)
|
|
68
|
+
def inspect
|
|
69
|
+
"#<CsvPreview::Sample headers=#{headers.inspect} rows=#{rows.size} column_count=#{column_count} " \
|
|
70
|
+
"total_columns=#{total_columns} rows_read=#{rows_read} total_rows=#{total_rows.inspect} " \
|
|
71
|
+
"delimiter=#{delimiter.inspect} encoding=#{encoding}>"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
protected
|
|
75
|
+
|
|
76
|
+
def state
|
|
77
|
+
[headers, rows, column_count, total_columns, total_rows, rows_read, @rows_truncated, @columns_truncated,
|
|
78
|
+
column_kinds, delimiter, encoding, @empty]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def freeze_strings(cells) = cells.map { |cell| cell.frozen? ? cell : cell.dup.freeze }.freeze
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CsvPreview
|
|
4
|
+
# Turns raw cell text into a single line of literal, displayable text.
|
|
5
|
+
# See docs/spec/parsing-and-safety.md section 5.
|
|
6
|
+
# @api private
|
|
7
|
+
module Sanitizer
|
|
8
|
+
ELLIPSIS = "…"
|
|
9
|
+
BREAKS = /(?:\r\n|[\t\n\v\f\r\u0085\u2028\u2029])+/
|
|
10
|
+
REMOVE = /[\u0000-\u0008\u000E-\u001F\u007F-\u0084\u0086-\u009F\u061C\u200B\u200E\u200F\u202A-\u202E\u2060\u2066-\u2069\uFEFF\uFFF9-\uFFFB]+/
|
|
11
|
+
MARK_RUN = /[\p{Mn}\p{Me}]{9,}/
|
|
12
|
+
EDGE_SPACE = /\A +| +\z/
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# @param text [String] one cell, valid UTF-8
|
|
17
|
+
# @param max_cell_chars [Integer] cap in extended grapheme clusters
|
|
18
|
+
# @param capped [Boolean] true when the raw cell was already cut upstream; forces the trailing ellipsis
|
|
19
|
+
# @return [String] frozen
|
|
20
|
+
def clean(text, max_cell_chars:, capped: false)
|
|
21
|
+
s = text.gsub(BREAKS, " ")
|
|
22
|
+
s.gsub!(REMOVE, "")
|
|
23
|
+
s.gsub!(MARK_RUN) { |run| run[0, 8] }
|
|
24
|
+
s.gsub!(EDGE_SPACE, "")
|
|
25
|
+
clusters = s.each_grapheme_cluster.first(max_cell_chars + 1)
|
|
26
|
+
return s.freeze if clusters.size <= max_cell_chars && !capped
|
|
27
|
+
|
|
28
|
+
(clusters.first(max_cell_chars - 1).join.sub(/ +\z/, "") + ELLIPSIS).freeze
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|