csv_preview 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/railtie"
4
+
5
+ module CsvPreview
6
+ # Opt-in automatic registration of {ActiveStoragePreviewer}. Loaded by `lib/csv_preview.rb` only when
7
+ # Rails is present. With `config.csv_preview.register_previewer = true` the previewer is put first in
8
+ # `config.active_storage.previewers`; by default (false) nothing changes.
9
+ class Railtie < Rails::Railtie
10
+ config.csv_preview = ActiveSupport::OrderedOptions.new
11
+ config.csv_preview.register_previewer = false
12
+
13
+ # The flag is read after initializer files have run, hence the after_initialize hook. Active Storage
14
+ # copies the previewer list in a hook registered by its "active_storage.configs" initializer, so
15
+ # declaring this one before it registers this hook first. The list is changed in place, which keeps
16
+ # `ActiveStorage.previewers` and the config value the same Array.
17
+ initializer "csv_preview.register_previewer", before: "active_storage.configs" do |app|
18
+ config.after_initialize do
19
+ if app.config.csv_preview.register_previewer
20
+ if defined?(ActiveStorage::Engine)
21
+ previewers = (app.config.active_storage.previewers ||= [])
22
+ unless previewers.include?(CsvPreview::ActiveStoragePreviewer)
23
+ previewers.unshift(CsvPreview::ActiveStoragePreviewer)
24
+ end
25
+ else
26
+ Rails.logger&.warn("csv_preview: config.csv_preview.register_previewer is true, but Active Storage is " \
27
+ "not loaded; see #{CsvPreview::Help::RAILS}")
28
+ end
29
+ end
30
+
31
+ Railtie.warm_up(app)
32
+ end
33
+ end
34
+
35
+ class << self
36
+ # Pays the one-time font and Pango set-up cost at boot instead of in the first request, but only
37
+ # when the previewer is registered (by either route) and the app has not chosen its own font (an
38
+ # app that always passes `font:` never needs the bundled-font self-check, rendering.md 5.4).
39
+ # @api private
40
+ def warm_up(app)
41
+ return unless defined?(ActiveStorage::Engine)
42
+ # A pending autoload means nothing has referenced the previewer, so it cannot be in the list.
43
+ # Checking that first keeps this gate from loading the previewer in apps that never opted in.
44
+ return if CsvPreview.autoload?(:ActiveStoragePreviewer)
45
+ return unless app.config.active_storage.previewers&.include?(CsvPreview::ActiveStoragePreviewer)
46
+ return unless CsvPreview.config.font.nil? && CsvPreview::Text.supported?
47
+
48
+ CsvPreview::Text.warm_up!
49
+ rescue CsvPreview::Error => e
50
+ Rails.logger&.warn("csv_preview: font warm-up failed (#{e.class}: #{e.message})")
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,259 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pathname"
4
+
5
+ module CsvPreview
6
+ # Turns a source (path or IO-like) into bounded, decoded UTF-8 text. Internal API.
7
+ #
8
+ # See docs/spec/parsing-and-safety.md section 3. Messages of raised errors never include input bytes.
9
+ # @api private
10
+ module Reader
11
+ # @!attribute text [String] valid UTF-8, BOM removed
12
+ # @!attribute encoding [String] canonical Ruby name of the source encoding
13
+ # @!attribute complete [Boolean] the whole source was read within max_input_bytes
14
+ # @!attribute bytes_read [Integer] never above max_input_bytes
15
+ Result = Data.define(:text, :encoding, :complete, :bytes_read)
16
+
17
+ CHUNK = 65_536
18
+ # Open flag for paths (see read_path); 0 where the platform has no O_NONBLOCK.
19
+ OPEN_NONBLOCK = defined?(File::NONBLOCK) ? File::NONBLOCK : 0
20
+ BINARY_WINDOW_BYTES = 8_192
21
+ BINARY_WINDOW_CHARS = 4_096
22
+ PARITY_WINDOW_BYTES = 4_096
23
+ SUSPICIOUS = "\x01-\x08\x0B\x0C\x0E-\x1F\x7F"
24
+ REPLACEMENT = "�"
25
+ # Appended to every UnreadableInput message.
26
+ NOT_TEXT_HINT = "Only CSV and TSV text can be previewed; show a fallback for other files, see #{Help::ERRORS}"
27
+
28
+ MAGIC = [
29
+ ["a PNG image", ->(b) { b.start_with?("\x89PNG\r\n\x1A\n".b) }],
30
+ ["a JPEG image", ->(b) { b.start_with?("\xFF\xD8\xFF".b) }],
31
+ ["a GIF image", ->(b) { b.start_with?("GIF87a".b, "GIF89a".b) }],
32
+ ["a PDF document", ->(b) { b.start_with?("%PDF-".b) }],
33
+ ["a ZIP archive (XLSX, ODS and similar)", ->(b) { b.start_with?("PK\x03\x04".b, "PK\x05\x06".b, "PK\x07\x08".b) }],
34
+ ["an OLE2 file (XLS and similar)", ->(b) { b.start_with?("\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b) }],
35
+ ["gzip data", ->(b) { b.start_with?("\x1F\x8B".b) }],
36
+ ["bzip2 data", ->(b) { b.match?(/\ABZh[1-9]\x31\x41\x59\x26\x53\x59/n) }],
37
+ ["xz data", ->(b) { b.start_with?("\xFD7zXZ\x00".b) }],
38
+ ["zstd data", ->(b) { b.start_with?("\x28\xB5\x2F\xFD".b) }],
39
+ ["a 7z archive", ->(b) { b.start_with?("7z\xBC\xAF\x27\x1C".b) }],
40
+ ["a RAR archive", ->(b) { b.start_with?("Rar!\x1A\x07".b) }],
41
+ ["a RIFF container", ->(b) { b.start_with?("RIFF".b) }],
42
+ ["a TIFF image", ->(b) { b.start_with?("II*\x00".b, "MM\x00*".b) }],
43
+ ["an ISO media file", ->(b) { b.byteslice(4, 4) == "ftyp".b }],
44
+ ["an ELF executable", ->(b) { b.start_with?("\x7FELF".b) }],
45
+ ["a SQLite database", ->(b) { b.start_with?("SQLite format 3\x00".b) }]
46
+ ].freeze
47
+
48
+ # Detection BOMs, UTF-32LE before UTF-16LE because they share FF FE.
49
+ BOMS = [
50
+ ["\xFF\xFE\x00\x00".b, Encoding::UTF_32LE],
51
+ ["\x00\x00\xFE\xFF".b, Encoding::UTF_32BE],
52
+ ["\xEF\xBB\xBF".b, Encoding::UTF_8],
53
+ ["\xFF\xFE".b, Encoding::UTF_16LE],
54
+ ["\xFE\xFF".b, Encoding::UTF_16BE]
55
+ ].freeze
56
+
57
+ module_function
58
+
59
+ # @param source [String, Pathname, #read] a path, or an object responding to `read(length)`
60
+ # @param config [Configuration] a resolved snapshot
61
+ # @param deadline [Deadline]
62
+ # @return [Result]
63
+ # @raise [UnreadableInput] binary or non-text content
64
+ # @raise [ArgumentError] unsupported source type or a non-regular file
65
+ # @raise [SystemCallError] opening a path failed
66
+ def call(source, config:, deadline:)
67
+ limit = config.max_input_bytes
68
+ if source.is_a?(String) || source.is_a?(Pathname)
69
+ buffer, complete = read_path(source.to_s, limit, deadline)
70
+ elsif source.respond_to?(:read)
71
+ buffer, complete = read_bounded(source, limit, deadline)
72
+ else
73
+ raise ArgumentError, "source must be a path or respond to #read"
74
+ end
75
+
76
+ decode(buffer, complete, config.encoding, deadline)
77
+ end
78
+
79
+ # Opened non-blocking so the open cannot wait for a writer (a FIFO), which no timeout could bound;
80
+ # the stat then refuses anything but a regular file, where non-blocking changes nothing.
81
+ def read_path(path, limit, deadline)
82
+ File.open(path, File::RDONLY | OPEN_NONBLOCK, binmode: true) do |file|
83
+ raise ArgumentError, "source path is not a regular file" unless file.stat.file?
84
+
85
+ buffer, eof = read_bounded(file, limit, deadline)
86
+ [buffer, eof || file.size <= limit]
87
+ end
88
+ end
89
+
90
+ # @return [Array(String, Boolean)] the binary buffer and whether end of file was seen
91
+ def read_bounded(io, limit, deadline)
92
+ buffer = String.new(capacity: [limit, CHUNK].min, encoding: Encoding::BINARY)
93
+ eof = false
94
+ while buffer.bytesize < limit
95
+ deadline.check!(:read)
96
+ want = [CHUNK, limit - buffer.bytesize].min
97
+ chunk = io.read(want)
98
+ unless chunk.nil? || chunk.is_a?(String)
99
+ raise ArgumentError, "source#read returned #{chunk.class}"
100
+ end
101
+ if chunk.nil? || chunk.empty?
102
+ eof = true
103
+ break
104
+ end
105
+
106
+ chunk = chunk.byteslice(0, want) if chunk.bytesize > want
107
+ buffer << chunk.b
108
+ end
109
+ [buffer, eof]
110
+ end
111
+
112
+ def decode(buffer, complete, option, deadline)
113
+ bytes_read = buffer.bytesize
114
+ reject_magic!(buffer)
115
+
116
+ if option == :auto
117
+ encoding, body = detect(buffer, complete)
118
+ else
119
+ encoding = Encoding.find(option)
120
+ body = trim_cut(strip_bom(buffer, encoding), encoding, complete)
121
+ end
122
+
123
+ # ASCII-compatible encodings are checked on the raw bytes before paying for a decode.
124
+ reject_binary_bytes!(buffer) if encoding.ascii_compatible?
125
+ text = to_utf8(body, encoding)
126
+ reject_binary_text!(text) unless encoding.ascii_compatible?
127
+ text = text.byteslice(3, text.bytesize - 3) if text.start_with?("\uFEFF")
128
+ deadline.check!(:decode)
129
+ Result.new(text: text, encoding: encoding.name, complete: complete, bytes_read: bytes_read)
130
+ end
131
+
132
+ # Step 1 of binary detection: magic numbers at offset 0.
133
+ def reject_magic!(buffer)
134
+ MAGIC.each do |label, matcher|
135
+ raise UnreadableInput, "input looks like #{label}, not CSV text. #{NOT_TEXT_HINT}" if matcher.call(buffer)
136
+ end
137
+ end
138
+
139
+ # Step 3 for ASCII-compatible encodings, on the raw bytes.
140
+ def reject_binary_bytes!(buffer)
141
+ window = buffer.byteslice(0, BINARY_WINDOW_BYTES).b
142
+ raise UnreadableInput, "input contains NUL bytes, so it is not CSV text. #{NOT_TEXT_HINT}" if window.include?("\0")
143
+
144
+ reject_control_ratio!(window, window.bytesize)
145
+ end
146
+
147
+ # Step 3 for UTF-16 and UTF-32, on the decoded text (NUL bytes are normal there).
148
+ def reject_binary_text!(text)
149
+ window = text[0, BINARY_WINDOW_CHARS]
150
+ raise UnreadableInput, "input contains NUL characters, so it is not CSV text. #{NOT_TEXT_HINT}" if window.include?("\0")
151
+
152
+ reject_control_ratio!(window, window.length)
153
+ end
154
+
155
+ def reject_control_ratio!(window, size)
156
+ return if window.count(SUSPICIOUS) <= [8, size / 20].max
157
+
158
+ raise UnreadableInput, "input contains too many control characters to be CSV text. #{NOT_TEXT_HINT}"
159
+ end
160
+
161
+ # @return [Array(Encoding, String)] the encoding and the BOM-stripped, cut-trimmed bytes to decode
162
+ def detect(buffer, complete)
163
+ BOMS.each do |bom, encoding|
164
+ return [encoding, trim_cut(buffer.byteslice(bom.bytesize..), encoding, complete)] if buffer.start_with?(bom)
165
+ end
166
+
167
+ parity = utf16_by_parity(buffer)
168
+ return [parity, trim_cut(buffer, parity, complete)] if parity
169
+
170
+ detect_utf8_or_windows1252(buffer, complete)
171
+ end
172
+
173
+ def utf16_by_parity(buffer)
174
+ n = [buffer.bytesize, PARITY_WINDOW_BYTES].min & ~1
175
+ return nil if n.zero?
176
+
177
+ even = odd = 0
178
+ (0...n).step(2) do |i|
179
+ even += 1 if buffer.getbyte(i).zero?
180
+ odd += 1 if buffer.getbyte(i + 1).zero?
181
+ end
182
+ units = n / 2.0
183
+ even_ratio = even / units
184
+ odd_ratio = odd / units
185
+ return Encoding::UTF_16LE if odd_ratio >= 0.30 && even_ratio <= 0.05
186
+ return Encoding::UTF_16BE if even_ratio >= 0.30 && odd_ratio <= 0.05
187
+
188
+ nil
189
+ end
190
+
191
+ def detect_utf8_or_windows1252(buffer, complete)
192
+ candidate = trim_cut(buffer, Encoding::UTF_8, complete)
193
+ candidate.force_encoding(Encoding::UTF_8)
194
+ return [Encoding::UTF_8, candidate] if candidate.valid_encoding?
195
+
196
+ clean = candidate.scrub("")
197
+ invalid = candidate.bytesize - clean.bytesize
198
+ multibyte = clean.length - clean.count("\x00-\x7F")
199
+ return [Encoding::UTF_8, candidate] if multibyte >= 1 && multibyte >= invalid
200
+
201
+ [Encoding::Windows_1252, buffer.force_encoding(Encoding::BINARY)]
202
+ end
203
+
204
+ def strip_bom(buffer, encoding)
205
+ bom = BOMS.find { |_, enc| enc == encoding }&.first
206
+ bom && buffer.start_with?(bom) ? buffer.byteslice(bom.bytesize..) : buffer
207
+ end
208
+
209
+ # Drops an incomplete trailing character when the byte limit cut the input.
210
+ def trim_cut(body, encoding, complete)
211
+ return body if complete
212
+
213
+ case encoding
214
+ when Encoding::UTF_8 then trim_utf8(body)
215
+ when Encoding::UTF_16LE, Encoding::UTF_16BE then trim_utf16(body, encoding)
216
+ when Encoding::UTF_32LE, Encoding::UTF_32BE then body.byteslice(0, body.bytesize - (body.bytesize % 4))
217
+ else body
218
+ end
219
+ end
220
+
221
+ def trim_utf8(body)
222
+ n = body.bytesize
223
+ j = n - 1
224
+ steps = 0
225
+ while steps < 3 && j >= 0 && (body.getbyte(j) & 0xC0) == 0x80
226
+ j -= 1
227
+ steps += 1
228
+ end
229
+ return body if j.negative?
230
+
231
+ lead = body.getbyte(j)
232
+ need = if lead >= 0xF0 && lead <= 0xF7 then 4
233
+ elsif lead >= 0xE0 && lead < 0xF0 then 3
234
+ elsif lead >= 0xC0 && lead < 0xE0 then 2
235
+ else 1
236
+ end
237
+ need > n - j ? body.byteslice(0, j) : body
238
+ end
239
+
240
+ def trim_utf16(body, encoding)
241
+ body = body.byteslice(0, body.bytesize - 1) if body.bytesize.odd?
242
+ return body if body.bytesize < 2
243
+
244
+ hi_index = encoding == Encoding::UTF_16LE ? body.bytesize - 1 : body.bytesize - 2
245
+ high = body.getbyte(hi_index)
246
+ high >= 0xD8 && high <= 0xDB ? body.byteslice(0, body.bytesize - 2) : body
247
+ end
248
+
249
+ # Never raises: invalid and undefined sequences become U+FFFD.
250
+ def to_utf8(body, encoding)
251
+ if encoding == Encoding::UTF_8
252
+ text = body.force_encoding(Encoding::UTF_8)
253
+ text.valid_encoding? ? text : text.scrub(REPLACEMENT)
254
+ else
255
+ body.force_encoding(encoding).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
256
+ end
257
+ end
258
+ end
259
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CsvPreview
4
+ # What was read from a source: cleaned header and data cells plus counts. An immutable value object;
5
+ # two samples are equal when every member is equal. Built by {CsvPreview.sample}.
6
+ #
7
+ # Strings are decoded, sanitised and cut to at most `max_cell_chars` grapheme clusters (the last
8
+ # being "…" when the cell was longer), but not yet fitted to column widths. Short rows are padded
9
+ # with "". See docs/spec/README.md section 9.3.
10
+ class Sample
11
+ # @return [Array<String>, nil] header cells, nil when no header row was detected or requested
12
+ attr_reader :headers
13
+ # @return [Array<Array<String>>] up to max_rows data rows, each exactly {#column_count} cells
14
+ attr_reader :rows
15
+ # @return [Integer] the widest of the header and the shown rows, at most max_columns
16
+ attr_reader :column_count
17
+ # @return [Integer] the widest row seen in the bytes read, ignoring trailing blank fields
18
+ attr_reader :total_columns
19
+ # @return [Integer, nil] data rows in the whole file, or nil when it was not fully read
20
+ attr_reader :total_rows
21
+ # @return [Integer] data rows seen within the bytes read (a lower bound when total_rows is nil)
22
+ attr_reader :rows_read
23
+ # @return [Array<Symbol>] :number or :text per kept column
24
+ attr_reader :column_kinds
25
+ # @return [String] the delimiter used
26
+ attr_reader :delimiter
27
+ # @return [String] the source encoding detected or used
28
+ attr_reader :encoding
29
+
30
+ # @api private
31
+ def initialize(headers:, rows:, column_count:, total_columns:, total_rows:, rows_read:, rows_truncated:,
32
+ columns_truncated:, column_kinds:, delimiter:, encoding:, empty:)
33
+ @headers = headers && freeze_strings(headers)
34
+ @rows = rows.map { |row| freeze_strings(row) }.freeze
35
+ @column_count = column_count
36
+ @total_columns = total_columns
37
+ @total_rows = total_rows
38
+ @rows_read = rows_read
39
+ @rows_truncated = rows_truncated ? true : false
40
+ @columns_truncated = columns_truncated ? true : false
41
+ @column_kinds = column_kinds.dup.freeze
42
+ @delimiter = delimiter.dup.freeze
43
+ @encoding = encoding.dup.freeze
44
+ @empty = empty ? true : false
45
+ freeze
46
+ end
47
+
48
+ # @return [Boolean] whether data rows were left out (or the file was not fully read)
49
+ def rows_truncated? = @rows_truncated
50
+
51
+ # @return [Boolean] whether columns were left out
52
+ def columns_truncated? = @columns_truncated
53
+
54
+ # @return [Boolean] true when the bytes read have no non-blank cells
55
+ def empty? = @empty
56
+
57
+ # @param other [Object]
58
+ # @return [Boolean] whether `other` is a Sample with equal members
59
+ def ==(other)
60
+ other.is_a?(Sample) && state == other.state
61
+ end
62
+ alias eql? ==
63
+
64
+ # @return [Integer] consistent with {#==}, so equal samples can be Hash keys
65
+ def hash = [Sample, *state].hash
66
+
67
+ # @return [String] the header, counts, delimiter and encoding (not the data rows)
68
+ def inspect
69
+ "#<CsvPreview::Sample headers=#{headers.inspect} rows=#{rows.size} column_count=#{column_count} " \
70
+ "total_columns=#{total_columns} rows_read=#{rows_read} total_rows=#{total_rows.inspect} " \
71
+ "delimiter=#{delimiter.inspect} encoding=#{encoding}>"
72
+ end
73
+
74
+ protected
75
+
76
+ def state
77
+ [headers, rows, column_count, total_columns, total_rows, rows_read, @rows_truncated, @columns_truncated,
78
+ column_kinds, delimiter, encoding, @empty]
79
+ end
80
+
81
+ private
82
+
83
+ def freeze_strings(cells) = cells.map { |cell| cell.frozen? ? cell : cell.dup.freeze }.freeze
84
+ end
85
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CsvPreview
4
+ # Turns raw cell text into a single line of literal, displayable text.
5
+ # See docs/spec/parsing-and-safety.md section 5.
6
+ # @api private
7
+ module Sanitizer
8
+ ELLIPSIS = "…"
9
+ BREAKS = /(?:\r\n|[\t\n\v\f\r\u0085\u2028\u2029])+/
10
+ REMOVE = /[\u0000-\u0008\u000E-\u001F\u007F-\u0084\u0086-\u009F\u061C\u200B\u200E\u200F\u202A-\u202E\u2060\u2066-\u2069\uFEFF\uFFF9-\uFFFB]+/
11
+ MARK_RUN = /[\p{Mn}\p{Me}]{9,}/
12
+ EDGE_SPACE = /\A +| +\z/
13
+
14
+ module_function
15
+
16
+ # @param text [String] one cell, valid UTF-8
17
+ # @param max_cell_chars [Integer] cap in extended grapheme clusters
18
+ # @param capped [Boolean] true when the raw cell was already cut upstream; forces the trailing ellipsis
19
+ # @return [String] frozen
20
+ def clean(text, max_cell_chars:, capped: false)
21
+ s = text.gsub(BREAKS, " ")
22
+ s.gsub!(REMOVE, "")
23
+ s.gsub!(MARK_RUN) { |run| run[0, 8] }
24
+ s.gsub!(EDGE_SPACE, "")
25
+ clusters = s.each_grapheme_cluster.first(max_cell_chars + 1)
26
+ return s.freeze if clusters.size <= max_cell_chars && !capped
27
+
28
+ (clusters.first(max_cell_chars - 1).join.sub(/ +\z/, "") + ELLIPSIS).freeze
29
+ end
30
+ end
31
+ end