calcsheet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +9 -0
- data/CHANGELOG.md +62 -0
- data/LICENSE.txt +21 -0
- data/README.md +921 -0
- data/lib/calcsheet/book.rb +164 -0
- data/lib/calcsheet/cast.rb +578 -0
- data/lib/calcsheet/column.rb +117 -0
- data/lib/calcsheet/column_letter.rb +34 -0
- data/lib/calcsheet/configuration.rb +186 -0
- data/lib/calcsheet/decimal_text.rb +19 -0
- data/lib/calcsheet/entry_name.rb +28 -0
- data/lib/calcsheet/errors.rb +186 -0
- data/lib/calcsheet/excel_serial.rb +65 -0
- data/lib/calcsheet/format_detector.rb +109 -0
- data/lib/calcsheet/formula_protection.rb +31 -0
- data/lib/calcsheet/grid.rb +20 -0
- data/lib/calcsheet/header_keys.rb +66 -0
- data/lib/calcsheet/limited_inflater.rb +326 -0
- data/lib/calcsheet/merge_check.rb +41 -0
- data/lib/calcsheet/odf.rb +45 -0
- data/lib/calcsheet/option_validation.rb +160 -0
- data/lib/calcsheet/options.rb +10 -0
- data/lib/calcsheet/raw_row.rb +18 -0
- data/lib/calcsheet/raw_types.rb +15 -0
- data/lib/calcsheet/readers/csv_dialect.rb +264 -0
- data/lib/calcsheet/readers/csv_input.rb +128 -0
- data/lib/calcsheet/readers/csv_reader.rb +239 -0
- data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
- data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
- data/lib/calcsheet/readers/ods/count.rb +30 -0
- data/lib/calcsheet/readers/ods/cursor.rb +121 -0
- data/lib/calcsheet/readers/ods/patterns.rb +70 -0
- data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
- data/lib/calcsheet/readers/ods/scanner.rb +302 -0
- data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
- data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
- data/lib/calcsheet/readers/ods_reader.rb +71 -0
- data/lib/calcsheet/readers/ox_errors.rb +40 -0
- data/lib/calcsheet/readers/references.rb +62 -0
- data/lib/calcsheet/readers/row_batches.rb +45 -0
- data/lib/calcsheet/readers/sax_sheet.rb +108 -0
- data/lib/calcsheet/readers/stream_remainder.rb +66 -0
- data/lib/calcsheet/readers/tag_scan.rb +119 -0
- data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
- data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
- data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
- data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
- data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
- data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
- data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
- data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
- data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
- data/lib/calcsheet/reading.rb +281 -0
- data/lib/calcsheet/row.rb +110 -0
- data/lib/calcsheet/sample.rb +10 -0
- data/lib/calcsheet/schema.rb +140 -0
- data/lib/calcsheet/sheet_info.rb +21 -0
- data/lib/calcsheet/sheet_names.rb +114 -0
- data/lib/calcsheet/sheet_selection.rb +58 -0
- data/lib/calcsheet/source.rb +263 -0
- data/lib/calcsheet/stream_body.rb +81 -0
- data/lib/calcsheet/time_zone.rb +96 -0
- data/lib/calcsheet/untrusted.rb +32 -0
- data/lib/calcsheet/version.rb +6 -0
- data/lib/calcsheet/write_book.rb +202 -0
- data/lib/calcsheet/write_sheet.rb +282 -0
- data/lib/calcsheet/writers/csv_writer.rb +138 -0
- data/lib/calcsheet/writers/ods_writer.rb +437 -0
- data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
- data/lib/calcsheet/writers/xml_text.rb +35 -0
- data/lib/calcsheet/writers/zip_buffered.rb +55 -0
- data/lib/calcsheet/zip_container.rb +190 -0
- data/lib/calcsheet/zip_directory.rb +202 -0
- data/lib/calcsheet/zip_end_record.rb +117 -0
- data/lib/calcsheet.rb +295 -0
- metadata +200 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zip"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require_relative "entry_name"
|
|
6
|
+
require_relative "errors"
|
|
7
|
+
require_relative "zip_end_record"
|
|
8
|
+
|
|
9
|
+
module Calcsheet
|
|
10
|
+
# Opens a zip archive with rubyzip once its central directory has been checked, the one door both the
|
|
11
|
+
# format detector and ZipContainer go through.
|
|
12
|
+
#
|
|
13
|
+
# Loading the central directory, rubyzip also reads the local header of every entry, extra field included (up to
|
|
14
|
+
# 64 KiB each), and parses the date of every entry. Entries that all point at one local header with a large extra
|
|
15
|
+
# field would make it read that field again for each of them: 10,000 entries of a 574 KB archive cost 686 MB. So
|
|
16
|
+
# the central directory is read first, from the offset the end records give, and the archive is refused unless
|
|
17
|
+
#
|
|
18
|
+
# * the central directory lies between its offset and the end records;
|
|
19
|
+
# * every entry has a central header, and the ZIP64 fields it needs;
|
|
20
|
+
# * the local header of every entry starts before the central directory, and the local header, its name, its
|
|
21
|
+
# extra field and the declared compressed data of no two entries overlap or reach into the central directory.
|
|
22
|
+
#
|
|
23
|
+
# Then all the local headers rubyzip reads fit in the archive once, and every declared compressed size is real.
|
|
24
|
+
# The entry count is checked against +max_entries+ before anything else is read.
|
|
25
|
+
#
|
|
26
|
+
# Byte layouts (APPNOTE.TXT 4.3.7, 4.3.12, 4.5.3; all little-endian, "v" 16 bits, "V" 32 bits, "Q<" 64 bits):
|
|
27
|
+
#
|
|
28
|
+
# - central header, 46 bytes: 0 signature, 4 version made by, 6 version needed, 8 flags, 10 method, 12 time,
|
|
29
|
+
# 14 date, 16 CRC-32, 20 compressed size (V), 24 uncompressed size (V), 28 name length (v), 30 extra length (v),
|
|
30
|
+
# 32 comment length (v), 34 disk, 36 internal attributes, 38 external attributes, 42 local header offset (V); then
|
|
31
|
+
# the name, the extra field and the comment. #read_entries unpacks it with "@20VVvvv@42V".
|
|
32
|
+
# - local header, 30 bytes: 0 signature, 4 version, 6 flags, 8 method, 10 time, 12 date, 14 CRC-32, 18 compressed
|
|
33
|
+
# size, 22 uncompressed size, 26 name length (v), 28 extra length (v); then the name, the extra field and the
|
|
34
|
+
# data (LOCAL_NAME_LENGTHS).
|
|
35
|
+
# - ZIP64 extra field (id 0x0001): the 64-bit values of only the fields that overflow, in this order: uncompressed
|
|
36
|
+
# size, compressed size, local header offset, then a 32-bit disk number (#zip64_fields).
|
|
37
|
+
#
|
|
38
|
+
# @api private
|
|
39
|
+
module ZipDirectory
|
|
40
|
+
CENTRAL_SIGNATURE = "PK\x01\x02".b.freeze
|
|
41
|
+
LOCAL_SIGNATURE = "PK\x03\x04".b.freeze
|
|
42
|
+
CENTRAL_HEADER_SIZE = 46
|
|
43
|
+
LOCAL_HEADER_SIZE = 30
|
|
44
|
+
# The name and extra field lengths of a local header, at offset 26.
|
|
45
|
+
LOCAL_NAME_LENGTHS = "@26vv"
|
|
46
|
+
# The header id of the ZIP64 extended information extra field.
|
|
47
|
+
ZIP64_EXTRA = 0x0001
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
# @param io [IO] a seekable binary IO, any position
|
|
52
|
+
# @param max_entries [Integer, nil] the most entries; nil for no limit
|
|
53
|
+
# @return [Array(Zip::File, Integer | nil)] the archive and the offset of its central directory (nil when the
|
|
54
|
+
# archive has no end of central directory record, which rubyzip then reports)
|
|
55
|
+
# @raise [ZipBombError] with limit +:max_zip_entries+ for more entries
|
|
56
|
+
# @raise [MalformedFileError] for a damaged archive, the zip library's error as +cause+ when it found the damage
|
|
57
|
+
def open(io, max_entries:)
|
|
58
|
+
directory_offset = check(io, max_entries)
|
|
59
|
+
io.seek(0, IO::SEEK_SET)
|
|
60
|
+
[ load(io), directory_offset ]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @return [Integer, nil] the offset of the checked central directory, where the entries' data must end; nil
|
|
64
|
+
# without an end of central directory record
|
|
65
|
+
# @raise [ZipBombError] for more entries than +max_entries+
|
|
66
|
+
# @raise [MalformedFileError] for a central directory that breaks a rule of the module
|
|
67
|
+
def check(io, max_entries)
|
|
68
|
+
directory = ZipEndRecord.directory(io)
|
|
69
|
+
return if directory.nil?
|
|
70
|
+
|
|
71
|
+
if max_entries && directory.entries > max_entries
|
|
72
|
+
raise ZipBombError.new(limit: :max_zip_entries, max: max_entries, actual: directory.entries)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
check_extent(directory)
|
|
76
|
+
ranges = read_entries(io, directory)
|
|
77
|
+
check_overlaps(ranges, directory.offset)
|
|
78
|
+
directory.offset
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# rubyzip raises Zip::Error for the damage it knows, and NoMethodError, TypeError or ArgumentError for records
|
|
82
|
+
# cut short: any of its errors is a damaged archive.
|
|
83
|
+
def load(io)
|
|
84
|
+
Zip::File.open_buffer(io)
|
|
85
|
+
rescue StandardError => error
|
|
86
|
+
raise MalformedFileError, "the zip archive cannot be read: #{error.message}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def check_extent(directory)
|
|
90
|
+
unless directory.offset + directory.size <= directory.end_position
|
|
91
|
+
raise MalformedFileError, "the zip archive is damaged: its central directory (#{directory.size} bytes at offset " \
|
|
92
|
+
"#{directory.offset}) extends past the end of central directory record"
|
|
93
|
+
end
|
|
94
|
+
return if directory.entries * CENTRAL_HEADER_SIZE <= directory.size
|
|
95
|
+
|
|
96
|
+
raise MalformedFileError, "the zip archive is damaged: #{directory.entries} entries cannot fit in a central " \
|
|
97
|
+
"directory of #{directory.size} bytes"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# @return [Array<Array(Integer, Integer, Integer)>] the start and end of what each entry occupies before the
|
|
101
|
+
# central directory, and its number
|
|
102
|
+
def read_entries(io, directory)
|
|
103
|
+
io.seek(directory.offset, IO::SEEK_SET)
|
|
104
|
+
data = io.read(directory.size).to_s.b
|
|
105
|
+
position = 0
|
|
106
|
+
names = {}
|
|
107
|
+
Array.new(directory.entries) do |index|
|
|
108
|
+
number = index + 1
|
|
109
|
+
header = data.byteslice(position, CENTRAL_HEADER_SIZE)
|
|
110
|
+
damaged(number) unless header.bytesize == CENTRAL_HEADER_SIZE && header.start_with?(CENTRAL_SIGNATURE)
|
|
111
|
+
|
|
112
|
+
compressed, uncompressed, name_length, extra_length, comment_length, offset = header.unpack("@20VVvvv@42V")
|
|
113
|
+
unique_name(names, data.byteslice(position + CENTRAL_HEADER_SIZE, name_length).to_s, number)
|
|
114
|
+
extra = data.byteslice(position + CENTRAL_HEADER_SIZE + name_length, extra_length).to_s
|
|
115
|
+
position += CENTRAL_HEADER_SIZE + name_length + extra_length + comment_length
|
|
116
|
+
damaged(number) if position > data.bytesize
|
|
117
|
+
|
|
118
|
+
fields = zip64_fields(extra, uncompressed, compressed, offset) or damaged(number)
|
|
119
|
+
occupied(io, number, directory.offset, **fields)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# rubyzip keeps the last of two entries of one name, ZipContainer the first of two equal once normalized: the
|
|
124
|
+
# archive would read differently, so it is refused.
|
|
125
|
+
def unique_name(names, name, number)
|
|
126
|
+
normalized = EntryName.normalize(name)
|
|
127
|
+
first = names[normalized]
|
|
128
|
+
names[normalized] = number
|
|
129
|
+
return unless first
|
|
130
|
+
|
|
131
|
+
raise MalformedFileError, "the zip archive is damaged: entries #{first} and #{number} are both named " \
|
|
132
|
+
"#{EntryName.show(normalized)}"
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# The compressed size and the local header offset, from the ZIP64 extra field when the 32-bit field overflows.
|
|
136
|
+
# The extra field holds a value only for each field that overflows, in field order: an uncompressed size that
|
|
137
|
+
# overflows is taken off first, though only the other two are needed.
|
|
138
|
+
# @return [Hash{Symbol => Integer}, nil] +compressed:+ and +offset:+; nil when the extra field lacks a value it
|
|
139
|
+
# must hold
|
|
140
|
+
def zip64_fields(extra, uncompressed, compressed, offset)
|
|
141
|
+
return { compressed: compressed, offset: offset } unless [ uncompressed, compressed, offset ].include?(ZipEndRecord::OVERFLOW_32)
|
|
142
|
+
|
|
143
|
+
values = zip64_values(extra) or return
|
|
144
|
+
values.shift if uncompressed == ZipEndRecord::OVERFLOW_32
|
|
145
|
+
compressed = values.shift if compressed == ZipEndRecord::OVERFLOW_32
|
|
146
|
+
offset = values.shift if offset == ZipEndRecord::OVERFLOW_32
|
|
147
|
+
{ compressed: compressed, offset: offset } unless compressed.nil? || offset.nil?
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The extra field is a sequence of blocks, each a 16-bit id and a 16-bit length ("vv") followed by its data; the
|
|
151
|
+
# ZIP64 block may come after others.
|
|
152
|
+
# @return [Array<Integer>, nil] the 64-bit values of the ZIP64 extra field (a trailing 32-bit disk number is not
|
|
153
|
+
# a whole value and drops out)
|
|
154
|
+
def zip64_values(extra)
|
|
155
|
+
position = 0
|
|
156
|
+
while position + 4 <= extra.bytesize
|
|
157
|
+
id, length = extra.unpack("@#{position}vv")
|
|
158
|
+
return extra.byteslice(position + 4, length).to_s.unpack("Q<*") if id == ZIP64_EXTRA
|
|
159
|
+
|
|
160
|
+
position += 4 + length
|
|
161
|
+
end
|
|
162
|
+
nil
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# @return [Array(Integer, Integer, Integer)] the bytes from the local header to the end of the declared data
|
|
166
|
+
def occupied(io, number, directory_offset, offset:, compressed:)
|
|
167
|
+
unless offset + LOCAL_HEADER_SIZE <= directory_offset
|
|
168
|
+
raise MalformedFileError, "the zip archive is damaged: the local header of entry #{number} is out of range " \
|
|
169
|
+
"(offset #{offset}, central directory at #{directory_offset})"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
io.seek(offset, IO::SEEK_SET)
|
|
173
|
+
local = io.read(LOCAL_HEADER_SIZE).to_s
|
|
174
|
+
# What rubyzip skips for a missing local header: the entry then fails when it is read.
|
|
175
|
+
name_length, extra_length = local.start_with?(LOCAL_SIGNATURE) ? local.unpack(LOCAL_NAME_LENGTHS) : [ 0, 0 ]
|
|
176
|
+
[ offset, offset + LOCAL_HEADER_SIZE + name_length + extra_length + compressed, number ]
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def check_overlaps(ranges, directory_offset)
|
|
180
|
+
previous = nil
|
|
181
|
+
ranges.sort_by { |start, _finish, number| [ start, number ] }.each do |range|
|
|
182
|
+
if range[1] > directory_offset
|
|
183
|
+
raise MalformedFileError, "the zip archive is damaged: the data of entry #{range[2]} extends past the end of " \
|
|
184
|
+
"the entries, into the central directory"
|
|
185
|
+
end
|
|
186
|
+
if previous && range[0] < previous[1]
|
|
187
|
+
first, second = [ previous[2], range[2] ].sort
|
|
188
|
+
raise MalformedFileError, "the zip archive is damaged: the local headers of entries #{first} and #{second} overlap"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
previous = range
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def damaged(number)
|
|
196
|
+
raise MalformedFileError, "the zip archive is damaged: central directory entry #{number} is damaged"
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
private_class_method :check, :load, :check_extent, :read_entries, :unique_name, :zip64_fields, :zip64_values,
|
|
200
|
+
:occupied, :check_overlaps, :damaged
|
|
201
|
+
end
|
|
202
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Calcsheet
|
|
6
|
+
# Reads the number of entries of a zip archive, and where its central directory lies, from its End Of Central
|
|
7
|
+
# Directory record (and the ZIP64 one when a field overflows), without loading the central directory. It finds the
|
|
8
|
+
# records the way rubyzip does (the last signature in rubyzip's window, the ZIP64 record in that window or else
|
|
9
|
+
# at the offset the ZIP64 locator gives), so both agree on the archive.
|
|
10
|
+
#
|
|
11
|
+
# Byte layouts (APPNOTE.TXT 4.3.14-4.3.16; little-endian):
|
|
12
|
+
#
|
|
13
|
+
# - end of central directory record, 22 bytes and a comment: 0 signature (V), 4 disk number (v), 6 disk of the
|
|
14
|
+
# central directory (v), 8 entries on this disk (v), 10 total entries (v), 12 central directory size (V),
|
|
15
|
+
# 16 central directory offset (V), 20 comment length (v): "VvvvvVVv", so record[4] is the entries, record[5] the
|
|
16
|
+
# size, record[6] the offset and record[7] the comment length.
|
|
17
|
+
# - ZIP64 locator, 20 bytes: 0 signature, 4 disk of the ZIP64 record, 8 offset of the ZIP64 record (Q<), 16 disks.
|
|
18
|
+
# - ZIP64 end of central directory record, 56 bytes and more: 0 signature, 4 record size (Q<), 12 version made by,
|
|
19
|
+
# 14 version needed, 16 disk number, 20 disk of the central directory, 24 entries on this disk (Q<), 32 total
|
|
20
|
+
# entries (Q<), 40 central directory size (Q<), 48 central directory offset (Q<): "@32Q<Q<Q<".
|
|
21
|
+
#
|
|
22
|
+
# @api private
|
|
23
|
+
module ZipEndRecord
|
|
24
|
+
EOCD_SIGNATURE = "PK\x05\x06".b.freeze
|
|
25
|
+
ZIP64_LOCATOR_SIGNATURE = "PK\x06\x07".b.freeze
|
|
26
|
+
ZIP64_EOCD_SIGNATURE = "PK\x06\x06".b.freeze
|
|
27
|
+
EOCD_SIZE = 22
|
|
28
|
+
MAX_COMMENT_SIZE = 65_535
|
|
29
|
+
ZIP64_LOCATOR_SIZE = 20
|
|
30
|
+
# rubyzip's MAX_END_OF_CD_SIZE: the comment, the EOCD and the ZIP64 locator.
|
|
31
|
+
WINDOW_SIZE = MAX_COMMENT_SIZE + EOCD_SIZE + ZIP64_LOCATOR_SIZE
|
|
32
|
+
# The fixed part of the ZIP64 record, up to the offset of the central directory.
|
|
33
|
+
ZIP64_EOCD_MIN_SIZE = 56
|
|
34
|
+
# A 16- or 32-bit field that overflows holds all ones; the value is then in the ZIP64 record or extra field.
|
|
35
|
+
OVERFLOW_16 = 0xFFFF
|
|
36
|
+
OVERFLOW_32 = 0xFFFFFFFF
|
|
37
|
+
|
|
38
|
+
# What the end records say.
|
|
39
|
+
#
|
|
40
|
+
# @attr_reader entries [Integer] the number of entries
|
|
41
|
+
# @attr_reader size [Integer] the bytes of the central directory
|
|
42
|
+
# @attr_reader offset [Integer] where the central directory starts
|
|
43
|
+
# @attr_reader end_position [Integer] the position of the record that follows the central directory (the ZIP64
|
|
44
|
+
# record, else the EOCD)
|
|
45
|
+
Directory = Data.define(:entries, :size, :offset, :end_position)
|
|
46
|
+
|
|
47
|
+
module_function
|
|
48
|
+
|
|
49
|
+
# @param io [IO] a seekable binary IO, any position (the position is not restored)
|
|
50
|
+
# @return [Directory, nil] +nil+ when no end of central directory record is found
|
|
51
|
+
# @raise [MalformedFileError] when the record is cut short, or a ZIP64 record is required but cannot be read
|
|
52
|
+
def directory(io)
|
|
53
|
+
io.seek(0, IO::SEEK_END)
|
|
54
|
+
size = io.pos
|
|
55
|
+
tail_start = [ size - WINDOW_SIZE, 0 ].max
|
|
56
|
+
io.seek(tail_start, IO::SEEK_SET)
|
|
57
|
+
tail = io.read.to_s.b
|
|
58
|
+
location = tail.rindex(EOCD_SIGNATURE)
|
|
59
|
+
return nil if location.nil?
|
|
60
|
+
|
|
61
|
+
record = tail.byteslice(location, EOCD_SIZE).unpack("VvvvvVVv")
|
|
62
|
+
signature_in_comment!(tail, location) if record.include?(nil) || record[7] != tail.bytesize - location - EOCD_SIZE
|
|
63
|
+
raise MalformedFileError, "the end of central directory record is cut short" if record.include?(nil)
|
|
64
|
+
return zip64_directory(io, tail, location, tail_start) if overflowing?(record)
|
|
65
|
+
|
|
66
|
+
Directory.new(entries: record[4], size: record[5], offset: record[6], end_position: tail_start + location)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# rubyzip reads the last signature in its window as the record, so an archive comment holding one would make it
|
|
70
|
+
# read records the checks of ZipDirectory did not see. When an earlier signature is a record whose comment
|
|
71
|
+
# reaches exactly to the end of the file, that is the case: the archive is refused. (Other bytes after the
|
|
72
|
+
# record, which rubyzip also tolerates, are kept.)
|
|
73
|
+
def signature_in_comment!(tail, location)
|
|
74
|
+
earlier = location
|
|
75
|
+
# rindex from -1 would search from the end again: the search stops at the start of the window.
|
|
76
|
+
while earlier.positive? && (earlier = tail.rindex(EOCD_SIGNATURE, earlier - 1))
|
|
77
|
+
length = tail.byteslice(earlier + EOCD_SIZE - 2, 2).to_s.unpack1("v")
|
|
78
|
+
next unless length == tail.bytesize - earlier - EOCD_SIZE
|
|
79
|
+
|
|
80
|
+
raise MalformedFileError, "the zip archive comment holds an end of central directory signature, which the zip " \
|
|
81
|
+
"library would read as the record; such archives are not supported"
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Whether a field of the EOCD holds all ones (the disk numbers and entry counts, the size, the offset), which
|
|
86
|
+
# defers to the ZIP64 record, as rubyzip reads it.
|
|
87
|
+
def overflowing?(record)
|
|
88
|
+
record[1..4].include?(OVERFLOW_16) || record[5] == OVERFLOW_32 || record[6] == OVERFLOW_32
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The ZIP64 record is searched in the window first, then read where the locator points.
|
|
92
|
+
def zip64_directory(io, tail, eocd_location, tail_start)
|
|
93
|
+
locator = tail.rindex(ZIP64_LOCATOR_SIGNATURE, eocd_location)
|
|
94
|
+
raise MalformedFileError, "the ZIP64 end of central directory locator is missing" if locator.nil?
|
|
95
|
+
|
|
96
|
+
location = tail.rindex(ZIP64_EOCD_SIGNATURE, locator)
|
|
97
|
+
if location
|
|
98
|
+
record = tail.byteslice(location, ZIP64_EOCD_MIN_SIZE)
|
|
99
|
+
position = tail_start + location
|
|
100
|
+
else
|
|
101
|
+
position = tail.byteslice(locator + 8, 8).to_s.unpack1("Q<")
|
|
102
|
+
io.seek(position, IO::SEEK_SET)
|
|
103
|
+
record = io.read(ZIP64_EOCD_MIN_SIZE)
|
|
104
|
+
end
|
|
105
|
+
unless record && record.bytesize == ZIP64_EOCD_MIN_SIZE
|
|
106
|
+
raise MalformedFileError, "the ZIP64 end of central directory record cannot be read"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
entries, size, offset = record.unpack("@32Q<Q<Q<")
|
|
110
|
+
Directory.new(entries: entries, size: size, offset: offset, end_position: position)
|
|
111
|
+
rescue SystemCallError, RangeError, TypeError
|
|
112
|
+
raise MalformedFileError, "the ZIP64 end of central directory record cannot be read"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
private_class_method :signature_in_comment!, :overflowing?, :zip64_directory
|
|
116
|
+
end
|
|
117
|
+
end
|
data/lib/calcsheet.rb
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require_relative "calcsheet/version"
|
|
5
|
+
require_relative "calcsheet/errors"
|
|
6
|
+
require_relative "calcsheet/configuration"
|
|
7
|
+
require_relative "calcsheet/options"
|
|
8
|
+
require_relative "calcsheet/source"
|
|
9
|
+
require_relative "calcsheet/format_detector"
|
|
10
|
+
require_relative "calcsheet/zip_container"
|
|
11
|
+
require_relative "calcsheet/time_zone"
|
|
12
|
+
require_relative "calcsheet/cast"
|
|
13
|
+
require_relative "calcsheet/column_letter"
|
|
14
|
+
require_relative "calcsheet/header_keys"
|
|
15
|
+
require_relative "calcsheet/schema"
|
|
16
|
+
require_relative "calcsheet/sheet_info"
|
|
17
|
+
require_relative "calcsheet/raw_row"
|
|
18
|
+
require_relative "calcsheet/raw_types"
|
|
19
|
+
require_relative "calcsheet/row"
|
|
20
|
+
require_relative "calcsheet/sample"
|
|
21
|
+
require_relative "calcsheet/merge_check"
|
|
22
|
+
require_relative "calcsheet/sheet_selection"
|
|
23
|
+
require_relative "calcsheet/reading"
|
|
24
|
+
require_relative "calcsheet/book"
|
|
25
|
+
require_relative "calcsheet/readers/csv_reader"
|
|
26
|
+
require_relative "calcsheet/readers/xlsx/number_format"
|
|
27
|
+
require_relative "calcsheet/readers/xlsx/workbook_meta"
|
|
28
|
+
require_relative "calcsheet/readers/xlsx/shared_strings"
|
|
29
|
+
require_relative "calcsheet/readers/xlsx_reader"
|
|
30
|
+
require_relative "calcsheet/readers/ods_reader"
|
|
31
|
+
require_relative "calcsheet/column"
|
|
32
|
+
require_relative "calcsheet/sheet_names"
|
|
33
|
+
require_relative "calcsheet/write_sheet"
|
|
34
|
+
require_relative "calcsheet/write_book"
|
|
35
|
+
require_relative "calcsheet/stream_body"
|
|
36
|
+
require_relative "calcsheet/writers/csv_writer"
|
|
37
|
+
require_relative "calcsheet/writers/xml_text"
|
|
38
|
+
require_relative "calcsheet/writers/xlsx_writer"
|
|
39
|
+
require_relative "calcsheet/writers/ods_writer"
|
|
40
|
+
|
|
41
|
+
# Streaming CSV, XLSX and ODS reader and writer.
|
|
42
|
+
#
|
|
43
|
+
# Reading: Calcsheet.open returns a Book. Writing: Calcsheet.write and Calcsheet.stream yield a WriteBook. Options come
|
|
44
|
+
# from Calcsheet.configure and can be overridden per call (see Configuration and the options reference of the README).
|
|
45
|
+
module Calcsheet
|
|
46
|
+
# The formats Calcsheet.write infers from the extension of a path.
|
|
47
|
+
WRITE_EXTENSIONS = { ".csv" => :csv, ".xlsx" => :xlsx, ".ods" => :ods }.freeze
|
|
48
|
+
|
|
49
|
+
class << self
|
|
50
|
+
# The reader backends by format (see Reading for their contract). A mutable registry: an application may add or
|
|
51
|
+
# replace a backend before it opens files (+Calcsheet.backends[:tsv] = MyReader+). Detection only ever answers
|
|
52
|
+
# +:csv+, +:xlsx+ or +:ods+, so a backend under another key is used only with an explicit +format:+.
|
|
53
|
+
# @return [Hash{Symbol => Class}]
|
|
54
|
+
def backends
|
|
55
|
+
@backends ||= { csv: Readers::CsvReader, xlsx: Readers::XlsxReader, ods: Readers::OdsReader }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Opens a spreadsheet for reading. With a block, yields the book, closes it afterwards and returns the
|
|
59
|
+
# block's value; without one, returns the book and the caller must call Book#close.
|
|
60
|
+
#
|
|
61
|
+
# A seekable IO is read from its start whatever its position; a non-seekable one is copied to a Tempfile, except a
|
|
62
|
+
# CSV stream opened with +format: :csv+, which is read directly, once (see Source).
|
|
63
|
+
#
|
|
64
|
+
# @param source [String, Pathname, #read] a path, or an IO (anything responding to +read+); an IO supplied by the
|
|
65
|
+
# caller is never closed
|
|
66
|
+
# @param format [Symbol, String, nil] a key of Calcsheet.backends, which skips detection; nil detects +:csv+,
|
|
67
|
+
# +:xlsx+ or +:ods+ from the content (the extension only breaks a tie)
|
|
68
|
+
# @param options [Hash] Options overrides for this book (any key of Configuration::DEFAULTS)
|
|
69
|
+
# @yieldparam book [Book]
|
|
70
|
+
# @return [Object, Book] the block's value with a block; the open book without one
|
|
71
|
+
# @raise [ArgumentError] an unknown or invalid option, a source that is neither a path nor an IO, or a +format:+
|
|
72
|
+
# that is not a Symbol or a String
|
|
73
|
+
# @raise [SystemCallError] a path that cannot be opened (Errno::ENOENT, Errno::EACCES...)
|
|
74
|
+
# @raise [FileTooLargeError] the source is larger than +max_file_size+ (a CSV stream read directly raises it while
|
|
75
|
+
# it is read instead)
|
|
76
|
+
# @raise [UnsupportedFormatError] an XLS or encrypted (OLE2) file, a zip that is neither XLSX nor ODS, an encrypted
|
|
77
|
+
# or unsupported zip entry, or a +format:+ without a backend
|
|
78
|
+
# @raise [MalformedFileError] a damaged zip archive or workbook part
|
|
79
|
+
# @raise [ZipBombError] the archive has more entries than +max_zip_entries+, or reading its workbook parts
|
|
80
|
+
# exceeds a zip limit
|
|
81
|
+
# @raise [TooManySheetsError] an XLSX workbook with more sheets than +max_sheets+ (an ODS lists its sheets, and
|
|
82
|
+
# raises it, at the first Book#sheets or read)
|
|
83
|
+
def open(source, format: nil, **options)
|
|
84
|
+
options = config.options(**options)
|
|
85
|
+
backend_class = format && backend_for(format)
|
|
86
|
+
input = Source.open(source, options, needs_seek: backend_class.nil? || backend_class.needs_seek?)
|
|
87
|
+
begin
|
|
88
|
+
format = format.nil? ? FormatDetector.detect(input.io, path: input.path, max_zip_entries: options.max_zip_entries) : format.to_sym
|
|
89
|
+
backend_class ||= backend_for(format)
|
|
90
|
+
book = Book.new(backend_class.new(input, options), format: format, options: options, source: input)
|
|
91
|
+
# Exception, not StandardError: an Interrupt or a timeout during detection must also close the Source,
|
|
92
|
+
# which deletes the Tempfile copy of a non-seekable stream (as Source#initialize does).
|
|
93
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
94
|
+
input.close
|
|
95
|
+
raise
|
|
96
|
+
end
|
|
97
|
+
return book unless block_given?
|
|
98
|
+
|
|
99
|
+
begin
|
|
100
|
+
yield book
|
|
101
|
+
ensure
|
|
102
|
+
book.close
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The writer backends by format (see WriteBook for their contract). A mutable registry, like Calcsheet.backends:
|
|
107
|
+
# Calcsheet.write and Calcsheet.stream take any of its keys as +format:+; only the extensions of WRITE_EXTENSIONS
|
|
108
|
+
# infer one from a path.
|
|
109
|
+
# @return [Hash{Symbol => Class}]
|
|
110
|
+
def writers
|
|
111
|
+
@writers ||= { csv: Writers::CsvWriter, xlsx: Writers::XlsxWriter, ods: Writers::OdsWriter }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Writes a spreadsheet: yields a WriteBook, then completes the file.
|
|
115
|
+
#
|
|
116
|
+
# A path is written in binary mode and closed. An IO (any object responding to +write+ or +<<+, non-seekable ones
|
|
117
|
+
# included) is never closed, and when the block raises it keeps the partial output already written.
|
|
118
|
+
#
|
|
119
|
+
# A path is written to a temporary file beside its target (the file a symlink points to), renamed over it once the
|
|
120
|
+
# block completed: an existing file keeps its content and its permissions until then, and when the block raises
|
|
121
|
+
# the temporary file is deleted. A target that is not a regular file, or in a directory that is not writable, is
|
|
122
|
+
# written directly, and when the block raises it keeps the partial output, as an IO does.
|
|
123
|
+
#
|
|
124
|
+
# @param target [String, Pathname, #write, #<<] a path or an IO
|
|
125
|
+
# @param format [Symbol, String, nil] a key of Calcsheet.writers; nil infers it from the extension of a path
|
|
126
|
+
# (.csv, .xlsx, .ods); required for an IO
|
|
127
|
+
# @param options [Hash] Options overrides for this file (any key of Configuration::DEFAULTS)
|
|
128
|
+
# @yieldparam book [WriteBook]
|
|
129
|
+
# @return [nil]
|
|
130
|
+
# @raise [ArgumentError] a missing block or format, an invalid target, an unknown or invalid option (before
|
|
131
|
+
# anything is written), or an invalid sheet, column or row given to the book
|
|
132
|
+
# @raise [UnsupportedFormatError] no writer for the format
|
|
133
|
+
# @raise [WriteError] a value does not fit its column type, or the book is used on after a sheet block raised
|
|
134
|
+
# @raise [SystemCallError] the path cannot be written (Errno::EACCES, Errno::ENOENT, Errno::ELOOP...)
|
|
135
|
+
def write(target, format: nil, **options, &block)
|
|
136
|
+
raise ArgumentError, "a block is required" unless block
|
|
137
|
+
|
|
138
|
+
path = target.is_a?(String) || target.is_a?(Pathname) ? target.to_s : nil
|
|
139
|
+
io = path.nil? && WriteBook.output(target)
|
|
140
|
+
raise ArgumentError, "target must be a path or respond to write or <<, got #{target.inspect}" if path.nil? && io.nil?
|
|
141
|
+
|
|
142
|
+
format = format.nil? ? write_format(path) : format
|
|
143
|
+
writer_class, options = prepare_writing(format, options)
|
|
144
|
+
return WriteBook.write(io, writer_class, options, &block) if path.nil?
|
|
145
|
+
|
|
146
|
+
write_file(path, writer_class, options, &block)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Builds a Rack body that writes the file while it is iterated: StreamBody#each runs the block and yields
|
|
150
|
+
# chunks of at least 64 KiB. Format and options are checked now, before any byte is produced; what the block
|
|
151
|
+
# raises (WriteError, ArgumentError) propagates from StreamBody#each.
|
|
152
|
+
#
|
|
153
|
+
# @param format [Symbol, String] a key of Calcsheet.writers
|
|
154
|
+
# @param options [Hash] Options overrides for this file (any key of Configuration::DEFAULTS)
|
|
155
|
+
# @yieldparam book [WriteBook] at each StreamBody#each
|
|
156
|
+
# @return [StreamBody]
|
|
157
|
+
# @raise [ArgumentError] a missing block, a format that is not a Symbol or a String, or an unknown or invalid
|
|
158
|
+
# option
|
|
159
|
+
# @raise [UnsupportedFormatError] no writer for the format
|
|
160
|
+
def stream(format:, **options, &block)
|
|
161
|
+
raise ArgumentError, "a block is required" unless block
|
|
162
|
+
|
|
163
|
+
writer_class, options = prepare_writing(format, options)
|
|
164
|
+
StreamBody.new(writer_class, options, block)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
private
|
|
168
|
+
|
|
169
|
+
def write_format(path)
|
|
170
|
+
raise ArgumentError, "format: is required when writing to an IO" if path.nil?
|
|
171
|
+
|
|
172
|
+
WRITE_EXTENSIONS.fetch(File.extname(path).downcase) do
|
|
173
|
+
raise ArgumentError, "cannot infer the format of #{path.inspect}: pass format: (#{WRITE_EXTENSIONS.values.join(", ")})"
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def prepare_writing(format, overrides)
|
|
178
|
+
options = config.options(**overrides)
|
|
179
|
+
writer_class = writer_for(format)
|
|
180
|
+
[ writer_class, WriteBook.prepare_options(writer_class, format.to_sym, options, explicit: overrides.keys) ]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Writes a temporary file beside the target (the file a symlink points to) and renames it over the target only
|
|
184
|
+
# when the block completed and the file was closed, so that an existing file stays untouched until then. When the
|
|
185
|
+
# block raises (or exits with break/throw) or closing fails, the temporary file is deleted and the original error
|
|
186
|
+
# propagates. An existing target's permissions are kept; a new file gets the default ones (0666 less the umask).
|
|
187
|
+
#
|
|
188
|
+
# A target that exists and is not a regular file (a FIFO, a device) cannot be replaced, and a directory that is
|
|
189
|
+
# not writable cannot hold the temporary file: then the target is written directly, as an IO would be, and a
|
|
190
|
+
# failed block leaves what was written.
|
|
191
|
+
def write_file(path, writer_class, options, &block)
|
|
192
|
+
target = real_target(path)
|
|
193
|
+
return write_directly(path, writer_class, options, &block) if direct?(target)
|
|
194
|
+
# The rename would replace a file that opening it could not write.
|
|
195
|
+
raise Errno::EACCES, path if File.exist?(target) && !File.writable?(target)
|
|
196
|
+
|
|
197
|
+
temporary = File.join(File.dirname(target), ".calcsheet-#{SecureRandom.hex(8)}.tmp")
|
|
198
|
+
file = begin
|
|
199
|
+
File.open(temporary, File::WRONLY | File::CREAT | File::EXCL, 0o666, binmode: true)
|
|
200
|
+
rescue SystemCallError => error
|
|
201
|
+
raise error.class, path # the user's path, not the temporary one
|
|
202
|
+
end
|
|
203
|
+
completed = false
|
|
204
|
+
begin
|
|
205
|
+
mode = File.stat(target).mode & 0o7777 if File.exist?(target)
|
|
206
|
+
file.chmod(mode) if mode
|
|
207
|
+
WriteBook.write(file, writer_class, options, &block)
|
|
208
|
+
file.close
|
|
209
|
+
File.rename(temporary, target)
|
|
210
|
+
completed = true
|
|
211
|
+
ensure
|
|
212
|
+
discard(file, temporary) unless completed
|
|
213
|
+
end
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
MAX_SYMLINKS = 40
|
|
218
|
+
|
|
219
|
+
# The file a symlink points to, so that the symlink is kept, also when that file does not exist yet (then its
|
|
220
|
+
# directory must); the path itself otherwise. Paths resolve as the kernel resolves them: File.realpath when the
|
|
221
|
+
# final file exists, and for a dangling link each link's target resolved by #link_target, so that ".." after a
|
|
222
|
+
# symlinked directory leaves the real one. Errors name the path given.
|
|
223
|
+
# @raise [Errno::ELOOP] for a symlink loop
|
|
224
|
+
# @raise [Errno::ENOENT] for a dangling symlink into a missing directory
|
|
225
|
+
def real_target(path)
|
|
226
|
+
return path unless File.symlink?(path)
|
|
227
|
+
return File.realpath(path) if File.exist?(path)
|
|
228
|
+
|
|
229
|
+
link = path
|
|
230
|
+
MAX_SYMLINKS.times do
|
|
231
|
+
target = link_target(link)
|
|
232
|
+
return target unless File.symlink?(target)
|
|
233
|
+
return File.realpath(target) if File.exist?(target)
|
|
234
|
+
|
|
235
|
+
link = target
|
|
236
|
+
end
|
|
237
|
+
raise Errno::ELOOP, path
|
|
238
|
+
rescue Errno::ENOENT, Errno::ENOTDIR
|
|
239
|
+
raise Errno::ENOENT, path
|
|
240
|
+
rescue Errno::ELOOP
|
|
241
|
+
raise Errno::ELOOP, path
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The file a symlink names, resolved as the kernel does: its text (relative to the link's physical directory)
|
|
245
|
+
# with the directory part resolved physically, so that a "<symlinked directory>/.." in the text leaves the real
|
|
246
|
+
# directory, where File.expand_path would collapse it lexically. The file itself need not exist; a missing
|
|
247
|
+
# directory is kept as written, and opening the file reports it.
|
|
248
|
+
def link_target(link)
|
|
249
|
+
text = File.readlink(link)
|
|
250
|
+
raw = text.start_with?("/") ? text : File.join(File.realpath(File.dirname(link)), text)
|
|
251
|
+
parent = File.dirname(raw)
|
|
252
|
+
File.exist?(parent) ? File.expand_path(File.basename(raw), File.realpath(parent)) : File.expand_path(raw)
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def direct?(target)
|
|
256
|
+
(File.exist?(target) && !File.file?(target)) || !File.writable?(File.dirname(target))
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def write_directly(path, writer_class, options, &block)
|
|
260
|
+
File.open(path, "wb") do |file|
|
|
261
|
+
WriteBook.write(file, writer_class, options, &block)
|
|
262
|
+
end
|
|
263
|
+
nil
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def discard(file, path)
|
|
267
|
+
begin
|
|
268
|
+
file.close unless file.closed?
|
|
269
|
+
rescue StandardError
|
|
270
|
+
nil # the error that interrupted the write wins
|
|
271
|
+
end
|
|
272
|
+
begin
|
|
273
|
+
File.delete(path)
|
|
274
|
+
rescue SystemCallError
|
|
275
|
+
nil # already gone, or cannot be deleted: the original error still wins
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def writer_for(format)
|
|
280
|
+
check_format_type(format)
|
|
281
|
+
writers.fetch(format.to_sym) { raise UnsupportedFormatError, "no writer for format #{format.to_sym.inspect}" }
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def check_format_type(format)
|
|
285
|
+
return if format.is_a?(Symbol) || format.is_a?(String)
|
|
286
|
+
|
|
287
|
+
raise ArgumentError, "format: expected a Symbol or a String, got #{format.inspect}"
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def backend_for(format)
|
|
291
|
+
check_format_type(format)
|
|
292
|
+
backends.fetch(format.to_sym) { raise UnsupportedFormatError, "no reader for format #{format.to_sym.inspect}" }
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
end
|