calcsheet 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +9 -0
  3. data/CHANGELOG.md +62 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +921 -0
  6. data/lib/calcsheet/book.rb +164 -0
  7. data/lib/calcsheet/cast.rb +578 -0
  8. data/lib/calcsheet/column.rb +117 -0
  9. data/lib/calcsheet/column_letter.rb +34 -0
  10. data/lib/calcsheet/configuration.rb +186 -0
  11. data/lib/calcsheet/decimal_text.rb +19 -0
  12. data/lib/calcsheet/entry_name.rb +28 -0
  13. data/lib/calcsheet/errors.rb +186 -0
  14. data/lib/calcsheet/excel_serial.rb +65 -0
  15. data/lib/calcsheet/format_detector.rb +109 -0
  16. data/lib/calcsheet/formula_protection.rb +31 -0
  17. data/lib/calcsheet/grid.rb +20 -0
  18. data/lib/calcsheet/header_keys.rb +66 -0
  19. data/lib/calcsheet/limited_inflater.rb +326 -0
  20. data/lib/calcsheet/merge_check.rb +41 -0
  21. data/lib/calcsheet/odf.rb +45 -0
  22. data/lib/calcsheet/option_validation.rb +160 -0
  23. data/lib/calcsheet/options.rb +10 -0
  24. data/lib/calcsheet/raw_row.rb +18 -0
  25. data/lib/calcsheet/raw_types.rb +15 -0
  26. data/lib/calcsheet/readers/csv_dialect.rb +264 -0
  27. data/lib/calcsheet/readers/csv_input.rb +128 -0
  28. data/lib/calcsheet/readers/csv_reader.rb +239 -0
  29. data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
  30. data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
  31. data/lib/calcsheet/readers/ods/count.rb +30 -0
  32. data/lib/calcsheet/readers/ods/cursor.rb +121 -0
  33. data/lib/calcsheet/readers/ods/patterns.rb +70 -0
  34. data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
  35. data/lib/calcsheet/readers/ods/scanner.rb +302 -0
  36. data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
  37. data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
  38. data/lib/calcsheet/readers/ods_reader.rb +71 -0
  39. data/lib/calcsheet/readers/ox_errors.rb +40 -0
  40. data/lib/calcsheet/readers/references.rb +62 -0
  41. data/lib/calcsheet/readers/row_batches.rb +45 -0
  42. data/lib/calcsheet/readers/sax_sheet.rb +108 -0
  43. data/lib/calcsheet/readers/stream_remainder.rb +66 -0
  44. data/lib/calcsheet/readers/tag_scan.rb +119 -0
  45. data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
  46. data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
  47. data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
  48. data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
  49. data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
  50. data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
  51. data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
  52. data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
  53. data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
  54. data/lib/calcsheet/reading.rb +281 -0
  55. data/lib/calcsheet/row.rb +110 -0
  56. data/lib/calcsheet/sample.rb +10 -0
  57. data/lib/calcsheet/schema.rb +140 -0
  58. data/lib/calcsheet/sheet_info.rb +21 -0
  59. data/lib/calcsheet/sheet_names.rb +114 -0
  60. data/lib/calcsheet/sheet_selection.rb +58 -0
  61. data/lib/calcsheet/source.rb +263 -0
  62. data/lib/calcsheet/stream_body.rb +81 -0
  63. data/lib/calcsheet/time_zone.rb +96 -0
  64. data/lib/calcsheet/untrusted.rb +32 -0
  65. data/lib/calcsheet/version.rb +6 -0
  66. data/lib/calcsheet/write_book.rb +202 -0
  67. data/lib/calcsheet/write_sheet.rb +282 -0
  68. data/lib/calcsheet/writers/csv_writer.rb +138 -0
  69. data/lib/calcsheet/writers/ods_writer.rb +437 -0
  70. data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
  71. data/lib/calcsheet/writers/xml_text.rb +35 -0
  72. data/lib/calcsheet/writers/zip_buffered.rb +55 -0
  73. data/lib/calcsheet/zip_container.rb +190 -0
  74. data/lib/calcsheet/zip_directory.rb +202 -0
  75. data/lib/calcsheet/zip_end_record.rb +117 -0
  76. data/lib/calcsheet.rb +295 -0
  77. metadata +200 -0
@@ -0,0 +1,202 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zip"
4
+ require "zlib"
5
+ require_relative "entry_name"
6
+ require_relative "errors"
7
+ require_relative "zip_end_record"
8
+
9
+ module Calcsheet
10
+ # Opens a zip archive with rubyzip once its central directory has been checked, the one door both the
11
+ # format detector and ZipContainer go through.
12
+ #
13
+ # Loading the central directory, rubyzip also reads the local header of every entry, extra field included (up to
14
+ # 64 KiB each), and parses the date of every entry. Entries that all point at one local header with a large extra
15
+ # field would make it read that field again for each of them: 10,000 entries of a 574 KB archive cost 686 MB. So
16
+ # the central directory is read first, from the offset the end records give, and the archive is refused unless
17
+ #
18
+ # * the central directory lies between its offset and the end records;
19
+ # * every entry has a central header, and the ZIP64 fields it needs;
20
+ # * the local header of every entry starts before the central directory, and the local header, its name, its
21
+ # extra field and the declared compressed data of no two entries overlap or reach into the central directory.
22
+ #
23
+ # Then all the local headers rubyzip reads fit in the archive once, and every declared compressed size is real.
24
+ # The entry count is checked against +max_entries+ before anything else is read.
25
+ #
26
+ # Byte layouts (APPNOTE.TXT 4.3.7, 4.3.12, 4.5.3; all little-endian, "v" 16 bits, "V" 32 bits, "Q<" 64 bits):
27
+ #
28
+ # - central header, 46 bytes: 0 signature, 4 version made by, 6 version needed, 8 flags, 10 method, 12 time,
29
+ # 14 date, 16 CRC-32, 20 compressed size (V), 24 uncompressed size (V), 28 name length (v), 30 extra length (v),
30
+ # 32 comment length (v), 34 disk, 36 internal attributes, 38 external attributes, 42 local header offset (V); then
31
+ # the name, the extra field and the comment. #read_entries unpacks it with "@20VVvvv@42V".
32
+ # - local header, 30 bytes: 0 signature, 4 version, 6 flags, 8 method, 10 time, 12 date, 14 CRC-32, 18 compressed
33
+ # size, 22 uncompressed size, 26 name length (v), 28 extra length (v); then the name, the extra field and the
34
+ # data (LOCAL_NAME_LENGTHS).
35
+ # - ZIP64 extra field (id 0x0001): the 64-bit values of only the fields that overflow, in this order: uncompressed
36
+ # size, compressed size, local header offset, then a 32-bit disk number (#zip64_fields).
37
+ #
38
+ # @api private
39
+ module ZipDirectory
40
+ CENTRAL_SIGNATURE = "PK\x01\x02".b.freeze
41
+ LOCAL_SIGNATURE = "PK\x03\x04".b.freeze
42
+ CENTRAL_HEADER_SIZE = 46
43
+ LOCAL_HEADER_SIZE = 30
44
+ # The name and extra field lengths of a local header, at offset 26.
45
+ LOCAL_NAME_LENGTHS = "@26vv"
46
+ # The header id of the ZIP64 extended information extra field.
47
+ ZIP64_EXTRA = 0x0001
48
+
49
+ module_function
50
+
51
+ # @param io [IO] a seekable binary IO, any position
52
+ # @param max_entries [Integer, nil] the most entries; nil for no limit
53
+ # @return [Array(Zip::File, Integer | nil)] the archive and the offset of its central directory (nil when the
54
+ # archive has no end of central directory record, which rubyzip then reports)
55
+ # @raise [ZipBombError] with limit +:max_zip_entries+ for more entries
56
+ # @raise [MalformedFileError] for a damaged archive, the zip library's error as +cause+ when it found the damage
57
+ def open(io, max_entries:)
58
+ directory_offset = check(io, max_entries)
59
+ io.seek(0, IO::SEEK_SET)
60
+ [ load(io), directory_offset ]
61
+ end
62
+
63
+ # @return [Integer, nil] the offset of the checked central directory, where the entries' data must end; nil
64
+ # without an end of central directory record
65
+ # @raise [ZipBombError] for more entries than +max_entries+
66
+ # @raise [MalformedFileError] for a central directory that breaks a rule of the module
67
+ def check(io, max_entries)
68
+ directory = ZipEndRecord.directory(io)
69
+ return if directory.nil?
70
+
71
+ if max_entries && directory.entries > max_entries
72
+ raise ZipBombError.new(limit: :max_zip_entries, max: max_entries, actual: directory.entries)
73
+ end
74
+
75
+ check_extent(directory)
76
+ ranges = read_entries(io, directory)
77
+ check_overlaps(ranges, directory.offset)
78
+ directory.offset
79
+ end
80
+
81
+ # rubyzip raises Zip::Error for the damage it knows, and NoMethodError, TypeError or ArgumentError for records
82
+ # cut short: any of its errors is a damaged archive.
83
+ def load(io)
84
+ Zip::File.open_buffer(io)
85
+ rescue StandardError => error
86
+ raise MalformedFileError, "the zip archive cannot be read: #{error.message}"
87
+ end
88
+
89
+ def check_extent(directory)
90
+ unless directory.offset + directory.size <= directory.end_position
91
+ raise MalformedFileError, "the zip archive is damaged: its central directory (#{directory.size} bytes at offset " \
92
+ "#{directory.offset}) extends past the end of central directory record"
93
+ end
94
+ return if directory.entries * CENTRAL_HEADER_SIZE <= directory.size
95
+
96
+ raise MalformedFileError, "the zip archive is damaged: #{directory.entries} entries cannot fit in a central " \
97
+ "directory of #{directory.size} bytes"
98
+ end
99
+
100
+ # @return [Array<Array(Integer, Integer, Integer)>] the start and end of what each entry occupies before the
101
+ # central directory, and its number
102
+ def read_entries(io, directory)
103
+ io.seek(directory.offset, IO::SEEK_SET)
104
+ data = io.read(directory.size).to_s.b
105
+ position = 0
106
+ names = {}
107
+ Array.new(directory.entries) do |index|
108
+ number = index + 1
109
+ header = data.byteslice(position, CENTRAL_HEADER_SIZE)
110
+ damaged(number) unless header.bytesize == CENTRAL_HEADER_SIZE && header.start_with?(CENTRAL_SIGNATURE)
111
+
112
+ compressed, uncompressed, name_length, extra_length, comment_length, offset = header.unpack("@20VVvvv@42V")
113
+ unique_name(names, data.byteslice(position + CENTRAL_HEADER_SIZE, name_length).to_s, number)
114
+ extra = data.byteslice(position + CENTRAL_HEADER_SIZE + name_length, extra_length).to_s
115
+ position += CENTRAL_HEADER_SIZE + name_length + extra_length + comment_length
116
+ damaged(number) if position > data.bytesize
117
+
118
+ fields = zip64_fields(extra, uncompressed, compressed, offset) or damaged(number)
119
+ occupied(io, number, directory.offset, **fields)
120
+ end
121
+ end
122
+
123
+ # rubyzip keeps the last of two entries of one name, ZipContainer the first of two equal once normalized: the
124
+ # archive would read differently, so it is refused.
125
+ def unique_name(names, name, number)
126
+ normalized = EntryName.normalize(name)
127
+ first = names[normalized]
128
+ names[normalized] = number
129
+ return unless first
130
+
131
+ raise MalformedFileError, "the zip archive is damaged: entries #{first} and #{number} are both named " \
132
+ "#{EntryName.show(normalized)}"
133
+ end
134
+
135
+ # The compressed size and the local header offset, from the ZIP64 extra field when the 32-bit field overflows.
136
+ # The extra field holds a value only for each field that overflows, in field order: an uncompressed size that
137
+ # overflows is taken off first, though only the other two are needed.
138
+ # @return [Hash{Symbol => Integer}, nil] +compressed:+ and +offset:+; nil when the extra field lacks a value it
139
+ # must hold
140
+ def zip64_fields(extra, uncompressed, compressed, offset)
141
+ return { compressed: compressed, offset: offset } unless [ uncompressed, compressed, offset ].include?(ZipEndRecord::OVERFLOW_32)
142
+
143
+ values = zip64_values(extra) or return
144
+ values.shift if uncompressed == ZipEndRecord::OVERFLOW_32
145
+ compressed = values.shift if compressed == ZipEndRecord::OVERFLOW_32
146
+ offset = values.shift if offset == ZipEndRecord::OVERFLOW_32
147
+ { compressed: compressed, offset: offset } unless compressed.nil? || offset.nil?
148
+ end
149
+
150
+ # The extra field is a sequence of blocks, each a 16-bit id and a 16-bit length ("vv") followed by its data; the
151
+ # ZIP64 block may come after others.
152
+ # @return [Array<Integer>, nil] the 64-bit values of the ZIP64 extra field (a trailing 32-bit disk number is not
153
+ # a whole value and drops out)
154
+ def zip64_values(extra)
155
+ position = 0
156
+ while position + 4 <= extra.bytesize
157
+ id, length = extra.unpack("@#{position}vv")
158
+ return extra.byteslice(position + 4, length).to_s.unpack("Q<*") if id == ZIP64_EXTRA
159
+
160
+ position += 4 + length
161
+ end
162
+ nil
163
+ end
164
+
165
+ # @return [Array(Integer, Integer, Integer)] the bytes from the local header to the end of the declared data
166
+ def occupied(io, number, directory_offset, offset:, compressed:)
167
+ unless offset + LOCAL_HEADER_SIZE <= directory_offset
168
+ raise MalformedFileError, "the zip archive is damaged: the local header of entry #{number} is out of range " \
169
+ "(offset #{offset}, central directory at #{directory_offset})"
170
+ end
171
+
172
+ io.seek(offset, IO::SEEK_SET)
173
+ local = io.read(LOCAL_HEADER_SIZE).to_s
174
+ # What rubyzip skips for a missing local header: the entry then fails when it is read.
175
+ name_length, extra_length = local.start_with?(LOCAL_SIGNATURE) ? local.unpack(LOCAL_NAME_LENGTHS) : [ 0, 0 ]
176
+ [ offset, offset + LOCAL_HEADER_SIZE + name_length + extra_length + compressed, number ]
177
+ end
178
+
179
+ def check_overlaps(ranges, directory_offset)
180
+ previous = nil
181
+ ranges.sort_by { |start, _finish, number| [ start, number ] }.each do |range|
182
+ if range[1] > directory_offset
183
+ raise MalformedFileError, "the zip archive is damaged: the data of entry #{range[2]} extends past the end of " \
184
+ "the entries, into the central directory"
185
+ end
186
+ if previous && range[0] < previous[1]
187
+ first, second = [ previous[2], range[2] ].sort
188
+ raise MalformedFileError, "the zip archive is damaged: the local headers of entries #{first} and #{second} overlap"
189
+ end
190
+
191
+ previous = range
192
+ end
193
+ end
194
+
195
+ def damaged(number)
196
+ raise MalformedFileError, "the zip archive is damaged: central directory entry #{number} is damaged"
197
+ end
198
+
199
+ private_class_method :check, :load, :check_extent, :read_entries, :unique_name, :zip64_fields, :zip64_values,
200
+ :occupied, :check_overlaps, :damaged
201
+ end
202
+ end
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "errors"
4
+
5
+ module Calcsheet
6
+ # Reads the number of entries of a zip archive, and where its central directory lies, from its End Of Central
7
+ # Directory record (and the ZIP64 one when a field overflows), without loading the central directory. It finds the
8
+ # records the way rubyzip does (the last signature in rubyzip's window, the ZIP64 record in that window or else
9
+ # at the offset the ZIP64 locator gives), so both agree on the archive.
10
+ #
11
+ # Byte layouts (APPNOTE.TXT 4.3.14-4.3.16; little-endian):
12
+ #
13
+ # - end of central directory record, 22 bytes and a comment: 0 signature (V), 4 disk number (v), 6 disk of the
14
+ # central directory (v), 8 entries on this disk (v), 10 total entries (v), 12 central directory size (V),
15
+ # 16 central directory offset (V), 20 comment length (v): "VvvvvVVv", so record[4] is the entries, record[5] the
16
+ # size, record[6] the offset and record[7] the comment length.
17
+ # - ZIP64 locator, 20 bytes: 0 signature, 4 disk of the ZIP64 record, 8 offset of the ZIP64 record (Q<), 16 disks.
18
+ # - ZIP64 end of central directory record, 56 bytes and more: 0 signature, 4 record size (Q<), 12 version made by,
19
+ # 14 version needed, 16 disk number, 20 disk of the central directory, 24 entries on this disk (Q<), 32 total
20
+ # entries (Q<), 40 central directory size (Q<), 48 central directory offset (Q<): "@32Q<Q<Q<".
21
+ #
22
+ # @api private
23
+ module ZipEndRecord
24
+ EOCD_SIGNATURE = "PK\x05\x06".b.freeze
25
+ ZIP64_LOCATOR_SIGNATURE = "PK\x06\x07".b.freeze
26
+ ZIP64_EOCD_SIGNATURE = "PK\x06\x06".b.freeze
27
+ EOCD_SIZE = 22
28
+ MAX_COMMENT_SIZE = 65_535
29
+ ZIP64_LOCATOR_SIZE = 20
30
+ # rubyzip's MAX_END_OF_CD_SIZE: the comment, the EOCD and the ZIP64 locator.
31
+ WINDOW_SIZE = MAX_COMMENT_SIZE + EOCD_SIZE + ZIP64_LOCATOR_SIZE
32
+ # The fixed part of the ZIP64 record, up to the offset of the central directory.
33
+ ZIP64_EOCD_MIN_SIZE = 56
34
+ # A 16- or 32-bit field that overflows holds all ones; the value is then in the ZIP64 record or extra field.
35
+ OVERFLOW_16 = 0xFFFF
36
+ OVERFLOW_32 = 0xFFFFFFFF
37
+
38
+ # What the end records say.
39
+ #
40
+ # @attr_reader entries [Integer] the number of entries
41
+ # @attr_reader size [Integer] the bytes of the central directory
42
+ # @attr_reader offset [Integer] where the central directory starts
43
+ # @attr_reader end_position [Integer] the position of the record that follows the central directory (the ZIP64
44
+ # record, else the EOCD)
45
+ Directory = Data.define(:entries, :size, :offset, :end_position)
46
+
47
+ module_function
48
+
49
+ # @param io [IO] a seekable binary IO, any position (the position is not restored)
50
+ # @return [Directory, nil] +nil+ when no end of central directory record is found
51
+ # @raise [MalformedFileError] when the record is cut short, or a ZIP64 record is required but cannot be read
52
+ def directory(io)
53
+ io.seek(0, IO::SEEK_END)
54
+ size = io.pos
55
+ tail_start = [ size - WINDOW_SIZE, 0 ].max
56
+ io.seek(tail_start, IO::SEEK_SET)
57
+ tail = io.read.to_s.b
58
+ location = tail.rindex(EOCD_SIGNATURE)
59
+ return nil if location.nil?
60
+
61
+ record = tail.byteslice(location, EOCD_SIZE).unpack("VvvvvVVv")
62
+ signature_in_comment!(tail, location) if record.include?(nil) || record[7] != tail.bytesize - location - EOCD_SIZE
63
+ raise MalformedFileError, "the end of central directory record is cut short" if record.include?(nil)
64
+ return zip64_directory(io, tail, location, tail_start) if overflowing?(record)
65
+
66
+ Directory.new(entries: record[4], size: record[5], offset: record[6], end_position: tail_start + location)
67
+ end
68
+
69
+ # rubyzip reads the last signature in its window as the record, so an archive comment holding one would make it
70
+ # read records the checks of ZipDirectory did not see. When an earlier signature is a record whose comment
71
+ # reaches exactly to the end of the file, that is the case: the archive is refused. (Other bytes after the
72
+ # record, which rubyzip also tolerates, are kept.)
73
+ def signature_in_comment!(tail, location)
74
+ earlier = location
75
+ # rindex from -1 would search from the end again: the search stops at the start of the window.
76
+ while earlier.positive? && (earlier = tail.rindex(EOCD_SIGNATURE, earlier - 1))
77
+ length = tail.byteslice(earlier + EOCD_SIZE - 2, 2).to_s.unpack1("v")
78
+ next unless length == tail.bytesize - earlier - EOCD_SIZE
79
+
80
+ raise MalformedFileError, "the zip archive comment holds an end of central directory signature, which the zip " \
81
+ "library would read as the record; such archives are not supported"
82
+ end
83
+ end
84
+
85
+ # Whether a field of the EOCD holds all ones (the disk numbers and entry counts, the size, the offset), which
86
+ # defers to the ZIP64 record, as rubyzip reads it.
87
+ def overflowing?(record)
88
+ record[1..4].include?(OVERFLOW_16) || record[5] == OVERFLOW_32 || record[6] == OVERFLOW_32
89
+ end
90
+
91
+ # The ZIP64 record is searched in the window first, then read where the locator points.
92
+ def zip64_directory(io, tail, eocd_location, tail_start)
93
+ locator = tail.rindex(ZIP64_LOCATOR_SIGNATURE, eocd_location)
94
+ raise MalformedFileError, "the ZIP64 end of central directory locator is missing" if locator.nil?
95
+
96
+ location = tail.rindex(ZIP64_EOCD_SIGNATURE, locator)
97
+ if location
98
+ record = tail.byteslice(location, ZIP64_EOCD_MIN_SIZE)
99
+ position = tail_start + location
100
+ else
101
+ position = tail.byteslice(locator + 8, 8).to_s.unpack1("Q<")
102
+ io.seek(position, IO::SEEK_SET)
103
+ record = io.read(ZIP64_EOCD_MIN_SIZE)
104
+ end
105
+ unless record && record.bytesize == ZIP64_EOCD_MIN_SIZE
106
+ raise MalformedFileError, "the ZIP64 end of central directory record cannot be read"
107
+ end
108
+
109
+ entries, size, offset = record.unpack("@32Q<Q<Q<")
110
+ Directory.new(entries: entries, size: size, offset: offset, end_position: position)
111
+ rescue SystemCallError, RangeError, TypeError
112
+ raise MalformedFileError, "the ZIP64 end of central directory record cannot be read"
113
+ end
114
+
115
+ private_class_method :signature_in_comment!, :overflowing?, :zip64_directory
116
+ end
117
+ end
data/lib/calcsheet.rb ADDED
@@ -0,0 +1,295 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require_relative "calcsheet/version"
5
+ require_relative "calcsheet/errors"
6
+ require_relative "calcsheet/configuration"
7
+ require_relative "calcsheet/options"
8
+ require_relative "calcsheet/source"
9
+ require_relative "calcsheet/format_detector"
10
+ require_relative "calcsheet/zip_container"
11
+ require_relative "calcsheet/time_zone"
12
+ require_relative "calcsheet/cast"
13
+ require_relative "calcsheet/column_letter"
14
+ require_relative "calcsheet/header_keys"
15
+ require_relative "calcsheet/schema"
16
+ require_relative "calcsheet/sheet_info"
17
+ require_relative "calcsheet/raw_row"
18
+ require_relative "calcsheet/raw_types"
19
+ require_relative "calcsheet/row"
20
+ require_relative "calcsheet/sample"
21
+ require_relative "calcsheet/merge_check"
22
+ require_relative "calcsheet/sheet_selection"
23
+ require_relative "calcsheet/reading"
24
+ require_relative "calcsheet/book"
25
+ require_relative "calcsheet/readers/csv_reader"
26
+ require_relative "calcsheet/readers/xlsx/number_format"
27
+ require_relative "calcsheet/readers/xlsx/workbook_meta"
28
+ require_relative "calcsheet/readers/xlsx/shared_strings"
29
+ require_relative "calcsheet/readers/xlsx_reader"
30
+ require_relative "calcsheet/readers/ods_reader"
31
+ require_relative "calcsheet/column"
32
+ require_relative "calcsheet/sheet_names"
33
+ require_relative "calcsheet/write_sheet"
34
+ require_relative "calcsheet/write_book"
35
+ require_relative "calcsheet/stream_body"
36
+ require_relative "calcsheet/writers/csv_writer"
37
+ require_relative "calcsheet/writers/xml_text"
38
+ require_relative "calcsheet/writers/xlsx_writer"
39
+ require_relative "calcsheet/writers/ods_writer"
40
+
41
+ # Streaming CSV, XLSX and ODS reader and writer.
42
+ #
43
+ # Reading: Calcsheet.open returns a Book. Writing: Calcsheet.write and Calcsheet.stream yield a WriteBook. Options come
44
+ # from Calcsheet.configure and can be overridden per call (see Configuration and the options reference of the README).
45
+ module Calcsheet
46
+ # The formats Calcsheet.write infers from the extension of a path.
47
+ WRITE_EXTENSIONS = { ".csv" => :csv, ".xlsx" => :xlsx, ".ods" => :ods }.freeze
48
+
49
+ class << self
50
+ # The reader backends by format (see Reading for their contract). A mutable registry: an application may add or
51
+ # replace a backend before it opens files (+Calcsheet.backends[:tsv] = MyReader+). Detection only ever answers
52
+ # +:csv+, +:xlsx+ or +:ods+, so a backend under another key is used only with an explicit +format:+.
53
+ # @return [Hash{Symbol => Class}]
54
+ def backends
55
+ @backends ||= { csv: Readers::CsvReader, xlsx: Readers::XlsxReader, ods: Readers::OdsReader }
56
+ end
57
+
58
+ # Opens a spreadsheet for reading. With a block, yields the book, closes it afterwards and returns the
59
+ # block's value; without one, returns the book and the caller must call Book#close.
60
+ #
61
+ # A seekable IO is read from its start whatever its position; a non-seekable one is copied to a Tempfile, except a
62
+ # CSV stream opened with +format: :csv+, which is read directly, once (see Source).
63
+ #
64
+ # @param source [String, Pathname, #read] a path, or an IO (anything responding to +read+); an IO supplied by the
65
+ # caller is never closed
66
+ # @param format [Symbol, String, nil] a key of Calcsheet.backends, which skips detection; nil detects +:csv+,
67
+ # +:xlsx+ or +:ods+ from the content (the extension only breaks a tie)
68
+ # @param options [Hash] Options overrides for this book (any key of Configuration::DEFAULTS)
69
+ # @yieldparam book [Book]
70
+ # @return [Object, Book] the block's value with a block; the open book without one
71
+ # @raise [ArgumentError] an unknown or invalid option, a source that is neither a path nor an IO, or a +format:+
72
+ # that is not a Symbol or a String
73
+ # @raise [SystemCallError] a path that cannot be opened (Errno::ENOENT, Errno::EACCES...)
74
+ # @raise [FileTooLargeError] the source is larger than +max_file_size+ (a CSV stream read directly raises it while
75
+ # it is read instead)
76
+ # @raise [UnsupportedFormatError] an XLS or encrypted (OLE2) file, a zip that is neither XLSX nor ODS, an encrypted
77
+ # or unsupported zip entry, or a +format:+ without a backend
78
+ # @raise [MalformedFileError] a damaged zip archive or workbook part
79
+ # @raise [ZipBombError] the archive has more entries than +max_zip_entries+, or reading its workbook parts
80
+ # exceeds a zip limit
81
+ # @raise [TooManySheetsError] an XLSX workbook with more sheets than +max_sheets+ (an ODS lists its sheets, and
82
+ # raises it, at the first Book#sheets or read)
83
+ def open(source, format: nil, **options)
84
+ options = config.options(**options)
85
+ backend_class = format && backend_for(format)
86
+ input = Source.open(source, options, needs_seek: backend_class.nil? || backend_class.needs_seek?)
87
+ begin
88
+ format = format.nil? ? FormatDetector.detect(input.io, path: input.path, max_zip_entries: options.max_zip_entries) : format.to_sym
89
+ backend_class ||= backend_for(format)
90
+ book = Book.new(backend_class.new(input, options), format: format, options: options, source: input)
91
+ # Exception, not StandardError: an Interrupt or a timeout during detection must also close the Source,
92
+ # which deletes the Tempfile copy of a non-seekable stream (as Source#initialize does).
93
+ rescue Exception # rubocop:disable Lint/RescueException
94
+ input.close
95
+ raise
96
+ end
97
+ return book unless block_given?
98
+
99
+ begin
100
+ yield book
101
+ ensure
102
+ book.close
103
+ end
104
+ end
105
+
106
+ # The writer backends by format (see WriteBook for their contract). A mutable registry, like Calcsheet.backends:
107
+ # Calcsheet.write and Calcsheet.stream take any of its keys as +format:+; only the extensions of WRITE_EXTENSIONS
108
+ # infer one from a path.
109
+ # @return [Hash{Symbol => Class}]
110
+ def writers
111
+ @writers ||= { csv: Writers::CsvWriter, xlsx: Writers::XlsxWriter, ods: Writers::OdsWriter }
112
+ end
113
+
114
+ # Writes a spreadsheet: yields a WriteBook, then completes the file.
115
+ #
116
+ # A path is written in binary mode and closed. An IO (any object responding to +write+ or +<<+, non-seekable ones
117
+ # included) is never closed, and when the block raises it keeps the partial output already written.
118
+ #
119
+ # A path is written to a temporary file beside its target (the file a symlink points to), renamed over it once the
120
+ # block completed: an existing file keeps its content and its permissions until then, and when the block raises
121
+ # the temporary file is deleted. A target that is not a regular file, or in a directory that is not writable, is
122
+ # written directly, and when the block raises it keeps the partial output, as an IO does.
123
+ #
124
+ # @param target [String, Pathname, #write, #<<] a path or an IO
125
+ # @param format [Symbol, String, nil] a key of Calcsheet.writers; nil infers it from the extension of a path
126
+ # (.csv, .xlsx, .ods); required for an IO
127
+ # @param options [Hash] Options overrides for this file (any key of Configuration::DEFAULTS)
128
+ # @yieldparam book [WriteBook]
129
+ # @return [nil]
130
+ # @raise [ArgumentError] a missing block or format, an invalid target, an unknown or invalid option (before
131
+ # anything is written), or an invalid sheet, column or row given to the book
132
+ # @raise [UnsupportedFormatError] no writer for the format
133
+ # @raise [WriteError] a value does not fit its column type, or the book is used on after a sheet block raised
134
+ # @raise [SystemCallError] the path cannot be written (Errno::EACCES, Errno::ENOENT, Errno::ELOOP...)
135
+ def write(target, format: nil, **options, &block)
136
+ raise ArgumentError, "a block is required" unless block
137
+
138
+ path = target.is_a?(String) || target.is_a?(Pathname) ? target.to_s : nil
139
+ io = path.nil? && WriteBook.output(target)
140
+ raise ArgumentError, "target must be a path or respond to write or <<, got #{target.inspect}" if path.nil? && io.nil?
141
+
142
+ format = format.nil? ? write_format(path) : format
143
+ writer_class, options = prepare_writing(format, options)
144
+ return WriteBook.write(io, writer_class, options, &block) if path.nil?
145
+
146
+ write_file(path, writer_class, options, &block)
147
+ end
148
+
149
+ # Builds a Rack body that writes the file while it is iterated: StreamBody#each runs the block and yields
150
+ # chunks of at least 64 KiB. Format and options are checked now, before any byte is produced; what the block
151
+ # raises (WriteError, ArgumentError) propagates from StreamBody#each.
152
+ #
153
+ # @param format [Symbol, String] a key of Calcsheet.writers
154
+ # @param options [Hash] Options overrides for this file (any key of Configuration::DEFAULTS)
155
+ # @yieldparam book [WriteBook] at each StreamBody#each
156
+ # @return [StreamBody]
157
+ # @raise [ArgumentError] a missing block, a format that is not a Symbol or a String, or an unknown or invalid
158
+ # option
159
+ # @raise [UnsupportedFormatError] no writer for the format
160
+ def stream(format:, **options, &block)
161
+ raise ArgumentError, "a block is required" unless block
162
+
163
+ writer_class, options = prepare_writing(format, options)
164
+ StreamBody.new(writer_class, options, block)
165
+ end
166
+
167
+ private
168
+
169
+ def write_format(path)
170
+ raise ArgumentError, "format: is required when writing to an IO" if path.nil?
171
+
172
+ WRITE_EXTENSIONS.fetch(File.extname(path).downcase) do
173
+ raise ArgumentError, "cannot infer the format of #{path.inspect}: pass format: (#{WRITE_EXTENSIONS.values.join(", ")})"
174
+ end
175
+ end
176
+
177
+ def prepare_writing(format, overrides)
178
+ options = config.options(**overrides)
179
+ writer_class = writer_for(format)
180
+ [ writer_class, WriteBook.prepare_options(writer_class, format.to_sym, options, explicit: overrides.keys) ]
181
+ end
182
+
183
+ # Writes a temporary file beside the target (the file a symlink points to) and renames it over the target only
184
+ # when the block completed and the file was closed, so that an existing file stays untouched until then. When the
185
+ # block raises (or exits with break/throw) or closing fails, the temporary file is deleted and the original error
186
+ # propagates. An existing target's permissions are kept; a new file gets the default ones (0666 less the umask).
187
+ #
188
+ # A target that exists and is not a regular file (a FIFO, a device) cannot be replaced, and a directory that is
189
+ # not writable cannot hold the temporary file: then the target is written directly, as an IO would be, and a
190
+ # failed block leaves what was written.
191
+ def write_file(path, writer_class, options, &block)
192
+ target = real_target(path)
193
+ return write_directly(path, writer_class, options, &block) if direct?(target)
194
+ # The rename would replace a file that opening it could not write.
195
+ raise Errno::EACCES, path if File.exist?(target) && !File.writable?(target)
196
+
197
+ temporary = File.join(File.dirname(target), ".calcsheet-#{SecureRandom.hex(8)}.tmp")
198
+ file = begin
199
+ File.open(temporary, File::WRONLY | File::CREAT | File::EXCL, 0o666, binmode: true)
200
+ rescue SystemCallError => error
201
+ raise error.class, path # the user's path, not the temporary one
202
+ end
203
+ completed = false
204
+ begin
205
+ mode = File.stat(target).mode & 0o7777 if File.exist?(target)
206
+ file.chmod(mode) if mode
207
+ WriteBook.write(file, writer_class, options, &block)
208
+ file.close
209
+ File.rename(temporary, target)
210
+ completed = true
211
+ ensure
212
+ discard(file, temporary) unless completed
213
+ end
214
+ nil
215
+ end
216
+
217
+ MAX_SYMLINKS = 40
218
+
219
+ # The file a symlink points to, so that the symlink is kept, also when that file does not exist yet (then its
220
+ # directory must); the path itself otherwise. Paths resolve as the kernel resolves them: File.realpath when the
221
+ # final file exists, and for a dangling link each link's target resolved by #link_target, so that ".." after a
222
+ # symlinked directory leaves the real one. Errors name the path given.
223
+ # @raise [Errno::ELOOP] for a symlink loop
224
+ # @raise [Errno::ENOENT] for a dangling symlink into a missing directory
225
+ def real_target(path)
226
+ return path unless File.symlink?(path)
227
+ return File.realpath(path) if File.exist?(path)
228
+
229
+ link = path
230
+ MAX_SYMLINKS.times do
231
+ target = link_target(link)
232
+ return target unless File.symlink?(target)
233
+ return File.realpath(target) if File.exist?(target)
234
+
235
+ link = target
236
+ end
237
+ raise Errno::ELOOP, path
238
+ rescue Errno::ENOENT, Errno::ENOTDIR
239
+ raise Errno::ENOENT, path
240
+ rescue Errno::ELOOP
241
+ raise Errno::ELOOP, path
242
+ end
243
+
244
+ # The file a symlink names, resolved as the kernel does: its text (relative to the link's physical directory)
245
+ # with the directory part resolved physically, so that a "<symlinked directory>/.." in the text leaves the real
246
+ # directory, where File.expand_path would collapse it lexically. The file itself need not exist; a missing
247
+ # directory is kept as written, and opening the file reports it.
248
+ def link_target(link)
249
+ text = File.readlink(link)
250
+ raw = text.start_with?("/") ? text : File.join(File.realpath(File.dirname(link)), text)
251
+ parent = File.dirname(raw)
252
+ File.exist?(parent) ? File.expand_path(File.basename(raw), File.realpath(parent)) : File.expand_path(raw)
253
+ end
254
+
255
+ def direct?(target)
256
+ (File.exist?(target) && !File.file?(target)) || !File.writable?(File.dirname(target))
257
+ end
258
+
259
+ def write_directly(path, writer_class, options, &block)
260
+ File.open(path, "wb") do |file|
261
+ WriteBook.write(file, writer_class, options, &block)
262
+ end
263
+ nil
264
+ end
265
+
266
+ def discard(file, path)
267
+ begin
268
+ file.close unless file.closed?
269
+ rescue StandardError
270
+ nil # the error that interrupted the write wins
271
+ end
272
+ begin
273
+ File.delete(path)
274
+ rescue SystemCallError
275
+ nil # already gone, or cannot be deleted: the original error still wins
276
+ end
277
+ end
278
+
279
+ def writer_for(format)
280
+ check_format_type(format)
281
+ writers.fetch(format.to_sym) { raise UnsupportedFormatError, "no writer for format #{format.to_sym.inspect}" }
282
+ end
283
+
284
+ def check_format_type(format)
285
+ return if format.is_a?(Symbol) || format.is_a?(String)
286
+
287
+ raise ArgumentError, "format: expected a Symbol or a String, got #{format.inspect}"
288
+ end
289
+
290
+ def backend_for(format)
291
+ check_format_type(format)
292
+ backends.fetch(format.to_sym) { raise UnsupportedFormatError, "no reader for format #{format.to_sym.inspect}" }
293
+ end
294
+ end
295
+ end