calcsheet 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +9 -0
  3. data/CHANGELOG.md +62 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +921 -0
  6. data/lib/calcsheet/book.rb +164 -0
  7. data/lib/calcsheet/cast.rb +578 -0
  8. data/lib/calcsheet/column.rb +117 -0
  9. data/lib/calcsheet/column_letter.rb +34 -0
  10. data/lib/calcsheet/configuration.rb +186 -0
  11. data/lib/calcsheet/decimal_text.rb +19 -0
  12. data/lib/calcsheet/entry_name.rb +28 -0
  13. data/lib/calcsheet/errors.rb +186 -0
  14. data/lib/calcsheet/excel_serial.rb +65 -0
  15. data/lib/calcsheet/format_detector.rb +109 -0
  16. data/lib/calcsheet/formula_protection.rb +31 -0
  17. data/lib/calcsheet/grid.rb +20 -0
  18. data/lib/calcsheet/header_keys.rb +66 -0
  19. data/lib/calcsheet/limited_inflater.rb +326 -0
  20. data/lib/calcsheet/merge_check.rb +41 -0
  21. data/lib/calcsheet/odf.rb +45 -0
  22. data/lib/calcsheet/option_validation.rb +160 -0
  23. data/lib/calcsheet/options.rb +10 -0
  24. data/lib/calcsheet/raw_row.rb +18 -0
  25. data/lib/calcsheet/raw_types.rb +15 -0
  26. data/lib/calcsheet/readers/csv_dialect.rb +264 -0
  27. data/lib/calcsheet/readers/csv_input.rb +128 -0
  28. data/lib/calcsheet/readers/csv_reader.rb +239 -0
  29. data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
  30. data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
  31. data/lib/calcsheet/readers/ods/count.rb +30 -0
  32. data/lib/calcsheet/readers/ods/cursor.rb +121 -0
  33. data/lib/calcsheet/readers/ods/patterns.rb +70 -0
  34. data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
  35. data/lib/calcsheet/readers/ods/scanner.rb +302 -0
  36. data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
  37. data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
  38. data/lib/calcsheet/readers/ods_reader.rb +71 -0
  39. data/lib/calcsheet/readers/ox_errors.rb +40 -0
  40. data/lib/calcsheet/readers/references.rb +62 -0
  41. data/lib/calcsheet/readers/row_batches.rb +45 -0
  42. data/lib/calcsheet/readers/sax_sheet.rb +108 -0
  43. data/lib/calcsheet/readers/stream_remainder.rb +66 -0
  44. data/lib/calcsheet/readers/tag_scan.rb +119 -0
  45. data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
  46. data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
  47. data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
  48. data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
  49. data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
  50. data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
  51. data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
  52. data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
  53. data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
  54. data/lib/calcsheet/reading.rb +281 -0
  55. data/lib/calcsheet/row.rb +110 -0
  56. data/lib/calcsheet/sample.rb +10 -0
  57. data/lib/calcsheet/schema.rb +140 -0
  58. data/lib/calcsheet/sheet_info.rb +21 -0
  59. data/lib/calcsheet/sheet_names.rb +114 -0
  60. data/lib/calcsheet/sheet_selection.rb +58 -0
  61. data/lib/calcsheet/source.rb +263 -0
  62. data/lib/calcsheet/stream_body.rb +81 -0
  63. data/lib/calcsheet/time_zone.rb +96 -0
  64. data/lib/calcsheet/untrusted.rb +32 -0
  65. data/lib/calcsheet/version.rb +6 -0
  66. data/lib/calcsheet/write_book.rb +202 -0
  67. data/lib/calcsheet/write_sheet.rb +282 -0
  68. data/lib/calcsheet/writers/csv_writer.rb +138 -0
  69. data/lib/calcsheet/writers/ods_writer.rb +437 -0
  70. data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
  71. data/lib/calcsheet/writers/xml_text.rb +35 -0
  72. data/lib/calcsheet/writers/zip_buffered.rb +55 -0
  73. data/lib/calcsheet/zip_container.rb +190 -0
  74. data/lib/calcsheet/zip_directory.rb +202 -0
  75. data/lib/calcsheet/zip_end_record.rb +117 -0
  76. data/lib/calcsheet.rb +295 -0
  77. metadata +200 -0
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zip"
4
+ require "zlib"
5
+ require_relative "entry_name"
6
+ require_relative "errors"
7
+ require_relative "odf"
8
+ require_relative "zip_directory"
9
+
10
+ module Calcsheet
11
+ # Chooses the backend for a file from its content. The extension only breaks
12
+ # a tie between formats the content cannot tell apart.
13
+ #
14
+ # @api private
15
+ module FormatDetector
16
+ # The formats detection answers, and an explicit +format:+ may name.
17
+ FORMATS = %i[csv xlsx ods].freeze
18
+ # The first bytes of an OLE2 compound file: XLS, and encrypted XLSX.
19
+ OLE2_SIGNATURE = "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b.freeze
20
+ # A zip archive starts with a local header, or with the end record when it is empty.
21
+ ZIP_SIGNATURES = [ ZipDirectory::LOCAL_SIGNATURE, ZipEndRecord::EOCD_SIGNATURE ].freeze
22
+ # The entry that makes a zip an XLSX workbook.
23
+ XLSX_ENTRY = "xl/workbook.xml"
24
+ # The extensions that break a tie between XLSX and ODS.
25
+ EXTENSIONS = { ".xlsx" => :xlsx, ".xlsm" => :xlsx, ".ods" => :ods }.freeze
26
+
27
+ module_function
28
+
29
+ # Detects the format of a seekable IO and rewinds it. Content that is neither OLE2 nor a zip archive is CSV,
30
+ # whatever it holds: only an unknown explicit +format+ raises for itself. Only the zip central directory and the
31
+ # start of the +mimetype+ entry are read, never a whole entry; the entry count is checked against
32
+ # +max_zip_entries+ from the end of central directory record, and the central directory by ZipDirectory, before
33
+ # rubyzip loads it.
34
+ #
35
+ # @param io [IO] a seekable binary IO, any position
36
+ # @param path [String, nil] the file name, used only to break a tie
37
+ # @param format [Symbol, String, nil] an explicit format, which wins over the content (Calcsheet.open passes none:
38
+ # it skips detection itself)
39
+ # @param max_zip_entries [Integer, nil] the most entries a zip may have; nil for no limit
40
+ # @return [Symbol] +:csv+, +:xlsx+ or +:ods+
41
+ # @raise [UnsupportedFormatError] for an OLE2 file (XLS, an encrypted workbook), a zip that is neither XLSX nor ODS
42
+ # or both without a telling extension, and an explicit +format+ not in FORMATS
43
+ # @raise [MalformedFileError] for a zip file whose central directory or mimetype entry cannot be read
44
+ # @raise [ZipBombError] with limit +:max_zip_entries+ when the zip has more entries
45
+ def detect(io, path: nil, format: nil, max_zip_entries: nil)
46
+ return explicit(format) unless format.nil?
47
+
48
+ io.seek(0, IO::SEEK_SET)
49
+ head = io.read(OLE2_SIGNATURE.bytesize)&.b.to_s
50
+ return raise_xls if head == OLE2_SIGNATURE
51
+ return :csv unless ZIP_SIGNATURES.any? { head.start_with?(_1) }
52
+
53
+ detect_zip(io, path, max_zip_entries)
54
+ ensure
55
+ io.seek(0, IO::SEEK_SET) if format.nil?
56
+ end
57
+
58
+ def explicit(format)
59
+ name = format.to_sym
60
+ return name if FORMATS.include?(name)
61
+
62
+ raise UnsupportedFormatError, "unsupported format #{name.inspect}; use :csv, :xlsx or :ods"
63
+ end
64
+
65
+ def raise_xls
66
+ raise UnsupportedFormatError,
67
+ "XLS (Excel 97-2003) files and encrypted or password-protected workbooks (OLE2) are not supported; " \
68
+ "save the file without a password as XLSX, ODS or CSV"
69
+ end
70
+
71
+ def detect_zip(io, path, max_entries)
72
+ archive, = ZipDirectory.open(io, max_entries: max_entries)
73
+ # Entries by the name ZipContainer finds them by; ZipDirectory refused names that are equal once normalized.
74
+ files = archive.entries.reject(&:directory?).to_h { |entry| [ EntryName.normalize(entry.name), entry ] }
75
+ candidates = []
76
+ candidates << :xlsx if files.key?(XLSX_ENTRY)
77
+ candidates << :ods if ods_mimetype?(files[Odf::MIMETYPE_ENTRY])
78
+ return candidates.first if candidates.size == 1
79
+
80
+ by_extension = EXTENSIONS[File.extname(path.to_s).downcase]
81
+ return by_extension if candidates.size > 1 && candidates.include?(by_extension)
82
+
83
+ raise UnsupportedFormatError, unsupported_zip_message(candidates)
84
+ rescue Calcsheet::Error
85
+ raise
86
+ rescue StandardError => error
87
+ # Any error of rubyzip reading the mimetype entry is a damaged archive.
88
+ raise MalformedFileError, "the zip archive cannot be read: #{error.message}"
89
+ end
90
+
91
+ def ods_mimetype?(entry)
92
+ return false unless entry
93
+
94
+ # Two bytes more than the media type: a trailing line break (CR LF) is tolerated, as strip removes it, and a
95
+ # longer content still compares unequal, without reading the whole entry.
96
+ entry.get_input_stream { |stream| stream.read(Odf::MIMETYPE.bytesize + 2).to_s.strip } == Odf::MIMETYPE
97
+ end
98
+
99
+ def unsupported_zip_message(candidates)
100
+ if candidates.empty?
101
+ "the zip archive is neither an XLSX workbook nor an ODS spreadsheet"
102
+ else
103
+ "the zip archive looks like both XLSX and ODS and the file name does not say which"
104
+ end
105
+ end
106
+
107
+ private_class_method :explicit, :raise_xls, :detect_zip, :ods_mimetype?, :unsupported_zip_message
108
+ end
109
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ # CSV formula protection: a spreadsheet opening a CSV evaluates a field that starts with "=", "+", "-", "@", a
5
+ # tab or a carriage return as a formula, so the CSV writer puts a "'" before such text (+formula_protection+), and
6
+ # the CSV reader removes that "'" again with +strip_formula_prefix: true+.
7
+ #
8
+ # @api private
9
+ module FormulaProtection
10
+ # The first characters that make a spreadsheet evaluate a field (OWASP CSV injection).
11
+ TRIGGERS = [ "=", "+", "-", "@", "\t", "\r" ].freeze
12
+ # A text that starts with a trigger.
13
+ START = Regexp.new("\\A[#{Regexp.escape(TRIGGERS.join)}]")
14
+
15
+ module_function
16
+
17
+ # @param text [String]
18
+ # @return [String] +text+ with a "'" before it when it starts with a trigger; +text+ itself otherwise
19
+ def protect(text)
20
+ START.match?(text) ? "'#{text}" : text
21
+ end
22
+
23
+ # @param text [String]
24
+ # @return [String] +text+ without the "'" #protect adds; +text+ itself otherwise
25
+ def unprotect(text)
26
+ return text unless text.start_with?("'") && TRIGGERS.include?(text[1])
27
+
28
+ text.byteslice(1, text.bytesize - 1)
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ # The size of a worksheet, the same in Excel (since 2007) and LibreOffice: the readers refuse content beyond it,
5
+ # the writers continue in a new sheet at MAX_ROWS and refuse more than MAX_COLUMNS columns.
6
+ #
7
+ # @api private
8
+ module Grid
9
+ # The rows of a sheet, 2**20.
10
+ MAX_ROWS = 1_048_576
11
+ # The columns of a sheet, 2**14: A to XFD.
12
+ MAX_COLUMNS = 16_384
13
+ # The 0-based index of the last column, and its letters.
14
+ LAST_COLUMN_INDEX = MAX_COLUMNS - 1
15
+ LAST_COLUMN = "XFD"
16
+ # The characters of an Excel cell, counted as UTF-16 code units: a character beyond the Basic Multilingual Plane
17
+ # counts twice.
18
+ MAX_CELL_CHARACTERS = 32_767
19
+ end
20
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bigdecimal"
4
+ require_relative "column_letter"
5
+ require_relative "decimal_text"
6
+
7
+ module Calcsheet
8
+ # Header texts as names: the matching rule of the column schema and the keys of rows read without a schema.
9
+ #
10
+ # @api private
11
+ module HeaderKeys
12
+ # One whitespace character at a time: a run matched as /[[:space:]]+/ costs memory in proportion to its length.
13
+ SPACE = /[[:space:]]/
14
+ # A character that is not whitespace, searched for from either end by #strip.
15
+ NON_SPACE = /[^[:space:]]/
16
+
17
+ module_function
18
+
19
+ # The rule shared by every comparison of header names.
20
+ # @return [String] downcased, with all whitespace removed; nil gives ""
21
+ def normalize(value)
22
+ text(value).gsub(SPACE, "").downcase
23
+ end
24
+
25
+ # The text of a header cell: a decimal as its plain digits ("2.5", "10"), not BigDecimal#to_s ("0.25e1").
26
+ # @return [String]
27
+ def text(value)
28
+ value.is_a?(BigDecimal) ? DecimalText.plain(value) : value.to_s
29
+ end
30
+
31
+ # @param header_values [Array] the cells of the header row
32
+ # @return [Array<String>] one unique key per cell: the text stripped of whitespace, Unicode whitespace included
33
+ # as for #normalize (original case), the column letter for a blank cell, and "Name (2)", "Name (3)"... for
34
+ # repeats, counted by occurrence of the stripped text
35
+ def build(header_values)
36
+ stripped = header_values.map { |value| strip(text(value)) }
37
+ texts = stripped.each_with_index.map { |text, index| text.empty? ? ColumnLetter.from_index(index) : text }
38
+ reserved = stripped.reject(&:empty?).to_set
39
+ occurrences = Hash.new(0)
40
+ # The suffix after the last one given to each text: every suffix below it is taken, by a header or an earlier
41
+ # key, and stays taken, so the next search starts there and each suffix is tried once overall.
42
+ next_counter = {}
43
+ used = Set.new
44
+
45
+ texts.map do |text|
46
+ occurrences[text] += 1
47
+ key = text
48
+ if occurrences[text] > 1
49
+ counter = [ occurrences[text], next_counter[text] || 0 ].max
50
+ counter += 1 while used.include?(key = "#{text} (#{counter})") || reserved.include?(key)
51
+ next_counter[text] = counter + 1
52
+ end
53
+ used << key
54
+ key
55
+ end
56
+ end
57
+
58
+ # The text without the whitespace at its ends, found by searching for a character that is not whitespace: an
59
+ # anchored run such as /\A[[:space:]]+/ costs memory in proportion to its length.
60
+ def strip(text)
61
+ first = text.index(NON_SPACE) or return ""
62
+ last = text.rindex(NON_SPACE)
63
+ first.zero? && last == text.length - 1 ? text : text[first..last]
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,326 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zlib"
4
+ require_relative "entry_name"
5
+
6
+ module Calcsheet
7
+ # A read-only, IO-like view of one zip entry that inflates lazily in bounded
8
+ # steps and counts the bytes it actually produces, never trusting the sizes of
9
+ # the central directory. Obtained from ZipContainer#stream.
10
+ #
11
+ # The bytes handed out are checked against +max_compression_ratio+ times the
12
+ # compressed size the central directory declares, or the +compression_ratio_floor+
13
+ # when that is more, and against the container's +max_uncompressed_size+, shared by
14
+ # every stream and read of the container. An entry declared with compressed size 0
15
+ # may thus inflate only up to the floor (without a floor its first byte exceeds the
16
+ # ratio), and a deflated one is read for a single input chunk of INPUT_CHUNK_SIZE
17
+ # bytes: a deflate stream that does not end within it is truncated.
18
+ # The CRC-32 is verified when the entry has been inflated to its end; a mismatch
19
+ # is raised by the read that would hand out the last byte (or report the end),
20
+ # so a consumer that reads everything never receives damaged data silently.
21
+ #
22
+ # Supported methods, which are the ones Ox's SAX parser uses on an object that
23
+ # is not a File or a StringIO: +readpartial(maxlen)+ (raises EOFError at the end)
24
+ # and +read(length = nil, outbuf = nil)+ (+nil+ at the end for a positive length).
25
+ # Also +eof?+ and +close+. +gets+, +getc+, +each_line+ and seeking are not
26
+ # supported.
27
+ #
28
+ # Once a limit or damage error has been raised the stream raises it again on
29
+ # every later read.
30
+ #
31
+ # With +max_cell_bytes+, the inflated bytes are also an XML part whose texts and tags (the bytes between two
32
+ # "<") may be at most 6 times +max_cell_bytes+ plus 64 KiB long: Ox buffers a text or a tag whole before its
33
+ # handler sees it, so a single huge one would otherwise cost its size in memory, twice. Six bytes of XML are the
34
+ # most one character of text takes ("&quot;"), 64 KiB room for the tags, whose namespace declarations can be
35
+ # long. The check costs two byte searches per inflated piece (TokenBound). A stream that a byte scanner passes over
36
+ # before the parser reads it (ZipContainer#stream with +bound_tokens: false+) leaves the check to the TokenBound of
37
+ # what is handed to the parser (StreamRemainder).
38
+ #
39
+ # @api private
40
+ class LimitedInflater
41
+ # Raised for a text or a tag longer than the bound of +max_cell_bytes+; the parsers add the position of the cell.
42
+ class TokenTooLong < MalformedFileError
43
+ # @return [Integer] the max_cell_bytes the bound comes from
44
+ attr_reader :max_cell_bytes
45
+
46
+ def initialize(message, max_cell_bytes:)
47
+ super(message)
48
+ @max_cell_bytes = max_cell_bytes
49
+ end
50
+ end
51
+
52
+ # Bounds the bytes between two "<" of the pieces of an XML stream, in order: 6 times +max_cell_bytes+ plus
53
+ # 64 KiB (see LimitedInflater).
54
+ class TokenBound
55
+ # Bytes of XML per byte of text, and the room for tags, of the bound on the bytes between two "<".
56
+ TOKEN_BYTES_PER_TEXT_BYTE = 6
57
+ TAG_ALLOWANCE = 65_536
58
+
59
+ # @param name [String] the entry name as messages show it
60
+ # @param max_cell_bytes [Integer]
61
+ def initialize(name, max_cell_bytes)
62
+ @name = name
63
+ @max_cell_bytes = max_cell_bytes
64
+ @max = max_cell_bytes * TOKEN_BYTES_PER_TEXT_BYTE + TAG_ALLOWANCE
65
+ # The bytes after the last "<" seen so far, in this piece and the ones before it: the run still open.
66
+ @token = 0
67
+ end
68
+
69
+ # Counts the next piece of the stream.
70
+ # @raise [TokenTooLong] when a text or a tag is longer than the bound
71
+ def check(piece)
72
+ # A piece no longer than the bound holds no run longer than it between two "<": only the bytes before its
73
+ # first "<" and after its last one can make one, with the pieces around it.
74
+ return check_each(piece) if piece.bytesize > @max
75
+
76
+ first = piece.byteindex("<")
77
+ if first.nil?
78
+ @token += piece.bytesize
79
+ else
80
+ @token += first
81
+ too_long if @token > @max
82
+ @token = piece.bytesize - piece.byterindex("<") - 1
83
+ end
84
+ too_long if @token > @max
85
+ end
86
+
87
+ private
88
+
89
+ # Every run of a piece longer than the bound.
90
+ def check_each(piece)
91
+ from = 0
92
+ while (found = piece.byteindex("<", from))
93
+ @token += found - from
94
+ too_long if @token > @max
95
+ @token = 0
96
+ from = found + 1
97
+ end
98
+ @token += piece.bytesize - from
99
+ too_long if @token > @max
100
+ end
101
+
102
+ def too_long
103
+ raise TokenTooLong.new("entry #{@name}: a text or a tag is longer than #{@max} bytes, more than " \
104
+ "max_cell_bytes (#{@max_cell_bytes} bytes) allows", max_cell_bytes: @max_cell_bytes)
105
+ end
106
+ end
107
+
108
+ # The compressed bytes read and inflated at a time.
109
+ INPUT_CHUNK_SIZE = 4 * 1024
110
+ # The fewest consumed bytes at the front of the buffer that #take drops: it drops them once they are at least
111
+ # this many and at least half of the buffer. A buffer consumed in full is reset whatever its size.
112
+ COMPACT_THRESHOLD = 64 * 1024
113
+ # The compression methods supported: stored and deflated.
114
+ STORED = 0
115
+ DEFLATED = 8
116
+
117
+ # @param io [IO] the seekable archive; the stream seeks it before each read
118
+ # @param entry [Zip::Entry] the central directory entry
119
+ # @param data_offset [Integer] the position of the first byte of compressed data
120
+ # @param container [ZipContainer] charged for every inflated byte
121
+ # @param max_ratio [Numeric, nil] the compression ratio limit, +nil+ for none
122
+ # @param ratio_floor [Integer] the bytes the entry may inflate to whatever its ratio
123
+ # @param max_cell_bytes [Integer, nil] bounds the bytes between two "<"; nil for no bound
124
+ def initialize(io:, entry:, data_offset:, container:, max_ratio:, ratio_floor: 0, max_cell_bytes: nil)
125
+ @io = io
126
+ # The name as messages show it, inspected and cut (EntryName.show): the untrusted archive gives it.
127
+ @name = EntryName.show(entry.name)
128
+ @method = entry.compression_method
129
+ @compressed_size = entry.compressed_size
130
+ @expected_crc = entry.crc
131
+ @position = data_offset
132
+ # The compressed data is read only up to the declared size. A deflated entry declared with size 0 can only
133
+ # be a lie: it is read for one input chunk, and inflates at most to the floor (without a floor, its first
134
+ # inflated byte is a bomb).
135
+ @input_left = @method == DEFLATED && @compressed_size.zero? ? INPUT_CHUNK_SIZE : @compressed_size
136
+ @container = container
137
+ @max_ratio = max_ratio
138
+ @max_inflated = max_ratio && [ max_ratio * @compressed_size, ratio_floor ].max
139
+ @buffer = "".b
140
+ @offset = 0
141
+ @inflated = 0
142
+ @crc = 0
143
+ @finished = false
144
+ @failure = nil
145
+ @crc_mismatch = false
146
+ @inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS) if @method == DEFLATED
147
+ @tokens = max_cell_bytes && TokenBound.new(@name, max_cell_bytes)
148
+ end
149
+
150
+ # @param length [Integer, nil] bytes wanted; +nil+ reads to the end
151
+ # @param outbuf [String, nil] receives the result when given
152
+ # @return [String, nil] up to +length+ bytes; +nil+ at the end of the entry for a positive +length+
153
+ # @raise [ZipBombError] when a limit is exceeded
154
+ # @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
155
+ def read(length = nil, outbuf = nil)
156
+ raise ArgumentError, "negative length #{length} given" if length&.negative?
157
+
158
+ guarded do
159
+ fill(length)
160
+ data = take(length)
161
+ verify_end
162
+ outbuf&.replace(data)
163
+ next nil if data.empty? && length&.positive?
164
+
165
+ outbuf || data
166
+ end
167
+ end
168
+
169
+ # @param maxlen [Integer] the most bytes to return
170
+ # @param outbuf [String, nil] receives the result when given
171
+ # @return [String] at least one byte (none for a +maxlen+ of 0)
172
+ # @raise [EOFError] at the end of the entry
173
+ # @raise [ZipBombError] when a limit is exceeded
174
+ # @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
175
+ def readpartial(maxlen, outbuf = nil)
176
+ raise ArgumentError, "negative length #{maxlen} given" if maxlen.negative?
177
+
178
+ guarded do
179
+ step while available.zero? && !@finished
180
+ raise EOFError, "end of file reached" if available.zero? && maxlen.positive?
181
+
182
+ data = take(maxlen)
183
+ verify_end
184
+ outbuf ? outbuf.replace(data) : data
185
+ end
186
+ end
187
+
188
+ # @return [Boolean] whether every byte has been handed out (inflating on to find out)
189
+ # @raise [ZipBombError] when a limit is exceeded
190
+ # @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
191
+ def eof?
192
+ guarded do
193
+ step while available.zero? && !@finished
194
+ verify_end
195
+ available.zero?
196
+ end
197
+ end
198
+
199
+ # Releases the inflater. Idempotent; further reads see an empty entry.
200
+ def close
201
+ @inflater.close if @inflater && !@inflater.closed?
202
+ @finished = true
203
+ @buffer = "".b
204
+ @offset = 0
205
+ nil
206
+ end
207
+
208
+ private
209
+
210
+ def guarded
211
+ raise @failure if @failure
212
+
213
+ yield
214
+ rescue ZipBombError, MalformedFileError, UnsupportedFormatError => error
215
+ @failure = error
216
+ raise
217
+ end
218
+
219
+ def fill(length)
220
+ step while !@finished && (length.nil? || available < length)
221
+ end
222
+
223
+ def available
224
+ @buffer.bytesize - @offset
225
+ end
226
+
227
+ # Hands out up to +length+ buffered bytes by moving an offset, so that many small reads of a large buffer
228
+ # stay linear; the consumed prefix is dropped once it is large and at least half of the buffer, and a buffer
229
+ # consumed in full is reset at once.
230
+ def take(length)
231
+ count = length.nil? ? available : [ length, available ].min
232
+ data = @buffer.byteslice(@offset, count)
233
+ @offset += count
234
+ if @offset == @buffer.bytesize
235
+ @buffer = "".b
236
+ @offset = 0
237
+ elsif @offset >= COMPACT_THRESHOLD && @offset * 2 >= @buffer.bytesize
238
+ @buffer = @buffer.byteslice(@offset..)
239
+ @offset = 0
240
+ end
241
+ data
242
+ end
243
+
244
+ # Reads one chunk of compressed data and inflates it into the buffer.
245
+ def step
246
+ case @method
247
+ when STORED then step_stored
248
+ when DEFLATED then step_deflated
249
+ else raise UnsupportedFormatError, "entry #{@name} uses compression method #{@method}, which is not supported"
250
+ end
251
+ rescue Zlib::Error => error
252
+ raise MalformedFileError, "entry #{@name} cannot be inflated: #{error.message}"
253
+ end
254
+
255
+ def step_stored
256
+ return finish if @input_left <= 0
257
+
258
+ emit(read_input(INPUT_CHUNK_SIZE))
259
+ end
260
+
261
+ def step_deflated
262
+ @inflater.inflate(read_input(INPUT_CHUNK_SIZE)) { |piece| emit(piece) }
263
+ # Zlib keeps the end of the output of a chunk that inflates to much more than its buffer (a long run): with
264
+ # the input exhausted, flushing hands it out, in pieces of the same size.
265
+ flush_deflated if @input_left.zero? && !@inflater.finished?
266
+ finish if @inflater.finished?
267
+ end
268
+
269
+ def flush_deflated
270
+ @inflater.inflate(nil) { |piece| emit(piece) }
271
+ rescue Zlib::BufError
272
+ raise MalformedFileError, "entry #{@name} is truncated: the compressed data ends early"
273
+ end
274
+
275
+ def read_input(length)
276
+ length = [ length, @input_left ].min
277
+ @io.seek(@position, IO::SEEK_SET)
278
+ data = length.positive? ? @io.read(length) : nil
279
+ raise MalformedFileError, "entry #{@name} is truncated: the compressed data ends early" if data.nil? || data.empty?
280
+
281
+ @position += data.bytesize
282
+ @input_left -= data.bytesize
283
+ data
284
+ end
285
+
286
+ # Counts, checks and buffers a piece of inflated data.
287
+ def emit(piece)
288
+ @inflated += piece.bytesize
289
+ check_ratio
290
+ @container.charge(piece.bytesize)
291
+ @tokens&.check(piece)
292
+ @crc = Zlib.crc32(piece, @crc)
293
+ @buffer << piece
294
+ end
295
+
296
+ def check_ratio
297
+ return if @max_inflated.nil? || @inflated <= @max_inflated
298
+
299
+ raise ZipBombError.new(limit: :max_compression_ratio, max: @max_ratio,
300
+ actual: (@inflated.to_f / [ @compressed_size, 1 ].max).ceil)
301
+ end
302
+
303
+ def finish
304
+ @finished = true
305
+ consumed = @inflater&.total_in
306
+ @inflater&.close
307
+ check_compressed_size(consumed)
308
+ @crc_mismatch = @crc != @expected_crc
309
+ end
310
+
311
+ # A deflate stream must end exactly where the central directory says its data ends.
312
+ def check_compressed_size(consumed)
313
+ return if consumed.nil? || @compressed_size.zero? || consumed == @compressed_size
314
+
315
+ raise MalformedFileError, "entry #{@name} is damaged: the compressed data is #{consumed} bytes, " \
316
+ "the central directory says #{@compressed_size}"
317
+ end
318
+
319
+ # Raises the CRC mismatch once everything inflated has been handed out.
320
+ def verify_end
321
+ return unless @crc_mismatch && available.zero?
322
+
323
+ raise MalformedFileError, "entry #{@name} is damaged: CRC mismatch"
324
+ end
325
+ end
326
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "header_keys"
4
+
5
+ module Calcsheet
6
+ # Whether sheets can be read as one, as Book#merge_check returns it: every non-empty sheet must have the header of
7
+ # the first one, after the normalization used for mapping by name (HeaderKeys.normalize), trailing blank header
8
+ # cells ignored.
9
+ #
10
+ # @attr_reader sheet_count [Integer] the sheets checked, empty ones included
11
+ # @attr_reader mismatches [Array<Hash>] { sheet:, expected:, actual: } per differing sheet, with the header values
12
+ # of the first non-empty sheet and of the differing one (frozen)
13
+ MergeCheck = Data.define(:sheet_count, :mismatches) do
14
+ # @api private
15
+ # @param headers [Array<Array(String, Array)>] sheet name and header values (nil for an empty sheet)
16
+ # @return [MergeCheck]
17
+ def self.of(headers)
18
+ present = headers.reject { |_name, values| values.nil? }
19
+ expected = present.first&.last
20
+ wanted = normalize(expected)
21
+ mismatches = present.drop(1).filter_map do |name, values|
22
+ { sheet: name, expected: expected, actual: values } unless normalize(values) == wanted
23
+ end
24
+ new(sheet_count: headers.size, mismatches: mismatches.freeze)
25
+ end
26
+
27
+ # @api private
28
+ # @param values [Array, nil] header values
29
+ # @return [Array<String>] the normalized names, without the trailing blank ones
30
+ def self.normalize(values)
31
+ names = Array(values).map { |value| HeaderKeys.normalize(value) }
32
+ names.pop while names.last == ""
33
+ names
34
+ end
35
+
36
+ # @return [Boolean] whether no sheet differs
37
+ def mergeable?
38
+ mismatches.empty?
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "grid"
4
+
5
+ module Calcsheet
6
+ # What the ODS reader, the ODS writer and the format detector share of the OpenDocument format.
7
+ #
8
+ # @api private
9
+ module Odf
10
+ # The media type of a spreadsheet, the content of its first entry, +mimetype+.
11
+ MIMETYPE = "application/vnd.oasis.opendocument.spreadsheet"
12
+ # The name of that entry.
13
+ MIMETYPE_ENTRY = "mimetype"
14
+ # The URI of each namespace, by its conventional prefix. The reader finds a document's own prefixes from these
15
+ # URIs (Readers::Ods::Scanner); the writer declares them with these prefixes.
16
+ NAMESPACES = {
17
+ office: "urn:oasis:names:tc:opendocument:xmlns:office:1.0",
18
+ style: "urn:oasis:names:tc:opendocument:xmlns:style:1.0",
19
+ text: "urn:oasis:names:tc:opendocument:xmlns:text:1.0",
20
+ table: "urn:oasis:names:tc:opendocument:xmlns:table:1.0",
21
+ fo: "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0",
22
+ number: "urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0",
23
+ config: "urn:oasis:names:tc:opendocument:xmlns:config:1.0",
24
+ calcext: "urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0",
25
+ ooo: "http://openoffice.org/2004/office"
26
+ }.freeze
27
+ # The spaces of a cell's text:s elements: the reader takes at most these, so that a few bytes
28
+ # (<text:s text:c="999999999"/>) cannot expand into a huge text, and the writer writes a longer run of spaces as
29
+ # several text:s. As many as an Excel cell holds characters.
30
+ MAX_SPACES = Grid::MAX_CELL_CHARACTERS
31
+
32
+ module_function
33
+
34
+ # @param prefix [String] a namespace prefix of the document; "" for the default namespace
35
+ # @return [String] what comes before a local name: "prefix:", or "" for the default namespace
36
+ def qualifier(prefix)
37
+ prefix.empty? ? "" : "#{prefix}:"
38
+ end
39
+
40
+ # @return [String] the qualified name of +local+ in the namespace of +prefix+
41
+ def qualified(prefix, local)
42
+ "#{qualifier(prefix)}#{local}"
43
+ end
44
+ end
45
+ end