calcsheet 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +9 -0
  3. data/CHANGELOG.md +62 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +921 -0
  6. data/lib/calcsheet/book.rb +164 -0
  7. data/lib/calcsheet/cast.rb +578 -0
  8. data/lib/calcsheet/column.rb +117 -0
  9. data/lib/calcsheet/column_letter.rb +34 -0
  10. data/lib/calcsheet/configuration.rb +186 -0
  11. data/lib/calcsheet/decimal_text.rb +19 -0
  12. data/lib/calcsheet/entry_name.rb +28 -0
  13. data/lib/calcsheet/errors.rb +186 -0
  14. data/lib/calcsheet/excel_serial.rb +65 -0
  15. data/lib/calcsheet/format_detector.rb +109 -0
  16. data/lib/calcsheet/formula_protection.rb +31 -0
  17. data/lib/calcsheet/grid.rb +20 -0
  18. data/lib/calcsheet/header_keys.rb +66 -0
  19. data/lib/calcsheet/limited_inflater.rb +326 -0
  20. data/lib/calcsheet/merge_check.rb +41 -0
  21. data/lib/calcsheet/odf.rb +45 -0
  22. data/lib/calcsheet/option_validation.rb +160 -0
  23. data/lib/calcsheet/options.rb +10 -0
  24. data/lib/calcsheet/raw_row.rb +18 -0
  25. data/lib/calcsheet/raw_types.rb +15 -0
  26. data/lib/calcsheet/readers/csv_dialect.rb +264 -0
  27. data/lib/calcsheet/readers/csv_input.rb +128 -0
  28. data/lib/calcsheet/readers/csv_reader.rb +239 -0
  29. data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
  30. data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
  31. data/lib/calcsheet/readers/ods/count.rb +30 -0
  32. data/lib/calcsheet/readers/ods/cursor.rb +121 -0
  33. data/lib/calcsheet/readers/ods/patterns.rb +70 -0
  34. data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
  35. data/lib/calcsheet/readers/ods/scanner.rb +302 -0
  36. data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
  37. data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
  38. data/lib/calcsheet/readers/ods_reader.rb +71 -0
  39. data/lib/calcsheet/readers/ox_errors.rb +40 -0
  40. data/lib/calcsheet/readers/references.rb +62 -0
  41. data/lib/calcsheet/readers/row_batches.rb +45 -0
  42. data/lib/calcsheet/readers/sax_sheet.rb +108 -0
  43. data/lib/calcsheet/readers/stream_remainder.rb +66 -0
  44. data/lib/calcsheet/readers/tag_scan.rb +119 -0
  45. data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
  46. data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
  47. data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
  48. data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
  49. data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
  50. data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
  51. data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
  52. data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
  53. data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
  54. data/lib/calcsheet/reading.rb +281 -0
  55. data/lib/calcsheet/row.rb +110 -0
  56. data/lib/calcsheet/sample.rb +10 -0
  57. data/lib/calcsheet/schema.rb +140 -0
  58. data/lib/calcsheet/sheet_info.rb +21 -0
  59. data/lib/calcsheet/sheet_names.rb +114 -0
  60. data/lib/calcsheet/sheet_selection.rb +58 -0
  61. data/lib/calcsheet/source.rb +263 -0
  62. data/lib/calcsheet/stream_body.rb +81 -0
  63. data/lib/calcsheet/time_zone.rb +96 -0
  64. data/lib/calcsheet/untrusted.rb +32 -0
  65. data/lib/calcsheet/version.rb +6 -0
  66. data/lib/calcsheet/write_book.rb +202 -0
  67. data/lib/calcsheet/write_sheet.rb +282 -0
  68. data/lib/calcsheet/writers/csv_writer.rb +138 -0
  69. data/lib/calcsheet/writers/ods_writer.rb +437 -0
  70. data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
  71. data/lib/calcsheet/writers/xml_text.rb +35 -0
  72. data/lib/calcsheet/writers/zip_buffered.rb +55 -0
  73. data/lib/calcsheet/zip_container.rb +190 -0
  74. data/lib/calcsheet/zip_directory.rb +202 -0
  75. data/lib/calcsheet/zip_end_record.rb +117 -0
  76. data/lib/calcsheet.rb +295 -0
  77. metadata +200 -0
@@ -0,0 +1,350 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bigdecimal"
4
+ require "date"
5
+ require "stringio" # zip_kit 6.3 uses StringIO without requiring it
6
+ require "zip_kit"
7
+ require_relative "../column_letter"
8
+ require_relative "../decimal_text"
9
+ require_relative "../errors"
10
+ require_relative "../excel_serial"
11
+ require_relative "../grid"
12
+ require_relative "xml_text"
13
+ require_relative "zip_buffered"
14
+
15
+ module Calcsheet
16
+ module Writers
17
+ # XLSX writer backend, see the contract in WriteBook. Each sheet is one deflated zip entry streamed with
18
+ # zip_kit as its rows arrive, built as strings in a buffer of about BUFFER_SIZE bytes: memory stays flat and the
19
+ # output may be non-seekable. Text is written as inline strings (no shared strings table), never evaluated, so
20
+ # "+39..." and "=..." stay verbatim. The parts that list every sheet (workbook, relationships, content
21
+ # types) and the styles, known only at the end, are written after the sheets; zip entries have no required order.
22
+ #
23
+ # Styles: the default, the bold header, and one per number format a column needs: decimals with +scale+ (the
24
+ # full value is stored, the format shows +scale+ decimals), dates and datetimes. They are numbered in the order
25
+ # the columns first use them.
26
+ #
27
+ # @api private
28
+ class XlsxWriter
29
+ include ZipBuffered
30
+
31
+ # The characters beyond the Basic Multilingual Plane (String#count ranges), which Excel counts twice.
32
+ ASTRAL = "\u{10000}-\u{10FFFF}"
33
+
34
+ MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
35
+ REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
36
+ PACKAGE_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships"
37
+ CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types"
38
+ OFFICE_DOCUMENT = "#{REL_NS}/officeDocument".freeze
39
+ WORKSHEET = "#{REL_NS}/worksheet".freeze
40
+ STYLES = "#{REL_NS}/styles".freeze
41
+ SPREADSHEET_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml"
42
+ DECLARATION = %(<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n)
43
+ # The sheet view of a sheet whose first row is frozen.
44
+ FROZEN_HEADER = '<pane ySplit="1" topLeftCell="A2" activePane="bottomLeft" state="frozen"/>'
45
+
46
+ # The number format codes of :date and :datetime columns.
47
+ DATE_FORMAT = "yyyy-mm-dd"
48
+ DATETIME_FORMAT = "yyyy-mm-dd hh:mm:ss"
49
+ # Ids below are built-in formats.
50
+ FIRST_CUSTOM_FORMAT = 164
51
+ # The key of the bold header style in +@styles+, beside the number format codes.
52
+ HEADER_STYLE = :header
53
+
54
+
55
+ # @return [Integer] rows per sheet, header included
56
+ def self.max_rows_per_sheet
57
+ Grid::MAX_ROWS
58
+ end
59
+
60
+ # @param io [#<<] the output, never closed here
61
+ # @param _options [Options] as returned by WriteBook.prepare_options; XLSX has no format options
62
+ def initialize(io, _options)
63
+ @zip = ZipKit::Streamer.new(io)
64
+ @sheet_names = []
65
+ @styles = {}
66
+ @cell_formats = []
67
+ @number_formats = {}
68
+ @buffer = new_buffer
69
+ @entry = nil
70
+ end
71
+
72
+ # Opens the sheet's zip entry and writes its start, column widths and header row.
73
+ # @raise [ArgumentError] for more columns than XLSX holds
74
+ # @raise [WriteError] for a header longer than a cell holds
75
+ def start_sheet(name, columns, header:)
76
+ check_sheet(name, columns)
77
+ @sheet_names << name
78
+ @name = name
79
+ @columns = columns
80
+ @types = columns.map(&:type)
81
+ @row_number = 0
82
+ prepare_cells(columns)
83
+ @entry = @zip.write_deflated_file("xl/worksheets/sheet#{@sheet_names.size}.xml")
84
+ @buffer << sheet_start(columns, header)
85
+ write_header(columns) if header
86
+ end
87
+
88
+ # Writes one row of checked values (see the contract in WriteBook) from the constant parts of #prepare_cells.
89
+ # @raise [WriteError] for a date beyond the serials or a text longer than a cell; nothing of the row is written
90
+ def row(values)
91
+ mark = @buffer.bytesize
92
+ number = (@row_number + 1).to_s
93
+ @buffer << '<row r="' << number << '">'
94
+ index = 0
95
+ while index < values.size
96
+ value = values[index]
97
+ unless value.nil?
98
+ @buffer << @openings[index] << number << @middles[index]
99
+ cell(value, index)
100
+ @buffer << @closings[index]
101
+ end
102
+ index += 1
103
+ end
104
+ @buffer << "</row>"
105
+ @row_number += 1
106
+ flush if @buffer.bytesize >= BUFFER_SIZE
107
+ rescue StandardError
108
+ # Nothing of a rejected row is written (a WriteError of a date beyond the serials).
109
+ rollback(mark)
110
+ raise
111
+ end
112
+
113
+ # Ends the sheet's XML and closes its zip entry.
114
+ def end_sheet
115
+ @buffer << "</sheetData></worksheet>"
116
+ flush
117
+ @entry.close
118
+ @entry = nil
119
+ end
120
+
121
+ # Writes the parts that list the sheets, and the zip central directory. A workbook needs a sheet: a book without
122
+ # any gets an empty one.
123
+ def finish
124
+ empty_book_sheet if @sheet_names.empty?
125
+ write_parts("xl/styles.xml" => styles_xml, "xl/workbook.xml" => workbook_xml,
126
+ "xl/_rels/workbook.xml.rels" => workbook_relationships_xml, "_rels/.rels" => package_relationships_xml,
127
+ "[Content_Types].xml" => content_types_xml)
128
+ @zip.close
129
+ end
130
+
131
+ # @api private
132
+ # The first value of a checked row that #row would refuse, without writing anything: WriteSheet asks before it
133
+ # opens a continuation sheet for the row, so that a refused row leaves no sheet with only a header.
134
+ # @return [Array(Integer, String), nil] the column index and the reason
135
+ def row_problem(values)
136
+ values.each_with_index do |value, index|
137
+ next if value.nil?
138
+
139
+ reason =
140
+ case @types[index]
141
+ # A text of at most 32,767 bytes has at most as many UTF-16 code units (a UTF-8 character takes at least as
142
+ # many bytes as it takes units): only a longer one is counted.
143
+ when :string then length_reason(excel_length(value)) if value.bytesize > Grid::MAX_CELL_CHARACTERS && excel_length(value) > Grid::MAX_CELL_CHARACTERS
144
+ when :date, :datetime then date_reason(value) unless ExcelSerial.serial(value)
145
+ end
146
+ return [ index, reason ] if reason
147
+ end
148
+ nil
149
+ end
150
+
151
+ private
152
+
153
+ # @raise [ArgumentError] for more columns than XLSX holds
154
+ # @raise [WriteError] for a header longer than a cell holds (checked whether or not the header is written)
155
+ def check_sheet(name, columns)
156
+ check_columns(name, columns, "XLSX")
157
+
158
+ columns.each_with_index do |column, index|
159
+ header = column.header
160
+ next if header.bytesize <= Grid::MAX_CELL_CHARACTERS || (length = excel_length(header)) <= Grid::MAX_CELL_CHARACTERS
161
+
162
+ raise WriteError.at(length_reason(length), sheet: name, row: 1, column_index: index, header: header)
163
+ end
164
+ end
165
+
166
+ # The constant parts of each column's cells: "<c r=\"B" before the row number, its style and type up to the
167
+ # value after it, and what closes the cell.
168
+ def prepare_cells(columns)
169
+ @letters = columns.each_index.map { ColumnLetter.from_index(_1) }
170
+ @openings = @letters.map { %(<c r="#{_1}).freeze }
171
+ @middles = columns.map { |column| %("#{style_attribute(format_of(column))}#{value_opening(column.type)}).freeze }
172
+ @closings = columns.map { _1.type == :string ? "</t></is></c>" : "</v></c>" }
173
+ end
174
+
175
+ def value_opening(type)
176
+ case type
177
+ when :string then ' t="inlineStr"><is><t xml:space="preserve">'
178
+ when :boolean then ' t="b"><v>'
179
+ else "><v>"
180
+ end
181
+ end
182
+
183
+ def format_of(column)
184
+ case column.type
185
+ when :decimal then column.scale && (column.scale.zero? ? "0" : "0.#{"0" * column.scale}")
186
+ when :date then DATE_FORMAT
187
+ when :datetime then DATETIME_FORMAT
188
+ end
189
+ end
190
+
191
+ # @param key [String, Symbol, nil] a number format code, HEADER_STYLE, or nil for the default style
192
+ def style_attribute(key)
193
+ key.nil? ? "" : %( s="#{style(key)}")
194
+ end
195
+
196
+ # The cellXfs index of a style, registered at its first use; index 0 is the default style. Each cell format is
197
+ # +[numFmtId, fontId]+: the header is the General format (0) in the bold font (1, see #styles_xml), a number
198
+ # format its custom format in the default font (0).
199
+ def style(key)
200
+ @styles[key] ||= begin
201
+ @cell_formats << (key == HEADER_STYLE ? [ 0, 1 ] : [ number_format(key), 0 ])
202
+ @cell_formats.size
203
+ end
204
+ end
205
+
206
+ def number_format(code)
207
+ @number_formats[code] ||= FIRST_CUSTOM_FORMAT + @number_formats.size
208
+ end
209
+
210
+ def sheet_start(columns, header)
211
+ xml = +DECLARATION
212
+ xml << %(<worksheet xmlns="#{MAIN_NS}" xmlns:r="#{REL_NS}"><sheetViews><sheetView)
213
+ xml << ' tabSelected="1"' if @sheet_names.size == 1
214
+ xml << ' workbookViewId="0"'
215
+ xml << (header ? ">#{FROZEN_HEADER}</sheetView>" : "/>")
216
+ xml << "</sheetViews>"
217
+ xml << widths(columns)
218
+ xml << "<sheetData>"
219
+ end
220
+
221
+ def widths(columns)
222
+ cols = columns.each_with_index.filter_map do |column, index|
223
+ next if column.width.nil?
224
+
225
+ %(<col min="#{index + 1}" max="#{index + 1}" width="#{number_text(column.width)}" customWidth="1"/>)
226
+ end
227
+ cols.empty? ? "" : "<cols>#{cols.join}</cols>"
228
+ end
229
+
230
+ def number_text(number)
231
+ number.is_a?(Integer) ? number.to_s : Float(number).to_s
232
+ end
233
+
234
+ def write_header(columns)
235
+ style = style_attribute(HEADER_STYLE)
236
+ @buffer << '<row r="1">'
237
+ columns.each_with_index do |column, index|
238
+ @buffer << %(<c r="#{@letters[index]}1"#{style} t="inlineStr"><is><t xml:space="preserve">)
239
+ @buffer << XmlText.escape(column.header, ooxml: true) << "</t></is></c>"
240
+ end
241
+ @buffer << "</row>"
242
+ @row_number = 1
243
+ end
244
+
245
+ def cell(value, index)
246
+ case @types[index]
247
+ when :string
248
+ check_length(value, index) if value.bytesize > Grid::MAX_CELL_CHARACTERS
249
+ @buffer << XmlText.escape(value, ooxml: true)
250
+ when :integer then @buffer << value.to_s
251
+ when :decimal then @buffer << DecimalText.plain(value)
252
+ when :date then @buffer << (ExcelSerial.serial(value) || refuse(date_reason(value), index)).to_s
253
+ when :datetime then @buffer << datetime_serial(value, index)
254
+ else @buffer << (value ? "1" : "0") # :boolean
255
+ end
256
+ end
257
+
258
+ # A UTC Time whose fields are the wall clock, in whole seconds: the serial of its day plus the fraction of the
259
+ # day, as the double nearest to the exact value in its shortest text, as Excel writes it. Both integers are
260
+ # below 2**53, so the division is exact up to its correct rounding. Fewer digits would lose up to microseconds,
261
+ # enough for LibreOffice, which truncates to the second, to show 23:59:58 for 23:59:59.
262
+ def datetime_serial(time, index)
263
+ days, seconds = ExcelSerial.days_and_seconds(time)
264
+ serial = ExcelSerial.day_serial(days) or refuse(date_reason(time), index)
265
+ return serial.to_s if seconds.zero?
266
+
267
+ (serial * ExcelSerial::SECONDS_PER_DAY + seconds).fdiv(ExcelSerial::SECONDS_PER_DAY).to_s
268
+ end
269
+
270
+ # @raise [WriteError] the text has more characters than an Excel cell holds
271
+ def check_length(text, index)
272
+ length = excel_length(text)
273
+ refuse(length_reason(length), index) if length > Grid::MAX_CELL_CHARACTERS
274
+ end
275
+
276
+ def length_reason(length)
277
+ "an XLSX cell holds at most #{Grid::MAX_CELL_CHARACTERS} characters, got #{length}"
278
+ end
279
+
280
+ def date_reason(value)
281
+ "XLSX stores dates from 1900-01-01 to 9999-12-31, got #{value.inspect}"
282
+ end
283
+
284
+ # @raise [WriteError] at the row being written
285
+ def refuse(reason, index)
286
+ row = @row_number + 1
287
+ raise WriteError.at(reason, sheet: @name, row: row, column_index: index, header: @columns[index].header)
288
+ end
289
+
290
+ # The characters of +text+ as Excel counts them: one beyond the Basic Multilingual Plane counts twice.
291
+ def excel_length(text)
292
+ length = text.length
293
+ length += text.count(ASTRAL) unless text.ascii_only?
294
+ length
295
+ end
296
+
297
+ def styles_xml
298
+ formats = @number_formats.map { |code, id| %(<numFmt numFmtId="#{id}" formatCode="#{XmlText.escape(code)}"/>) }
299
+ xfs = [ [ 0, 0 ], *@cell_formats ].map do |format, font|
300
+ attributes = %(numFmtId="#{format}" fontId="#{font}" fillId="0" borderId="0" xfId="0")
301
+ attributes += ' applyNumberFormat="1"' unless format.zero?
302
+ attributes += ' applyFont="1"' unless font.zero?
303
+ "<xf #{attributes}/>"
304
+ end
305
+ font = '<sz val="11"/><name val="Calibri"/><family val="2"/>'
306
+ xml = +%(#{DECLARATION}<styleSheet xmlns="#{MAIN_NS}">)
307
+ xml << %(<numFmts count="#{formats.size}">#{formats.join}</numFmts>) unless formats.empty?
308
+ xml << %(<fonts count="2"><font>#{font}</font><font><b/>#{font}</font></fonts>)
309
+ xml << '<fills count="2"><fill><patternFill patternType="none"/></fill><fill><patternFill patternType="gray125"/></fill></fills>'
310
+ xml << '<borders count="1"><border><left/><right/><top/><bottom/><diagonal/></border></borders>'
311
+ xml << '<cellStyleXfs count="1"><xf numFmtId="0" fontId="0" fillId="0" borderId="0"/></cellStyleXfs>'
312
+ xml << %(<cellXfs count="#{xfs.size}">#{xfs.join}</cellXfs>)
313
+ xml << '<cellStyles count="1"><cellStyle name="Normal" xfId="0" builtinId="0"/></cellStyles></styleSheet>'
314
+ end
315
+
316
+ def workbook_xml
317
+ sheets = @sheet_names.each_with_index.map do |name, index|
318
+ %(<sheet name="#{XmlText.escape(name)}" sheetId="#{index + 1}" r:id="rId#{index + 1}"/>)
319
+ end
320
+ %(#{DECLARATION}<workbook xmlns="#{MAIN_NS}" xmlns:r="#{REL_NS}"><bookViews><workbookView activeTab="0"/></bookViews>) +
321
+ "<sheets>#{sheets.join}</sheets></workbook>"
322
+ end
323
+
324
+ def workbook_relationships_xml
325
+ relationships = @sheet_names.each_index.map do |index|
326
+ %(<Relationship Id="rId#{index + 1}" Type="#{WORKSHEET}" Target="worksheets/sheet#{index + 1}.xml"/>)
327
+ end
328
+ relationships << %(<Relationship Id="rId#{@sheet_names.size + 1}" Type="#{STYLES}" Target="styles.xml"/>)
329
+ %(#{DECLARATION}<Relationships xmlns="#{PACKAGE_REL_NS}">#{relationships.join}</Relationships>)
330
+ end
331
+
332
+ def package_relationships_xml
333
+ %(#{DECLARATION}<Relationships xmlns="#{PACKAGE_REL_NS}">) +
334
+ %(<Relationship Id="rId1" Type="#{OFFICE_DOCUMENT}" Target="xl/workbook.xml"/></Relationships>)
335
+ end
336
+
337
+ def content_types_xml
338
+ sheets = @sheet_names.each_index.map do |index|
339
+ %(<Override PartName="/xl/worksheets/sheet#{index + 1}.xml" ContentType="#{SPREADSHEET_TYPE}.worksheet+xml"/>)
340
+ end
341
+ %(#{DECLARATION}<Types xmlns="#{CONTENT_TYPES_NS}">) +
342
+ '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' \
343
+ '<Default Extension="xml" ContentType="application/xml"/>' +
344
+ %(<Override PartName="/xl/workbook.xml" ContentType="#{SPREADSHEET_TYPE}.sheet.main+xml"/>) +
345
+ sheets.join +
346
+ %(<Override PartName="/xl/styles.xml" ContentType="#{SPREADSHEET_TYPE}.styles+xml"/></Types>)
347
+ end
348
+ end
349
+ end
350
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ module Writers
5
+ # Text for the XML parts of XLSX and ODS files: markup escaped, the characters XML 1.0 does not allow
6
+ # removed. A carriage return becomes a character reference, since XML parsers turn a literal one into a line
7
+ # feed. Tab and line feed stay literal, so the result is for element content (and attributes without them).
8
+ #
9
+ # @api private
10
+ module XmlText
11
+ # The characters to replace or remove: markup, the carriage return and those outside the Char production of
12
+ # XML 1.0 (control characters other than tab, line feed and carriage return, U+FFFE and U+FFFF; a valid UTF-8
13
+ # String carries no surrogate).
14
+ SPECIAL = /[&<>"\r\u0000-\u0008\u000B\u000C\u000E-\u001F\uFFFE\uFFFF]/
15
+ # Characters missing from the table are removed.
16
+ REPLACEMENTS = { "&" => "&amp;", "<" => "&lt;", ">" => "&gt;", '"' => "&quot;", "\r" => "&#13;" }.freeze
17
+ # An underscore that starts what OOXML readers decode as an escape (_xHHHH_), overlapping ones included.
18
+ OOXML_ESCAPE = /_(?=x\h{4}_)/
19
+ # The escape of "_" itself.
20
+ ESCAPED_UNDERSCORE = "_x005F_"
21
+
22
+ module_function
23
+
24
+ # @param text [String] UTF-8
25
+ # @param ooxml [Boolean] also write the underscore of a literal "_xHHHH_" as "_x005F_", so that XLSX readers,
26
+ # which decode those escapes, read the text back unchanged (the XML of ODS has no such escapes)
27
+ # @return [String] +text+ itself when nothing changes
28
+ def escape(text, ooxml: false)
29
+ text = text.gsub(SPECIAL, REPLACEMENTS) if SPECIAL.match?(text)
30
+ text = text.gsub(OOXML_ESCAPE, ESCAPED_UNDERSCORE) if ooxml && text.include?("_x") && OOXML_ESCAPE.match?(text)
31
+ text
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../grid"
4
+
5
+ module Calcsheet
6
+ module Writers
7
+ # What the XLSX and ODS writers share: each streams a zip entry (+@entry+) from a String buffer (+@buffer+) of
8
+ # about BUFFER_SIZE bytes, flushed only after whole rows, so that a row the writer refuses can be taken back out of
9
+ # it; writes the parts known only at the end (+@zip+, a ZipKit::Streamer); and gives a book without sheets the
10
+ # empty one a workbook needs. The row loops stay in the writers.
11
+ #
12
+ # @api private
13
+ module ZipBuffered
14
+ # The bytes of rows the buffer collects before they go to the zip entry.
15
+ BUFFER_SIZE = 64 * 1024
16
+ # The name of the empty sheet of a book without sheets.
17
+ EMPTY_BOOK_SHEET = "Sheet1"
18
+
19
+ private
20
+
21
+ def new_buffer
22
+ String.new(capacity: BUFFER_SIZE * 2, encoding: Encoding::UTF_8)
23
+ end
24
+
25
+ # @param format [String] the format, for the message
26
+ # @raise [ArgumentError] for more columns than a sheet holds
27
+ def check_columns(name, columns, format)
28
+ return if columns.size <= Grid::MAX_COLUMNS
29
+
30
+ raise ArgumentError, "sheet '#{name}' has #{columns.size} columns, more than the #{Grid::MAX_COLUMNS} of #{format}"
31
+ end
32
+
33
+ # Takes back what a refused row wrote to the buffer since +mark+, its size before the row.
34
+ def rollback(mark)
35
+ @buffer.bytesplice(mark, @buffer.bytesize - mark, "")
36
+ end
37
+
38
+ def flush
39
+ @entry << @buffer
40
+ @buffer.clear
41
+ end
42
+
43
+ # The sheet of a book whose block wrote none.
44
+ def empty_book_sheet
45
+ start_sheet(EMPTY_BOOK_SHEET, [], header: false)
46
+ end_sheet
47
+ end
48
+
49
+ # @param parts [Hash{String => String}] the XML of each entry, by path
50
+ def write_parts(parts)
51
+ parts.each { |path, xml| @zip.write_deflated_file(path) { |sink| sink << xml } }
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zip"
4
+ require "zlib"
5
+ require_relative "entry_name"
6
+ require_relative "zip_directory"
7
+ require_relative "limited_inflater"
8
+
9
+ module Calcsheet
10
+ # The single door to a zip archive (XLSX and ODS are zip files) with the zip
11
+ # bomb protection enforced in one place:
12
+ #
13
+ # * +max_zip_entries+ is checked from the End Of Central Directory record, and the
14
+ # layout of the central directory by ZipDirectory, before rubyzip loads it;
15
+ # * +max_compression_ratio+ is checked per entry, above the +compression_ratio_floor+
16
+ # bytes any streamed entry may inflate to (a part kept in memory gets the fixed
17
+ # MEMORY_RATIO_FLOOR instead), and +max_uncompressed_size+ in total across every
18
+ # stream and read of the container, both on the bytes actually inflated, never on
19
+ # the sizes the archive declares;
20
+ # * +max_cell_bytes+ bounds the texts and tags of the XML entries (see LimitedInflater).
21
+ #
22
+ # A +nil+ limit disables that check. The container holds no resource of its own:
23
+ # it reads through the IO of the Source it was opened on, which stays owned by
24
+ # the caller (the readers hold the Source), and seeks it before every read, so
25
+ # any number of streams can be read side by side.
26
+ #
27
+ # @api private
28
+ class ZipContainer
29
+ # The steps of a #read bounded by +max_bytes+.
30
+ READ_CHUNK_BYTES = 65_536
31
+ # The bytes a part kept in memory (the XLSX shared strings) may inflate to whatever its ratio, instead of
32
+ # +compression_ratio_floor+, which does not apply to it: enough for a small file of long repetitive strings, while
33
+ # 16 MiB of empty strings cost about 54 MB (ADR 008).
34
+ MEMORY_RATIO_FLOOR = 16_777_216
35
+
36
+ # Opens an archive.
37
+ #
38
+ # @param source [Source] a source opened with +needs_seek: true+; its +io+ may be at any position
39
+ # @param options [Options] supplies +max_zip_entries+, +max_compression_ratio+, +compression_ratio_floor+,
40
+ # +max_uncompressed_size+ and +max_cell_bytes+
41
+ # @yieldparam container [ZipContainer]
42
+ # @return [Object, ZipContainer] the block value, or the container when no block is given
43
+ # @raise [ZipBombError] with limit +:max_zip_entries+ when the archive has too many entries
44
+ # @raise [MalformedFileError] (with the zip library's error as +cause+) for a damaged archive
45
+ def self.open(source, options)
46
+ container = new(source.io, options)
47
+ block_given? ? yield(container) : container
48
+ end
49
+
50
+ # @param io [IO] a seekable binary IO
51
+ # @param options [Options]
52
+ def initialize(io, options)
53
+ @io = io
54
+ @max_ratio = options.max_compression_ratio
55
+ @ratio_floor = options.compression_ratio_floor || 0
56
+ @max_total = options.max_uncompressed_size
57
+ @max_entries = options.max_zip_entries
58
+ @max_cell_bytes = options.max_cell_bytes
59
+ @inflated_total = 0
60
+ @zip = open_archive
61
+ end
62
+
63
+ # @return [Array<String>] the names of the file entries, backslashes read as slashes
64
+ def names
65
+ files.map(&:name)
66
+ end
67
+
68
+ # @param name [String] an entry name; a leading "/" and "\" separators are tolerated
69
+ # @return [Boolean] whether the archive has a file (not a directory) by that name
70
+ def entry?(name)
71
+ !find(name).nil?
72
+ end
73
+
74
+ # Reads a whole entry; meant for small parts. The bytes count against the limits like a stream's.
75
+ #
76
+ # @param max_bytes [Integer, nil] the most bytes the entry may inflate to; checked while it is read
77
+ # @return [String] binary
78
+ # @raise [MalformedFileError] when the entry is missing, damaged or longer than +max_bytes+
79
+ # @raise [ZipBombError] when a limit is exceeded
80
+ def read(name, max_bytes: nil)
81
+ input = stream(name)
82
+ return input.read if max_bytes.nil?
83
+
84
+ data = "".b
85
+ while (chunk = input.read(READ_CHUNK_BYTES))
86
+ data << chunk
87
+ raise MalformedFileError, "#{name} is longer than #{max_bytes} bytes" if data.bytesize > max_bytes
88
+ end
89
+ data
90
+ ensure
91
+ input&.close
92
+ end
93
+
94
+ # @param bound_tokens [Boolean] bound the texts and tags by +max_cell_bytes+ (LimitedInflater), for a stream that
95
+ # an XML parser reads; false for a reader that bounds what it buffers itself
96
+ # @param ratio_floor [Boolean] let the entry inflate to +compression_ratio_floor+ whatever its ratio; false for a
97
+ # part kept in memory (the shared strings), whose cost grows with what it inflates to, which may inflate to
98
+ # MEMORY_RATIO_FLOOR only
99
+ # @return [LimitedInflater] an IO-like object that inflates lazily and counts the inflated bytes
100
+ # @raise [MalformedFileError] when the entry is missing
101
+ # @raise [UnsupportedFormatError] for an encrypted entry or a compression method other than stored or deflate
102
+ def stream(name, bound_tokens: true, ratio_floor: true)
103
+ entry = find(name)
104
+ raise MalformedFileError, "the archive has no entry #{EntryName.show(name)}" if entry.nil?
105
+
106
+ check_supported(entry)
107
+ offset = data_offset(entry)
108
+ check_within_archive(entry, offset)
109
+ LimitedInflater.new(io: @io, entry: entry, data_offset: offset, container: self, max_ratio: @max_ratio,
110
+ ratio_floor: ratio_floor ? @ratio_floor : MEMORY_RATIO_FLOOR, max_cell_bytes: (@max_cell_bytes if bound_tokens))
111
+ end
112
+
113
+ # The bound on the texts and tags of the entry, for the bytes of a stream opened with +bound_tokens: false+ that
114
+ # an XML parser reads after a byte scanner (StreamRemainder).
115
+ # @return [LimitedInflater::TokenBound, nil] nil without +max_cell_bytes+
116
+ def token_bound(name)
117
+ @max_cell_bytes && LimitedInflater::TokenBound.new(EntryName.show(name), @max_cell_bytes)
118
+ end
119
+
120
+ # Adds inflated bytes to the total of the container. Called by its streams.
121
+ #
122
+ # @raise [ZipBombError] with limit +:max_uncompressed_size+ when the total passes the limit
123
+ def charge(bytes)
124
+ @inflated_total += bytes
125
+ return if @max_total.nil? || @inflated_total <= @max_total
126
+
127
+ raise ZipBombError.new(limit: :max_uncompressed_size, max: @max_total, actual: @inflated_total)
128
+ end
129
+
130
+ private
131
+
132
+ def open_archive
133
+ zip, @directory_offset = ZipDirectory.open(@io, max_entries: @max_entries)
134
+ zip
135
+ end
136
+
137
+ def files
138
+ @files ||= @zip.entries.reject(&:directory?)
139
+ end
140
+
141
+ # Exact name first; then with a leading "/" removed and "\\" read as "/" on both sides (rubyzip already
142
+ # reads backslashes of stored names as slashes).
143
+ def find(name)
144
+ entry = @zip.find_entry(name)
145
+ return entry if entry && !entry.directory?
146
+
147
+ normalized[normalize(name)]
148
+ end
149
+
150
+ def normalized
151
+ @normalized ||= files.each_with_object({}) { |entry, index| index[normalize(entry.name)] ||= entry }
152
+ end
153
+
154
+ def normalize(name)
155
+ EntryName.normalize(name)
156
+ end
157
+
158
+ def check_supported(entry)
159
+ if entry.encrypted?
160
+ raise UnsupportedFormatError, "entry #{EntryName.show(entry.name)} is encrypted; encrypted zip entries are not supported"
161
+ end
162
+ return if [ LimitedInflater::STORED, LimitedInflater::DEFLATED ].include?(entry.compression_method)
163
+
164
+ raise UnsupportedFormatError,
165
+ "entry #{EntryName.show(entry.name)} uses compression method #{entry.compression_method}, which is not supported"
166
+ end
167
+
168
+ # The declared compressed size is the denominator of the ratio limit, so it must be real: the data
169
+ # has to fit in the archive, before its central directory.
170
+ def check_within_archive(entry, offset)
171
+ @io.seek(0, IO::SEEK_END)
172
+ return if offset + entry.compressed_size <= [ @io.pos, @directory_offset ].compact.min
173
+
174
+ raise MalformedFileError, "entry #{EntryName.show(entry.name)} is damaged: its data extends past the end of the archive"
175
+ end
176
+
177
+ # Position of the first byte of the entry's data: after the local header, whose name and
178
+ # extra field lengths may differ from the central directory's.
179
+ def data_offset(entry)
180
+ @io.seek(entry.local_header_offset, IO::SEEK_SET)
181
+ header = @io.read(ZipDirectory::LOCAL_HEADER_SIZE)
182
+ unless header&.bytesize == ZipDirectory::LOCAL_HEADER_SIZE && header.start_with?(ZipDirectory::LOCAL_SIGNATURE)
183
+ raise MalformedFileError, "the local header of entry #{EntryName.show(entry.name)} is missing or damaged"
184
+ end
185
+
186
+ name_length, extra_length = header.unpack(ZipDirectory::LOCAL_NAME_LENGTHS)
187
+ entry.local_header_offset + ZipDirectory::LOCAL_HEADER_SIZE + name_length + extra_length
188
+ end
189
+ end
190
+ end