calcsheet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +9 -0
- data/CHANGELOG.md +62 -0
- data/LICENSE.txt +21 -0
- data/README.md +921 -0
- data/lib/calcsheet/book.rb +164 -0
- data/lib/calcsheet/cast.rb +578 -0
- data/lib/calcsheet/column.rb +117 -0
- data/lib/calcsheet/column_letter.rb +34 -0
- data/lib/calcsheet/configuration.rb +186 -0
- data/lib/calcsheet/decimal_text.rb +19 -0
- data/lib/calcsheet/entry_name.rb +28 -0
- data/lib/calcsheet/errors.rb +186 -0
- data/lib/calcsheet/excel_serial.rb +65 -0
- data/lib/calcsheet/format_detector.rb +109 -0
- data/lib/calcsheet/formula_protection.rb +31 -0
- data/lib/calcsheet/grid.rb +20 -0
- data/lib/calcsheet/header_keys.rb +66 -0
- data/lib/calcsheet/limited_inflater.rb +326 -0
- data/lib/calcsheet/merge_check.rb +41 -0
- data/lib/calcsheet/odf.rb +45 -0
- data/lib/calcsheet/option_validation.rb +160 -0
- data/lib/calcsheet/options.rb +10 -0
- data/lib/calcsheet/raw_row.rb +18 -0
- data/lib/calcsheet/raw_types.rb +15 -0
- data/lib/calcsheet/readers/csv_dialect.rb +264 -0
- data/lib/calcsheet/readers/csv_input.rb +128 -0
- data/lib/calcsheet/readers/csv_reader.rb +239 -0
- data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
- data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
- data/lib/calcsheet/readers/ods/count.rb +30 -0
- data/lib/calcsheet/readers/ods/cursor.rb +121 -0
- data/lib/calcsheet/readers/ods/patterns.rb +70 -0
- data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
- data/lib/calcsheet/readers/ods/scanner.rb +302 -0
- data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
- data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
- data/lib/calcsheet/readers/ods_reader.rb +71 -0
- data/lib/calcsheet/readers/ox_errors.rb +40 -0
- data/lib/calcsheet/readers/references.rb +62 -0
- data/lib/calcsheet/readers/row_batches.rb +45 -0
- data/lib/calcsheet/readers/sax_sheet.rb +108 -0
- data/lib/calcsheet/readers/stream_remainder.rb +66 -0
- data/lib/calcsheet/readers/tag_scan.rb +119 -0
- data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
- data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
- data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
- data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
- data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
- data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
- data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
- data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
- data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
- data/lib/calcsheet/reading.rb +281 -0
- data/lib/calcsheet/row.rb +110 -0
- data/lib/calcsheet/sample.rb +10 -0
- data/lib/calcsheet/schema.rb +140 -0
- data/lib/calcsheet/sheet_info.rb +21 -0
- data/lib/calcsheet/sheet_names.rb +114 -0
- data/lib/calcsheet/sheet_selection.rb +58 -0
- data/lib/calcsheet/source.rb +263 -0
- data/lib/calcsheet/stream_body.rb +81 -0
- data/lib/calcsheet/time_zone.rb +96 -0
- data/lib/calcsheet/untrusted.rb +32 -0
- data/lib/calcsheet/version.rb +6 -0
- data/lib/calcsheet/write_book.rb +202 -0
- data/lib/calcsheet/write_sheet.rb +282 -0
- data/lib/calcsheet/writers/csv_writer.rb +138 -0
- data/lib/calcsheet/writers/ods_writer.rb +437 -0
- data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
- data/lib/calcsheet/writers/xml_text.rb +35 -0
- data/lib/calcsheet/writers/zip_buffered.rb +55 -0
- data/lib/calcsheet/zip_container.rb +190 -0
- data/lib/calcsheet/zip_directory.rb +202 -0
- data/lib/calcsheet/zip_end_record.rb +117 -0
- data/lib/calcsheet.rb +295 -0
- metadata +200 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zip"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require_relative "entry_name"
|
|
6
|
+
require_relative "errors"
|
|
7
|
+
require_relative "odf"
|
|
8
|
+
require_relative "zip_directory"
|
|
9
|
+
|
|
10
|
+
module Calcsheet
|
|
11
|
+
# Chooses the backend for a file from its content. The extension only breaks
|
|
12
|
+
# a tie between formats the content cannot tell apart.
|
|
13
|
+
#
|
|
14
|
+
# @api private
|
|
15
|
+
module FormatDetector
|
|
16
|
+
# The formats detection answers, and an explicit +format:+ may name.
|
|
17
|
+
FORMATS = %i[csv xlsx ods].freeze
|
|
18
|
+
# The first bytes of an OLE2 compound file: XLS, and encrypted XLSX.
|
|
19
|
+
OLE2_SIGNATURE = "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b.freeze
|
|
20
|
+
# A zip archive starts with a local header, or with the end record when it is empty.
|
|
21
|
+
ZIP_SIGNATURES = [ ZipDirectory::LOCAL_SIGNATURE, ZipEndRecord::EOCD_SIGNATURE ].freeze
|
|
22
|
+
# The entry that makes a zip an XLSX workbook.
|
|
23
|
+
XLSX_ENTRY = "xl/workbook.xml"
|
|
24
|
+
# The extensions that break a tie between XLSX and ODS.
|
|
25
|
+
EXTENSIONS = { ".xlsx" => :xlsx, ".xlsm" => :xlsx, ".ods" => :ods }.freeze
|
|
26
|
+
|
|
27
|
+
module_function
|
|
28
|
+
|
|
29
|
+
# Detects the format of a seekable IO and rewinds it. Content that is neither OLE2 nor a zip archive is CSV,
|
|
30
|
+
# whatever it holds: only an unknown explicit +format+ raises for itself. Only the zip central directory and the
|
|
31
|
+
# start of the +mimetype+ entry are read, never a whole entry; the entry count is checked against
|
|
32
|
+
# +max_zip_entries+ from the end of central directory record, and the central directory by ZipDirectory, before
|
|
33
|
+
# rubyzip loads it.
|
|
34
|
+
#
|
|
35
|
+
# @param io [IO] a seekable binary IO, any position
|
|
36
|
+
# @param path [String, nil] the file name, used only to break a tie
|
|
37
|
+
# @param format [Symbol, String, nil] an explicit format, which wins over the content (Calcsheet.open passes none:
|
|
38
|
+
# it skips detection itself)
|
|
39
|
+
# @param max_zip_entries [Integer, nil] the most entries a zip may have; nil for no limit
|
|
40
|
+
# @return [Symbol] +:csv+, +:xlsx+ or +:ods+
|
|
41
|
+
# @raise [UnsupportedFormatError] for an OLE2 file (XLS, an encrypted workbook), a zip that is neither XLSX nor ODS
|
|
42
|
+
# or both without a telling extension, and an explicit +format+ not in FORMATS
|
|
43
|
+
# @raise [MalformedFileError] for a zip file whose central directory or mimetype entry cannot be read
|
|
44
|
+
# @raise [ZipBombError] with limit +:max_zip_entries+ when the zip has more entries
|
|
45
|
+
def detect(io, path: nil, format: nil, max_zip_entries: nil)
|
|
46
|
+
return explicit(format) unless format.nil?
|
|
47
|
+
|
|
48
|
+
io.seek(0, IO::SEEK_SET)
|
|
49
|
+
head = io.read(OLE2_SIGNATURE.bytesize)&.b.to_s
|
|
50
|
+
return raise_xls if head == OLE2_SIGNATURE
|
|
51
|
+
return :csv unless ZIP_SIGNATURES.any? { head.start_with?(_1) }
|
|
52
|
+
|
|
53
|
+
detect_zip(io, path, max_zip_entries)
|
|
54
|
+
ensure
|
|
55
|
+
io.seek(0, IO::SEEK_SET) if format.nil?
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def explicit(format)
|
|
59
|
+
name = format.to_sym
|
|
60
|
+
return name if FORMATS.include?(name)
|
|
61
|
+
|
|
62
|
+
raise UnsupportedFormatError, "unsupported format #{name.inspect}; use :csv, :xlsx or :ods"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def raise_xls
|
|
66
|
+
raise UnsupportedFormatError,
|
|
67
|
+
"XLS (Excel 97-2003) files and encrypted or password-protected workbooks (OLE2) are not supported; " \
|
|
68
|
+
"save the file without a password as XLSX, ODS or CSV"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def detect_zip(io, path, max_entries)
|
|
72
|
+
archive, = ZipDirectory.open(io, max_entries: max_entries)
|
|
73
|
+
# Entries by the name ZipContainer finds them by; ZipDirectory refused names that are equal once normalized.
|
|
74
|
+
files = archive.entries.reject(&:directory?).to_h { |entry| [ EntryName.normalize(entry.name), entry ] }
|
|
75
|
+
candidates = []
|
|
76
|
+
candidates << :xlsx if files.key?(XLSX_ENTRY)
|
|
77
|
+
candidates << :ods if ods_mimetype?(files[Odf::MIMETYPE_ENTRY])
|
|
78
|
+
return candidates.first if candidates.size == 1
|
|
79
|
+
|
|
80
|
+
by_extension = EXTENSIONS[File.extname(path.to_s).downcase]
|
|
81
|
+
return by_extension if candidates.size > 1 && candidates.include?(by_extension)
|
|
82
|
+
|
|
83
|
+
raise UnsupportedFormatError, unsupported_zip_message(candidates)
|
|
84
|
+
rescue Calcsheet::Error
|
|
85
|
+
raise
|
|
86
|
+
rescue StandardError => error
|
|
87
|
+
# Any error of rubyzip reading the mimetype entry is a damaged archive.
|
|
88
|
+
raise MalformedFileError, "the zip archive cannot be read: #{error.message}"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def ods_mimetype?(entry)
|
|
92
|
+
return false unless entry
|
|
93
|
+
|
|
94
|
+
# Two bytes more than the media type: a trailing line break (CR LF) is tolerated, as strip removes it, and a
|
|
95
|
+
# longer content still compares unequal, without reading the whole entry.
|
|
96
|
+
entry.get_input_stream { |stream| stream.read(Odf::MIMETYPE.bytesize + 2).to_s.strip } == Odf::MIMETYPE
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def unsupported_zip_message(candidates)
|
|
100
|
+
if candidates.empty?
|
|
101
|
+
"the zip archive is neither an XLSX workbook nor an ODS spreadsheet"
|
|
102
|
+
else
|
|
103
|
+
"the zip archive looks like both XLSX and ODS and the file name does not say which"
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
private_class_method :explicit, :raise_xls, :detect_zip, :ods_mimetype?, :unsupported_zip_message
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
# CSV formula protection: a spreadsheet opening a CSV evaluates a field that starts with "=", "+", "-", "@", a
|
|
5
|
+
# tab or a carriage return as a formula, so the CSV writer puts a "'" before such text (+formula_protection+), and
|
|
6
|
+
# the CSV reader removes that "'" again with +strip_formula_prefix: true+.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
9
|
+
module FormulaProtection
|
|
10
|
+
# The first characters that make a spreadsheet evaluate a field (OWASP CSV injection).
|
|
11
|
+
TRIGGERS = [ "=", "+", "-", "@", "\t", "\r" ].freeze
|
|
12
|
+
# A text that starts with a trigger.
|
|
13
|
+
START = Regexp.new("\\A[#{Regexp.escape(TRIGGERS.join)}]")
|
|
14
|
+
|
|
15
|
+
module_function
|
|
16
|
+
|
|
17
|
+
# @param text [String]
|
|
18
|
+
# @return [String] +text+ with a "'" before it when it starts with a trigger; +text+ itself otherwise
|
|
19
|
+
def protect(text)
|
|
20
|
+
START.match?(text) ? "'#{text}" : text
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @param text [String]
|
|
24
|
+
# @return [String] +text+ without the "'" #protect adds; +text+ itself otherwise
|
|
25
|
+
def unprotect(text)
|
|
26
|
+
return text unless text.start_with?("'") && TRIGGERS.include?(text[1])
|
|
27
|
+
|
|
28
|
+
text.byteslice(1, text.bytesize - 1)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
# The size of a worksheet, the same in Excel (since 2007) and LibreOffice: the readers refuse content beyond it,
|
|
5
|
+
# the writers continue in a new sheet at MAX_ROWS and refuse more than MAX_COLUMNS columns.
|
|
6
|
+
#
|
|
7
|
+
# @api private
|
|
8
|
+
module Grid
|
|
9
|
+
# The rows of a sheet, 2**20.
|
|
10
|
+
MAX_ROWS = 1_048_576
|
|
11
|
+
# The columns of a sheet, 2**14: A to XFD.
|
|
12
|
+
MAX_COLUMNS = 16_384
|
|
13
|
+
# The 0-based index of the last column, and its letters.
|
|
14
|
+
LAST_COLUMN_INDEX = MAX_COLUMNS - 1
|
|
15
|
+
LAST_COLUMN = "XFD"
|
|
16
|
+
# The characters of an Excel cell, counted as UTF-16 code units: a character beyond the Basic Multilingual Plane
|
|
17
|
+
# counts twice.
|
|
18
|
+
MAX_CELL_CHARACTERS = 32_767
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
require_relative "column_letter"
|
|
5
|
+
require_relative "decimal_text"
|
|
6
|
+
|
|
7
|
+
module Calcsheet
|
|
8
|
+
# Header texts as names: the matching rule of the column schema and the keys of rows read without a schema.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
module HeaderKeys
|
|
12
|
+
# One whitespace character at a time: a run matched as /[[:space:]]+/ costs memory in proportion to its length.
|
|
13
|
+
SPACE = /[[:space:]]/
|
|
14
|
+
# A character that is not whitespace, searched for from either end by #strip.
|
|
15
|
+
NON_SPACE = /[^[:space:]]/
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# The rule shared by every comparison of header names.
|
|
20
|
+
# @return [String] downcased, with all whitespace removed; nil gives ""
|
|
21
|
+
def normalize(value)
|
|
22
|
+
text(value).gsub(SPACE, "").downcase
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# The text of a header cell: a decimal as its plain digits ("2.5", "10"), not BigDecimal#to_s ("0.25e1").
|
|
26
|
+
# @return [String]
|
|
27
|
+
def text(value)
|
|
28
|
+
value.is_a?(BigDecimal) ? DecimalText.plain(value) : value.to_s
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @param header_values [Array] the cells of the header row
|
|
32
|
+
# @return [Array<String>] one unique key per cell: the text stripped of whitespace, Unicode whitespace included
|
|
33
|
+
# as for #normalize (original case), the column letter for a blank cell, and "Name (2)", "Name (3)"... for
|
|
34
|
+
# repeats, counted by occurrence of the stripped text
|
|
35
|
+
def build(header_values)
|
|
36
|
+
stripped = header_values.map { |value| strip(text(value)) }
|
|
37
|
+
texts = stripped.each_with_index.map { |text, index| text.empty? ? ColumnLetter.from_index(index) : text }
|
|
38
|
+
reserved = stripped.reject(&:empty?).to_set
|
|
39
|
+
occurrences = Hash.new(0)
|
|
40
|
+
# The suffix after the last one given to each text: every suffix below it is taken, by a header or an earlier
|
|
41
|
+
# key, and stays taken, so the next search starts there and each suffix is tried once overall.
|
|
42
|
+
next_counter = {}
|
|
43
|
+
used = Set.new
|
|
44
|
+
|
|
45
|
+
texts.map do |text|
|
|
46
|
+
occurrences[text] += 1
|
|
47
|
+
key = text
|
|
48
|
+
if occurrences[text] > 1
|
|
49
|
+
counter = [ occurrences[text], next_counter[text] || 0 ].max
|
|
50
|
+
counter += 1 while used.include?(key = "#{text} (#{counter})") || reserved.include?(key)
|
|
51
|
+
next_counter[text] = counter + 1
|
|
52
|
+
end
|
|
53
|
+
used << key
|
|
54
|
+
key
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The text without the whitespace at its ends, found by searching for a character that is not whitespace: an
|
|
59
|
+
# anchored run such as /\A[[:space:]]+/ costs memory in proportion to its length.
|
|
60
|
+
def strip(text)
|
|
61
|
+
first = text.index(NON_SPACE) or return ""
|
|
62
|
+
last = text.rindex(NON_SPACE)
|
|
63
|
+
first.zero? && last == text.length - 1 ? text : text[first..last]
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zlib"
|
|
4
|
+
require_relative "entry_name"
|
|
5
|
+
|
|
6
|
+
module Calcsheet
|
|
7
|
+
# A read-only, IO-like view of one zip entry that inflates lazily in bounded
|
|
8
|
+
# steps and counts the bytes it actually produces, never trusting the sizes of
|
|
9
|
+
# the central directory. Obtained from ZipContainer#stream.
|
|
10
|
+
#
|
|
11
|
+
# The bytes handed out are checked against +max_compression_ratio+ times the
|
|
12
|
+
# compressed size the central directory declares, or the +compression_ratio_floor+
|
|
13
|
+
# when that is more, and against the container's +max_uncompressed_size+, shared by
|
|
14
|
+
# every stream and read of the container. An entry declared with compressed size 0
|
|
15
|
+
# may thus inflate only up to the floor (without a floor its first byte exceeds the
|
|
16
|
+
# ratio), and a deflated one is read for a single input chunk of INPUT_CHUNK_SIZE
|
|
17
|
+
# bytes: a deflate stream that does not end within it is truncated.
|
|
18
|
+
# The CRC-32 is verified when the entry has been inflated to its end; a mismatch
|
|
19
|
+
# is raised by the read that would hand out the last byte (or report the end),
|
|
20
|
+
# so a consumer that reads everything never receives damaged data silently.
|
|
21
|
+
#
|
|
22
|
+
# Supported methods, which are the ones Ox's SAX parser uses on an object that
|
|
23
|
+
# is not a File or a StringIO: +readpartial(maxlen)+ (raises EOFError at the end)
|
|
24
|
+
# and +read(length = nil, outbuf = nil)+ (+nil+ at the end for a positive length).
|
|
25
|
+
# Also +eof?+ and +close+. +gets+, +getc+, +each_line+ and seeking are not
|
|
26
|
+
# supported.
|
|
27
|
+
#
|
|
28
|
+
# Once a limit or damage error has been raised the stream raises it again on
|
|
29
|
+
# every later read.
|
|
30
|
+
#
|
|
31
|
+
# With +max_cell_bytes+, the inflated bytes are also an XML part whose texts and tags (the bytes between two
|
|
32
|
+
# "<") may be at most 6 times +max_cell_bytes+ plus 64 KiB long: Ox buffers a text or a tag whole before its
|
|
33
|
+
# handler sees it, so a single huge one would otherwise cost its size in memory, twice. Six bytes of XML are the
|
|
34
|
+
# most one character of text takes ("""), 64 KiB room for the tags, whose namespace declarations can be
|
|
35
|
+
# long. The check costs two byte searches per inflated piece (TokenBound). A stream that a byte scanner passes over
|
|
36
|
+
# before the parser reads it (ZipContainer#stream with +bound_tokens: false+) leaves the check to the TokenBound of
|
|
37
|
+
# what is handed to the parser (StreamRemainder).
|
|
38
|
+
#
|
|
39
|
+
# @api private
|
|
40
|
+
class LimitedInflater
|
|
41
|
+
# Raised for a text or a tag longer than the bound of +max_cell_bytes+; the parsers add the position of the cell.
|
|
42
|
+
class TokenTooLong < MalformedFileError
|
|
43
|
+
# @return [Integer] the max_cell_bytes the bound comes from
|
|
44
|
+
attr_reader :max_cell_bytes
|
|
45
|
+
|
|
46
|
+
def initialize(message, max_cell_bytes:)
|
|
47
|
+
super(message)
|
|
48
|
+
@max_cell_bytes = max_cell_bytes
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Bounds the bytes between two "<" of the pieces of an XML stream, in order: 6 times +max_cell_bytes+ plus
|
|
53
|
+
# 64 KiB (see LimitedInflater).
|
|
54
|
+
class TokenBound
|
|
55
|
+
# Bytes of XML per byte of text, and the room for tags, of the bound on the bytes between two "<".
|
|
56
|
+
TOKEN_BYTES_PER_TEXT_BYTE = 6
|
|
57
|
+
TAG_ALLOWANCE = 65_536
|
|
58
|
+
|
|
59
|
+
# @param name [String] the entry name as messages show it
|
|
60
|
+
# @param max_cell_bytes [Integer]
|
|
61
|
+
def initialize(name, max_cell_bytes)
|
|
62
|
+
@name = name
|
|
63
|
+
@max_cell_bytes = max_cell_bytes
|
|
64
|
+
@max = max_cell_bytes * TOKEN_BYTES_PER_TEXT_BYTE + TAG_ALLOWANCE
|
|
65
|
+
# The bytes after the last "<" seen so far, in this piece and the ones before it: the run still open.
|
|
66
|
+
@token = 0
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Counts the next piece of the stream.
|
|
70
|
+
# @raise [TokenTooLong] when a text or a tag is longer than the bound
|
|
71
|
+
def check(piece)
|
|
72
|
+
# A piece no longer than the bound holds no run longer than it between two "<": only the bytes before its
|
|
73
|
+
# first "<" and after its last one can make one, with the pieces around it.
|
|
74
|
+
return check_each(piece) if piece.bytesize > @max
|
|
75
|
+
|
|
76
|
+
first = piece.byteindex("<")
|
|
77
|
+
if first.nil?
|
|
78
|
+
@token += piece.bytesize
|
|
79
|
+
else
|
|
80
|
+
@token += first
|
|
81
|
+
too_long if @token > @max
|
|
82
|
+
@token = piece.bytesize - piece.byterindex("<") - 1
|
|
83
|
+
end
|
|
84
|
+
too_long if @token > @max
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
# Every run of a piece longer than the bound.
|
|
90
|
+
def check_each(piece)
|
|
91
|
+
from = 0
|
|
92
|
+
while (found = piece.byteindex("<", from))
|
|
93
|
+
@token += found - from
|
|
94
|
+
too_long if @token > @max
|
|
95
|
+
@token = 0
|
|
96
|
+
from = found + 1
|
|
97
|
+
end
|
|
98
|
+
@token += piece.bytesize - from
|
|
99
|
+
too_long if @token > @max
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def too_long
|
|
103
|
+
raise TokenTooLong.new("entry #{@name}: a text or a tag is longer than #{@max} bytes, more than " \
|
|
104
|
+
"max_cell_bytes (#{@max_cell_bytes} bytes) allows", max_cell_bytes: @max_cell_bytes)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The compressed bytes read and inflated at a time.
|
|
109
|
+
INPUT_CHUNK_SIZE = 4 * 1024
|
|
110
|
+
# The fewest consumed bytes at the front of the buffer that #take drops: it drops them once they are at least
|
|
111
|
+
# this many and at least half of the buffer. A buffer consumed in full is reset whatever its size.
|
|
112
|
+
COMPACT_THRESHOLD = 64 * 1024
|
|
113
|
+
# The compression methods supported: stored and deflated.
|
|
114
|
+
STORED = 0
|
|
115
|
+
DEFLATED = 8
|
|
116
|
+
|
|
117
|
+
# @param io [IO] the seekable archive; the stream seeks it before each read
|
|
118
|
+
# @param entry [Zip::Entry] the central directory entry
|
|
119
|
+
# @param data_offset [Integer] the position of the first byte of compressed data
|
|
120
|
+
# @param container [ZipContainer] charged for every inflated byte
|
|
121
|
+
# @param max_ratio [Numeric, nil] the compression ratio limit, +nil+ for none
|
|
122
|
+
# @param ratio_floor [Integer] the bytes the entry may inflate to whatever its ratio
|
|
123
|
+
# @param max_cell_bytes [Integer, nil] bounds the bytes between two "<"; nil for no bound
|
|
124
|
+
def initialize(io:, entry:, data_offset:, container:, max_ratio:, ratio_floor: 0, max_cell_bytes: nil)
|
|
125
|
+
@io = io
|
|
126
|
+
# The name as messages show it, inspected and cut (EntryName.show): the untrusted archive gives it.
|
|
127
|
+
@name = EntryName.show(entry.name)
|
|
128
|
+
@method = entry.compression_method
|
|
129
|
+
@compressed_size = entry.compressed_size
|
|
130
|
+
@expected_crc = entry.crc
|
|
131
|
+
@position = data_offset
|
|
132
|
+
# The compressed data is read only up to the declared size. A deflated entry declared with size 0 can only
|
|
133
|
+
# be a lie: it is read for one input chunk, and inflates at most to the floor (without a floor, its first
|
|
134
|
+
# inflated byte is a bomb).
|
|
135
|
+
@input_left = @method == DEFLATED && @compressed_size.zero? ? INPUT_CHUNK_SIZE : @compressed_size
|
|
136
|
+
@container = container
|
|
137
|
+
@max_ratio = max_ratio
|
|
138
|
+
@max_inflated = max_ratio && [ max_ratio * @compressed_size, ratio_floor ].max
|
|
139
|
+
@buffer = "".b
|
|
140
|
+
@offset = 0
|
|
141
|
+
@inflated = 0
|
|
142
|
+
@crc = 0
|
|
143
|
+
@finished = false
|
|
144
|
+
@failure = nil
|
|
145
|
+
@crc_mismatch = false
|
|
146
|
+
@inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS) if @method == DEFLATED
|
|
147
|
+
@tokens = max_cell_bytes && TokenBound.new(@name, max_cell_bytes)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# @param length [Integer, nil] bytes wanted; +nil+ reads to the end
|
|
151
|
+
# @param outbuf [String, nil] receives the result when given
|
|
152
|
+
# @return [String, nil] up to +length+ bytes; +nil+ at the end of the entry for a positive +length+
|
|
153
|
+
# @raise [ZipBombError] when a limit is exceeded
|
|
154
|
+
# @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
|
|
155
|
+
def read(length = nil, outbuf = nil)
|
|
156
|
+
raise ArgumentError, "negative length #{length} given" if length&.negative?
|
|
157
|
+
|
|
158
|
+
guarded do
|
|
159
|
+
fill(length)
|
|
160
|
+
data = take(length)
|
|
161
|
+
verify_end
|
|
162
|
+
outbuf&.replace(data)
|
|
163
|
+
next nil if data.empty? && length&.positive?
|
|
164
|
+
|
|
165
|
+
outbuf || data
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# @param maxlen [Integer] the most bytes to return
|
|
170
|
+
# @param outbuf [String, nil] receives the result when given
|
|
171
|
+
# @return [String] at least one byte (none for a +maxlen+ of 0)
|
|
172
|
+
# @raise [EOFError] at the end of the entry
|
|
173
|
+
# @raise [ZipBombError] when a limit is exceeded
|
|
174
|
+
# @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
|
|
175
|
+
def readpartial(maxlen, outbuf = nil)
|
|
176
|
+
raise ArgumentError, "negative length #{maxlen} given" if maxlen.negative?
|
|
177
|
+
|
|
178
|
+
guarded do
|
|
179
|
+
step while available.zero? && !@finished
|
|
180
|
+
raise EOFError, "end of file reached" if available.zero? && maxlen.positive?
|
|
181
|
+
|
|
182
|
+
data = take(maxlen)
|
|
183
|
+
verify_end
|
|
184
|
+
outbuf ? outbuf.replace(data) : data
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# @return [Boolean] whether every byte has been handed out (inflating on to find out)
|
|
189
|
+
# @raise [ZipBombError] when a limit is exceeded
|
|
190
|
+
# @raise [MalformedFileError] for damaged compressed data or a CRC mismatch
|
|
191
|
+
def eof?
|
|
192
|
+
guarded do
|
|
193
|
+
step while available.zero? && !@finished
|
|
194
|
+
verify_end
|
|
195
|
+
available.zero?
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Releases the inflater. Idempotent; further reads see an empty entry.
|
|
200
|
+
def close
|
|
201
|
+
@inflater.close if @inflater && !@inflater.closed?
|
|
202
|
+
@finished = true
|
|
203
|
+
@buffer = "".b
|
|
204
|
+
@offset = 0
|
|
205
|
+
nil
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
private
|
|
209
|
+
|
|
210
|
+
def guarded
|
|
211
|
+
raise @failure if @failure
|
|
212
|
+
|
|
213
|
+
yield
|
|
214
|
+
rescue ZipBombError, MalformedFileError, UnsupportedFormatError => error
|
|
215
|
+
@failure = error
|
|
216
|
+
raise
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def fill(length)
|
|
220
|
+
step while !@finished && (length.nil? || available < length)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def available
|
|
224
|
+
@buffer.bytesize - @offset
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Hands out up to +length+ buffered bytes by moving an offset, so that many small reads of a large buffer
|
|
228
|
+
# stay linear; the consumed prefix is dropped once it is large and at least half of the buffer, and a buffer
|
|
229
|
+
# consumed in full is reset at once.
|
|
230
|
+
def take(length)
|
|
231
|
+
count = length.nil? ? available : [ length, available ].min
|
|
232
|
+
data = @buffer.byteslice(@offset, count)
|
|
233
|
+
@offset += count
|
|
234
|
+
if @offset == @buffer.bytesize
|
|
235
|
+
@buffer = "".b
|
|
236
|
+
@offset = 0
|
|
237
|
+
elsif @offset >= COMPACT_THRESHOLD && @offset * 2 >= @buffer.bytesize
|
|
238
|
+
@buffer = @buffer.byteslice(@offset..)
|
|
239
|
+
@offset = 0
|
|
240
|
+
end
|
|
241
|
+
data
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Reads one chunk of compressed data and inflates it into the buffer.
|
|
245
|
+
def step
|
|
246
|
+
case @method
|
|
247
|
+
when STORED then step_stored
|
|
248
|
+
when DEFLATED then step_deflated
|
|
249
|
+
else raise UnsupportedFormatError, "entry #{@name} uses compression method #{@method}, which is not supported"
|
|
250
|
+
end
|
|
251
|
+
rescue Zlib::Error => error
|
|
252
|
+
raise MalformedFileError, "entry #{@name} cannot be inflated: #{error.message}"
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def step_stored
|
|
256
|
+
return finish if @input_left <= 0
|
|
257
|
+
|
|
258
|
+
emit(read_input(INPUT_CHUNK_SIZE))
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def step_deflated
|
|
262
|
+
@inflater.inflate(read_input(INPUT_CHUNK_SIZE)) { |piece| emit(piece) }
|
|
263
|
+
# Zlib keeps the end of the output of a chunk that inflates to much more than its buffer (a long run): with
|
|
264
|
+
# the input exhausted, flushing hands it out, in pieces of the same size.
|
|
265
|
+
flush_deflated if @input_left.zero? && !@inflater.finished?
|
|
266
|
+
finish if @inflater.finished?
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def flush_deflated
|
|
270
|
+
@inflater.inflate(nil) { |piece| emit(piece) }
|
|
271
|
+
rescue Zlib::BufError
|
|
272
|
+
raise MalformedFileError, "entry #{@name} is truncated: the compressed data ends early"
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def read_input(length)
|
|
276
|
+
length = [ length, @input_left ].min
|
|
277
|
+
@io.seek(@position, IO::SEEK_SET)
|
|
278
|
+
data = length.positive? ? @io.read(length) : nil
|
|
279
|
+
raise MalformedFileError, "entry #{@name} is truncated: the compressed data ends early" if data.nil? || data.empty?
|
|
280
|
+
|
|
281
|
+
@position += data.bytesize
|
|
282
|
+
@input_left -= data.bytesize
|
|
283
|
+
data
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Counts, checks and buffers a piece of inflated data.
|
|
287
|
+
def emit(piece)
|
|
288
|
+
@inflated += piece.bytesize
|
|
289
|
+
check_ratio
|
|
290
|
+
@container.charge(piece.bytesize)
|
|
291
|
+
@tokens&.check(piece)
|
|
292
|
+
@crc = Zlib.crc32(piece, @crc)
|
|
293
|
+
@buffer << piece
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def check_ratio
|
|
297
|
+
return if @max_inflated.nil? || @inflated <= @max_inflated
|
|
298
|
+
|
|
299
|
+
raise ZipBombError.new(limit: :max_compression_ratio, max: @max_ratio,
|
|
300
|
+
actual: (@inflated.to_f / [ @compressed_size, 1 ].max).ceil)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def finish
|
|
304
|
+
@finished = true
|
|
305
|
+
consumed = @inflater&.total_in
|
|
306
|
+
@inflater&.close
|
|
307
|
+
check_compressed_size(consumed)
|
|
308
|
+
@crc_mismatch = @crc != @expected_crc
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# A deflate stream must end exactly where the central directory says its data ends.
|
|
312
|
+
def check_compressed_size(consumed)
|
|
313
|
+
return if consumed.nil? || @compressed_size.zero? || consumed == @compressed_size
|
|
314
|
+
|
|
315
|
+
raise MalformedFileError, "entry #{@name} is damaged: the compressed data is #{consumed} bytes, " \
|
|
316
|
+
"the central directory says #{@compressed_size}"
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# Raises the CRC mismatch once everything inflated has been handed out.
|
|
320
|
+
def verify_end
|
|
321
|
+
return unless @crc_mismatch && available.zero?
|
|
322
|
+
|
|
323
|
+
raise MalformedFileError, "entry #{@name} is damaged: CRC mismatch"
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "header_keys"
|
|
4
|
+
|
|
5
|
+
module Calcsheet
|
|
6
|
+
# Whether sheets can be read as one, as Book#merge_check returns it: every non-empty sheet must have the header of
|
|
7
|
+
# the first one, after the normalization used for mapping by name (HeaderKeys.normalize), trailing blank header
|
|
8
|
+
# cells ignored.
|
|
9
|
+
#
|
|
10
|
+
# @attr_reader sheet_count [Integer] the sheets checked, empty ones included
|
|
11
|
+
# @attr_reader mismatches [Array<Hash>] { sheet:, expected:, actual: } per differing sheet, with the header values
|
|
12
|
+
# of the first non-empty sheet and of the differing one (frozen)
|
|
13
|
+
MergeCheck = Data.define(:sheet_count, :mismatches) do
|
|
14
|
+
# @api private
|
|
15
|
+
# @param headers [Array<Array(String, Array)>] sheet name and header values (nil for an empty sheet)
|
|
16
|
+
# @return [MergeCheck]
|
|
17
|
+
def self.of(headers)
|
|
18
|
+
present = headers.reject { |_name, values| values.nil? }
|
|
19
|
+
expected = present.first&.last
|
|
20
|
+
wanted = normalize(expected)
|
|
21
|
+
mismatches = present.drop(1).filter_map do |name, values|
|
|
22
|
+
{ sheet: name, expected: expected, actual: values } unless normalize(values) == wanted
|
|
23
|
+
end
|
|
24
|
+
new(sheet_count: headers.size, mismatches: mismatches.freeze)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @api private
|
|
28
|
+
# @param values [Array, nil] header values
|
|
29
|
+
# @return [Array<String>] the normalized names, without the trailing blank ones
|
|
30
|
+
def self.normalize(values)
|
|
31
|
+
names = Array(values).map { |value| HeaderKeys.normalize(value) }
|
|
32
|
+
names.pop while names.last == ""
|
|
33
|
+
names
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @return [Boolean] whether no sheet differs
|
|
37
|
+
def mergeable?
|
|
38
|
+
mismatches.empty?
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "grid"
|
|
4
|
+
|
|
5
|
+
module Calcsheet
|
|
6
|
+
# What the ODS reader, the ODS writer and the format detector share of the OpenDocument format.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
9
|
+
module Odf
|
|
10
|
+
# The media type of a spreadsheet, the content of its first entry, +mimetype+.
|
|
11
|
+
MIMETYPE = "application/vnd.oasis.opendocument.spreadsheet"
|
|
12
|
+
# The name of that entry.
|
|
13
|
+
MIMETYPE_ENTRY = "mimetype"
|
|
14
|
+
# The URI of each namespace, by its conventional prefix. The reader finds a document's own prefixes from these
|
|
15
|
+
# URIs (Readers::Ods::Scanner); the writer declares them with these prefixes.
|
|
16
|
+
NAMESPACES = {
|
|
17
|
+
office: "urn:oasis:names:tc:opendocument:xmlns:office:1.0",
|
|
18
|
+
style: "urn:oasis:names:tc:opendocument:xmlns:style:1.0",
|
|
19
|
+
text: "urn:oasis:names:tc:opendocument:xmlns:text:1.0",
|
|
20
|
+
table: "urn:oasis:names:tc:opendocument:xmlns:table:1.0",
|
|
21
|
+
fo: "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0",
|
|
22
|
+
number: "urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0",
|
|
23
|
+
config: "urn:oasis:names:tc:opendocument:xmlns:config:1.0",
|
|
24
|
+
calcext: "urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0",
|
|
25
|
+
ooo: "http://openoffice.org/2004/office"
|
|
26
|
+
}.freeze
|
|
27
|
+
# The spaces of a cell's text:s elements: the reader takes at most these, so that a few bytes
|
|
28
|
+
# (<text:s text:c="999999999"/>) cannot expand into a huge text, and the writer writes a longer run of spaces as
|
|
29
|
+
# several text:s. As many as an Excel cell holds characters.
|
|
30
|
+
MAX_SPACES = Grid::MAX_CELL_CHARACTERS
|
|
31
|
+
|
|
32
|
+
module_function
|
|
33
|
+
|
|
34
|
+
# @param prefix [String] a namespace prefix of the document; "" for the default namespace
|
|
35
|
+
# @return [String] what comes before a local name: "prefix:", or "" for the default namespace
|
|
36
|
+
def qualifier(prefix)
|
|
37
|
+
prefix.empty? ? "" : "#{prefix}:"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @return [String] the qualified name of +local+ in the namespace of +prefix+
|
|
41
|
+
def qualified(prefix, local)
|
|
42
|
+
"#{qualifier(prefix)}#{local}"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|