calcsheet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +9 -0
- data/CHANGELOG.md +62 -0
- data/LICENSE.txt +21 -0
- data/README.md +921 -0
- data/lib/calcsheet/book.rb +164 -0
- data/lib/calcsheet/cast.rb +578 -0
- data/lib/calcsheet/column.rb +117 -0
- data/lib/calcsheet/column_letter.rb +34 -0
- data/lib/calcsheet/configuration.rb +186 -0
- data/lib/calcsheet/decimal_text.rb +19 -0
- data/lib/calcsheet/entry_name.rb +28 -0
- data/lib/calcsheet/errors.rb +186 -0
- data/lib/calcsheet/excel_serial.rb +65 -0
- data/lib/calcsheet/format_detector.rb +109 -0
- data/lib/calcsheet/formula_protection.rb +31 -0
- data/lib/calcsheet/grid.rb +20 -0
- data/lib/calcsheet/header_keys.rb +66 -0
- data/lib/calcsheet/limited_inflater.rb +326 -0
- data/lib/calcsheet/merge_check.rb +41 -0
- data/lib/calcsheet/odf.rb +45 -0
- data/lib/calcsheet/option_validation.rb +160 -0
- data/lib/calcsheet/options.rb +10 -0
- data/lib/calcsheet/raw_row.rb +18 -0
- data/lib/calcsheet/raw_types.rb +15 -0
- data/lib/calcsheet/readers/csv_dialect.rb +264 -0
- data/lib/calcsheet/readers/csv_input.rb +128 -0
- data/lib/calcsheet/readers/csv_reader.rb +239 -0
- data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
- data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
- data/lib/calcsheet/readers/ods/count.rb +30 -0
- data/lib/calcsheet/readers/ods/cursor.rb +121 -0
- data/lib/calcsheet/readers/ods/patterns.rb +70 -0
- data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
- data/lib/calcsheet/readers/ods/scanner.rb +302 -0
- data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
- data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
- data/lib/calcsheet/readers/ods_reader.rb +71 -0
- data/lib/calcsheet/readers/ox_errors.rb +40 -0
- data/lib/calcsheet/readers/references.rb +62 -0
- data/lib/calcsheet/readers/row_batches.rb +45 -0
- data/lib/calcsheet/readers/sax_sheet.rb +108 -0
- data/lib/calcsheet/readers/stream_remainder.rb +66 -0
- data/lib/calcsheet/readers/tag_scan.rb +119 -0
- data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
- data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
- data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
- data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
- data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
- data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
- data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
- data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
- data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
- data/lib/calcsheet/reading.rb +281 -0
- data/lib/calcsheet/row.rb +110 -0
- data/lib/calcsheet/sample.rb +10 -0
- data/lib/calcsheet/schema.rb +140 -0
- data/lib/calcsheet/sheet_info.rb +21 -0
- data/lib/calcsheet/sheet_names.rb +114 -0
- data/lib/calcsheet/sheet_selection.rb +58 -0
- data/lib/calcsheet/source.rb +263 -0
- data/lib/calcsheet/stream_body.rb +81 -0
- data/lib/calcsheet/time_zone.rb +96 -0
- data/lib/calcsheet/untrusted.rb +32 -0
- data/lib/calcsheet/version.rb +6 -0
- data/lib/calcsheet/write_book.rb +202 -0
- data/lib/calcsheet/write_sheet.rb +282 -0
- data/lib/calcsheet/writers/csv_writer.rb +138 -0
- data/lib/calcsheet/writers/ods_writer.rb +437 -0
- data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
- data/lib/calcsheet/writers/xml_text.rb +35 -0
- data/lib/calcsheet/writers/zip_buffered.rb +55 -0
- data/lib/calcsheet/zip_container.rb +190 -0
- data/lib/calcsheet/zip_directory.rb +202 -0
- data/lib/calcsheet/zip_end_record.rb +117 -0
- data/lib/calcsheet.rb +295 -0
- metadata +200 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
module Readers
|
|
5
|
+
# What the byte scanners (Xlsx::RowSkipper, Ods::Cursor) share: they search a buffer of the inflated XML for the
|
|
6
|
+
# opening of the tags they want, pass over comments, CDATA sections and processing instructions, and keep only
|
|
7
|
+
# what a tag cut by the end of a chunk needs. The including class keeps the binary +@buffer+ and the stream +@io+,
|
|
8
|
+
# and defines +located(message)+, the MalformedFileError of a message at its position.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
module TagScan
|
|
12
|
+
# The bytes read from the stream at a time.
|
|
13
|
+
CHUNK_BYTES = 65_536
|
|
14
|
+
# The longest tag a scanner buffers: a wanted start tag, or the name of one cut by a chunk boundary.
|
|
15
|
+
MAX_TAG_BYTES = 65_536
|
|
16
|
+
# A start or end tag from "<" to ">", with ">" allowed inside quoted attribute values.
|
|
17
|
+
TAG = /\G<(?>[^>"']*)(?>(?:"[^"]*"|'[^']*')(?>[^>"']*))*>/n
|
|
18
|
+
# A tag without quoted values or another "<", such as an end tag.
|
|
19
|
+
PLAIN_TAG = /\G<[^<>"']*>/n
|
|
20
|
+
# The bytes after "<" that open a processing instruction, and a comment, CDATA section or declaration.
|
|
21
|
+
QUESTION_MARK = 0x3F
|
|
22
|
+
EXCLAMATION_MARK = 0x21
|
|
23
|
+
# What ends the markup that starts with "<?", "<!--" and "<![CDATA[", and the length of each opening.
|
|
24
|
+
INSTRUCTION = [ "?>", 2 ].freeze
|
|
25
|
+
COMMENT = [ "-->", 4 ].freeze
|
|
26
|
+
CDATA = [ "]]>", 9 ].freeze
|
|
27
|
+
COMMENT_OPENING = "<!--".b.freeze
|
|
28
|
+
CDATA_OPENING = "<![CDATA[".b.freeze
|
|
29
|
+
# The "<!" markup whose text is passed over, with its terminator and opening length; another "<!" (a DOCTYPE)
|
|
30
|
+
# is passed over as a tag.
|
|
31
|
+
DECLARATIONS = [ [ COMMENT_OPENING, COMMENT ].freeze, [ CDATA_OPENING, CDATA ].freeze ].freeze
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
# @return [Array(String, Integer), Symbol, nil] the terminator and opening length of the markup at +start+;
|
|
36
|
+
# :incomplete when the buffer ends inside what may be an opening, :other for another "<!" declaration, nil
|
|
37
|
+
# for a tag
|
|
38
|
+
def markup_at(start)
|
|
39
|
+
case @buffer.getbyte(start + 1)
|
|
40
|
+
when QUESTION_MARK then INSTRUCTION
|
|
41
|
+
when EXCLAMATION_MARK
|
|
42
|
+
DECLARATIONS.each do |opening, markup|
|
|
43
|
+
piece = @buffer.byteslice(start, opening.bytesize)
|
|
44
|
+
return markup if piece == opening
|
|
45
|
+
return :incomplete if piece.bytesize < opening.bytesize && opening.start_with?(piece)
|
|
46
|
+
end
|
|
47
|
+
:other
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Whether the "<" at +start+ (a wanted tag or a markup opening) is not preceded by a complete tag: the search
|
|
52
|
+
# passes over the other tags without reading them, so the "<" before +start+, read with TAG's quoted values,
|
|
53
|
+
# must end before +start+. Otherwise +start+ is inside an attribute value, where a "<" is not XML but Ox would
|
|
54
|
+
# read it as text, and the scan cannot tell what the parser will see. +from+, where the search started, is
|
|
55
|
+
# always at a tag boundary (after a tag or markup, or at a "<" kept by #partial_tag), so the "<" before +start+
|
|
56
|
+
# is a tag's.
|
|
57
|
+
def enclosing_tag?(from, start)
|
|
58
|
+
return false if start == from
|
|
59
|
+
|
|
60
|
+
last = @buffer.byterindex("<", start - 1)
|
|
61
|
+
return false if last.nil? || last < from
|
|
62
|
+
# The common case without a MatchData: a tag without quotes (an end tag) that ends before the next "<".
|
|
63
|
+
return false if PLAIN_TAG.match?(@buffer, last)
|
|
64
|
+
|
|
65
|
+
match = TAG.match(@buffer, last)
|
|
66
|
+
!(match && match.end(0) <= start)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def inside_attribute
|
|
70
|
+
located("a \"<\" inside a tag, which XML does not allow")
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def long_name
|
|
74
|
+
located("an element name is longer than #{MAX_TAG_BYTES} bytes")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Reads past the markup at +start+, dropping its text as it goes.
|
|
78
|
+
# @return [Integer, nil] the position after its terminator; nil when the stream ends first
|
|
79
|
+
def skip_markup(start, terminator, opening)
|
|
80
|
+
from = start + opening
|
|
81
|
+
loop do
|
|
82
|
+
found = @buffer.byteindex(terminator, from)
|
|
83
|
+
return found + terminator.bytesize if found
|
|
84
|
+
|
|
85
|
+
# Keep only what could be the beginning of a terminator split between chunks.
|
|
86
|
+
discard([ from, @buffer.bytesize - terminator.bytesize + 1 ].max)
|
|
87
|
+
from = 0
|
|
88
|
+
return unless read_more
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Where a wanted tag cut by the end of the buffer may begin: the last "<" after +from+ when only name bytes
|
|
93
|
+
# (+partial_name+, anchored at it) follow it; the end of the buffer otherwise. A tag whose name has ended is not
|
|
94
|
+
# wanted (the search would have found it), so it is passed over unbuffered, whatever its length: the scan then
|
|
95
|
+
# resumes inside it, which is safe for well-formed XML, where no "<" occurs inside a tag (see #enclosing_tag?).
|
|
96
|
+
# @raise [MalformedFileError] for a last "<" inside the tag before it
|
|
97
|
+
def partial_tag(from, partial_name)
|
|
98
|
+
last = @buffer.byterindex("<")
|
|
99
|
+
return @buffer.bytesize if last.nil? || last < from
|
|
100
|
+
raise inside_attribute if enclosing_tag?(from, last)
|
|
101
|
+
|
|
102
|
+
@buffer.match?(partial_name, last) ? last : @buffer.bytesize
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def discard(position)
|
|
106
|
+
@buffer = @buffer.byteslice(position, @buffer.bytesize - position)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Appends the next chunk of the stream to the buffer.
|
|
110
|
+
# @return [Boolean] false at the end of the stream
|
|
111
|
+
def read_more
|
|
112
|
+
@buffer << @io.readpartial(CHUNK_BYTES).b
|
|
113
|
+
true
|
|
114
|
+
rescue EOFError
|
|
115
|
+
false
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
module Readers
|
|
5
|
+
module Xlsx
|
|
6
|
+
# The +_xHHHH_+ escapes OOXML uses in strings for characters XML cannot carry (control characters): HHHH is a
|
|
7
|
+
# UTF-16 code unit, so a character outside the BMP is a pair of escapes.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
module Escapes
|
|
11
|
+
# A surrogate pair of escapes (high, then low), or a single one.
|
|
12
|
+
PATTERN = /_x(D[89AB]\h\h)__x(D[C-F]\h\h)_|_x(\h{4})_/
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# Decodes the escapes of +text+ in place; a lone surrogate stays as text, since it is not a character.
|
|
17
|
+
# @param text [String] a mutable UTF-8 string
|
|
18
|
+
# @return [String] +text+
|
|
19
|
+
def decode!(text)
|
|
20
|
+
text.gsub!(PATTERN) { decode(Regexp.last_match) } if text.include?("_x")
|
|
21
|
+
text
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def decode(match)
|
|
25
|
+
if match[1]
|
|
26
|
+
((match[1].hex - 0xD800) * 0x400 + (match[2].hex - 0xDC00) + 0x10000).chr(Encoding::UTF_8)
|
|
27
|
+
else
|
|
28
|
+
code = match[3].hex
|
|
29
|
+
code.between?(0xD800, 0xDFFF) ? match[0] : code.chr(Encoding::UTF_8)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
private_class_method :decode
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
module Readers
|
|
5
|
+
module Xlsx
|
|
6
|
+
# Element and attribute names without their namespace prefix, because generators use other prefixes (+x:sheet+,
|
|
7
|
+
# +rel:id+). Read by bytes: a name may be ":" alone or end with ":" (an empty local name), which Ox's
|
|
8
|
+
# +strip_namespace+ refuses as damaged XML or drops, so the handlers that accept such names strip it here.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
module LocalName
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# @param name [String, Symbol]
|
|
15
|
+
# @return [String] "" for a name that ends with ":"
|
|
16
|
+
def of(name)
|
|
17
|
+
name = name.to_s
|
|
18
|
+
colon = name.byterindex(":")
|
|
19
|
+
colon ? name.byteslice(colon + 1, name.bytesize - colon - 1) : name
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @param name [Symbol] a name as Ox symbolizes it
|
|
23
|
+
# @return [Symbol] +name+ itself when it has no prefix, so that the common name costs no String
|
|
24
|
+
def symbol(name)
|
|
25
|
+
text = name.name
|
|
26
|
+
colon = text.byterindex(":") or return name
|
|
27
|
+
text.byteslice(colon + 1, text.bytesize - colon - 1).to_sym
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The memo of local names of a SAX handler (+@local+): every name is noted the first time it comes, so that a
|
|
31
|
+
# document costs one Hash lookup per name, but only the first MEMO_ENTRIES names, so that the memo stays bounded
|
|
32
|
+
# whatever names a document holds. The handler reads +@local[name] || local_name(name)+. (Seeding the memo with
|
|
33
|
+
# the expected names was measured: no faster for unprefixed names, slower for prefixed ones, since Ruby looks
|
|
34
|
+
# up a Hash of up to 8 entries linearly.)
|
|
35
|
+
module Memo
|
|
36
|
+
# The names a memo keeps.
|
|
37
|
+
MEMO_ENTRIES = 64
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def start_memo
|
|
42
|
+
@local = {}
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def local_name(name)
|
|
46
|
+
local = LocalName.symbol(name)
|
|
47
|
+
@local[name] = local if @local.size < MEMO_ENTRIES
|
|
48
|
+
local
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Calcsheet
|
|
4
|
+
module Readers
|
|
5
|
+
module Xlsx
|
|
6
|
+
# Decides from a cell style's number format how a number is read: a plain number, a date, a datetime, a
|
|
7
|
+
# time or duration, or a zero-padded number. XLSX stores all of them as numbers; only the format tells them apart.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
module NumberFormat
|
|
11
|
+
# The built-in ids that are not general. Ids 14-17 and 22 are the dates and datetimes Excel defines, 18-21 and
|
|
12
|
+
# 45-47 are times and durations, and 27-36 and 50-58 are the date formats that depend on the locale (East Asian).
|
|
13
|
+
BUILT_IN = [
|
|
14
|
+
[ 14..17, :date ], [ 18..21, :time ], [ 22..22, :datetime ], [ 27..36, :date ], [ 45..47, :time ], [ 50..58, :date ]
|
|
15
|
+
].each_with_object({}) { |(ids, kind), table| ids.each { |id| table[id] = kind } }.freeze
|
|
16
|
+
|
|
17
|
+
# The pieces of a format code: a quoted text, a backslash escape, spacing (_x) or fill (*x), a bracket section,
|
|
18
|
+
# a section separator or any other character.
|
|
19
|
+
ATOM = /"[^"]*"?|\\.|[_*].|\[[^\]]*\]?|;|./m
|
|
20
|
+
# A bracket of elapsed time: [h], [mm], [ss].
|
|
21
|
+
ELAPSED = /\A\[(?:h+|m+|s+)\]\z/i
|
|
22
|
+
# The AM/PM markers, read as a "p" (a time).
|
|
23
|
+
AM_PM = %r{am/pm|a/p}i
|
|
24
|
+
# A plain zero-padding format: two zeros or more and nothing else.
|
|
25
|
+
ONLY_ZEROS = /\A0{2,}\z/
|
|
26
|
+
# The letter runs that make a format a date (with an "m" that is a month).
|
|
27
|
+
DATE_LETTERS = %w[y d].freeze
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
# @param id [Integer] the +numFmtId+ of the cell style
|
|
31
|
+
# @param code [String, nil] the +formatCode+ of a custom format with that id; it wins over the built-in table
|
|
32
|
+
# @return [Array(Symbol, Integer | nil)] +:general+, +:date+, +:datetime+ or +:time+, and the width of a
|
|
33
|
+
# zero-padding format (+00000+ gives +[:general, 5]+) or nil
|
|
34
|
+
def classify(id, code = nil)
|
|
35
|
+
return [ BUILT_IN.fetch(id, :general), nil ] if code.nil?
|
|
36
|
+
|
|
37
|
+
atoms = first_section(code)
|
|
38
|
+
return [ :general, atoms.join.length ] if atoms.join.match?(ONLY_ZEROS)
|
|
39
|
+
|
|
40
|
+
[ kind_of(atoms), nil ]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
# The pieces of the code before its first unquoted ";".
|
|
46
|
+
def first_section(code)
|
|
47
|
+
code.scan(ATOM).take_while { _1 != ";" }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# The date and time letters left once quotes, escapes, spacing, fill and brackets are gone; an elapsed time
|
|
51
|
+
# bracket ([h], [mm], [ss]) stays as an h or an s.
|
|
52
|
+
def letters(atoms)
|
|
53
|
+
atoms.map { |atom| literal(atom) }.join.gsub(AM_PM, "p").downcase
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def literal(atom)
|
|
57
|
+
case atom
|
|
58
|
+
when /\A["\\_*]/ then ""
|
|
59
|
+
when /\A\[/ then elapsed(atom)
|
|
60
|
+
else atom
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def elapsed(atom)
|
|
65
|
+
return "" unless atom.match?(ELAPSED)
|
|
66
|
+
|
|
67
|
+
atom.downcase.start_with?("[s") ? "s" : "h"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def kind_of(atoms)
|
|
71
|
+
letters = letters(atoms)
|
|
72
|
+
runs = letters.scan(/y+|m+|d+|h+|s+/).map { _1[0] }
|
|
73
|
+
months = runs.each_index.select { |i| runs[i] == "m" && !minutes?(runs, i) }
|
|
74
|
+
date = months.any? || runs.intersect?(DATE_LETTERS)
|
|
75
|
+
time = letters.include?("p") || runs.intersect?(%w[h s]) || runs.count("m") > months.size
|
|
76
|
+
return :datetime if date && time
|
|
77
|
+
|
|
78
|
+
date ? :date : (time ? :time : :general)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# An m is minutes when the run before it is hours or the run after it is seconds; otherwise a month.
|
|
82
|
+
def minutes?(runs, index)
|
|
83
|
+
(index.positive? && runs[index - 1] == "h") || runs[index + 1] == "s"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../errors"
|
|
4
|
+
require_relative "../stream_remainder"
|
|
5
|
+
require_relative "../tag_scan"
|
|
6
|
+
|
|
7
|
+
module Calcsheet
|
|
8
|
+
module Readers
|
|
9
|
+
module Xlsx
|
|
10
|
+
# Finds where a row starts in the inflated XML of a sheet without parsing it (+start_at:+): it scans the
|
|
11
|
+
# bytes for +<row+ start tags, with or without a namespace prefix, and reads their +r+ attribute. Rows before
|
|
12
|
+
# the wanted one cost a regexp scan, not a SAX callback per element. Comments, CDATA sections and processing
|
|
13
|
+
# instructions are passed over, so a +<row+ in their text is not taken for a row; a "<" inside the tag before a
|
|
14
|
+
# row start tag or a markup opening (in an attribute value: not XML, but Ox reads it as text) usually raises
|
|
15
|
+
# MalformedFileError (a tag cut by a chunk boundary may be passed over unchecked; ADR 008: the XML before the
|
|
16
|
+
# wanted row is assumed well-formed). An unescaped ">" inside an attribute value of a row start tag (legal XML
|
|
17
|
+
# that no known generator writes) would end the tag early.
|
|
18
|
+
#
|
|
19
|
+
# Every byte is scanned once and the buffer stays bounded: a row start tag may be at most MAX_TAG_BYTES long,
|
|
20
|
+
# other tags are passed over unbuffered whatever their length, and the text of a comment or CDATA section is
|
|
21
|
+
# dropped as it is read.
|
|
22
|
+
#
|
|
23
|
+
# rest = RowSkipper.new(stream).skip_to(900) # => an IO beginning with '<row r="900"' (or a later row)
|
|
24
|
+
#
|
|
25
|
+
# @api private
|
|
26
|
+
class RowSkipper
|
|
27
|
+
include TagScan
|
|
28
|
+
|
|
29
|
+
# A start tag named row, with any prefix: "<row ", "<x:row>", "<row/>"; not "<rowBreaks" or "</row>"; or "<!"
|
|
30
|
+
# and "<?", which may open a comment, a CDATA section or a processing instruction. One character class for
|
|
31
|
+
# these keeps the scan as fast as one for row tags alone.
|
|
32
|
+
MARKUP = /<(?:[!?]|(?:[A-Za-z_][\w.-]*:)?row[\s>\/])/n
|
|
33
|
+
# The +r+ attribute of a row start tag.
|
|
34
|
+
NUMBER = /\sr\s*=\s*(?:"(\d+)"|'(\d+)')/n
|
|
35
|
+
# What may still become a row start tag or a markup opening at the end of the buffer: "<" and the name read so
|
|
36
|
+
# far, whatever the length of its namespace prefix; and the byte that ends such a name.
|
|
37
|
+
PARTIAL_NAME = /\G<[\w.:-]*\z/n
|
|
38
|
+
NAME_END = /[^\w.:-]/n
|
|
39
|
+
|
|
40
|
+
# The state of a scan: +@buffer+ holds the bytes read and not yet discarded (TagScan), +@last_row+ the number of
|
|
41
|
+
# the last row passed over (for errors), +@unnumbered+ whether a row without +r+ stopped the scan.
|
|
42
|
+
#
|
|
43
|
+
# @param io [#readpartial] the inflated sheet XML from its start; read forward only, never closed
|
|
44
|
+
# @param sheet [String, nil] the sheet name, for errors
|
|
45
|
+
def initialize(io, sheet: nil)
|
|
46
|
+
@io = io
|
|
47
|
+
@sheet = sheet
|
|
48
|
+
@unnumbered = false
|
|
49
|
+
@last_row = nil
|
|
50
|
+
@buffer = String.new(encoding: Encoding::BINARY)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @return [Boolean] whether #skip_to met a row without +r+, so rows cannot be found by number
|
|
54
|
+
def unnumbered?
|
|
55
|
+
@unnumbered
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Discards the bytes before the start tag of the first row numbered +row+ or later.
|
|
59
|
+
#
|
|
60
|
+
# @param row [Integer] a row number
|
|
61
|
+
# @param prefix [String] binary bytes the rest of the stream is handed out after
|
|
62
|
+
# @param bound [LimitedInflater::TokenBound, nil] for the StreamRemainder
|
|
63
|
+
# @return [StreamRemainder, nil] +prefix+ and the rest of the stream from that start tag; nil when no such row
|
|
64
|
+
# exists, or when a row without +r+ comes first (then #unnumbered? is true and the caller must parse from the
|
|
65
|
+
# start)
|
|
66
|
+
# @raise [MalformedFileError] for a row start tag longer than MAX_TAG_BYTES
|
|
67
|
+
def skip_to(row, prefix: "".b, bound: nil)
|
|
68
|
+
from = 0
|
|
69
|
+
loop do
|
|
70
|
+
start = @buffer.byteindex(MARKUP, from)
|
|
71
|
+
if start.nil?
|
|
72
|
+
discard(partial_tag(from, PARTIAL_NAME))
|
|
73
|
+
return unless @buffer.match?(PARTIAL_NAME, 0) ? name_end : read_more
|
|
74
|
+
|
|
75
|
+
from = 0
|
|
76
|
+
elsif enclosing_tag?(from, start)
|
|
77
|
+
raise inside_attribute
|
|
78
|
+
elsif (markup = markup_at(start))
|
|
79
|
+
case markup
|
|
80
|
+
when :other then from = start + 1
|
|
81
|
+
when :incomplete
|
|
82
|
+
discard(start)
|
|
83
|
+
return unless read_more
|
|
84
|
+
|
|
85
|
+
from = 0
|
|
86
|
+
else from = skip_markup(start, *markup) or return
|
|
87
|
+
end
|
|
88
|
+
else
|
|
89
|
+
close = @buffer.byteindex(">", start)
|
|
90
|
+
unless close
|
|
91
|
+
discard(start)
|
|
92
|
+
start = 0
|
|
93
|
+
close = tag_end or return
|
|
94
|
+
end
|
|
95
|
+
raise too_long if close - start >= MAX_TAG_BYTES
|
|
96
|
+
|
|
97
|
+
number = row_number(@buffer.byteslice(start, close - start))
|
|
98
|
+
return unnumbered! if number.nil?
|
|
99
|
+
if number >= row
|
|
100
|
+
return StreamRemainder.new(prefix + @buffer.byteslice(start, @buffer.bytesize - start), @io, bound: bound)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
@last_row = number
|
|
104
|
+
from = close + 1
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
private
|
|
110
|
+
|
|
111
|
+
def row_number(tag)
|
|
112
|
+
match = NUMBER.match(tag) or return
|
|
113
|
+
(match[1] || match[2]).to_i
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def unnumbered!
|
|
117
|
+
@unnumbered = true
|
|
118
|
+
nil
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The buffer begins with a row start tag whose ">" has not been read: reads on until it, scanning each
|
|
122
|
+
# chunk once.
|
|
123
|
+
# @return [Integer, nil] the position of the ">"; nil when the stream ends first
|
|
124
|
+
# @raise [MalformedFileError] when the tag is already longer than MAX_TAG_BYTES (the caller checks the tag
|
|
125
|
+
# that ends at the position returned)
|
|
126
|
+
def tag_end
|
|
127
|
+
loop do
|
|
128
|
+
from = @buffer.bytesize
|
|
129
|
+
raise too_long if from >= MAX_TAG_BYTES
|
|
130
|
+
return unless read_more
|
|
131
|
+
|
|
132
|
+
close = @buffer.byteindex(">", from)
|
|
133
|
+
return close if close
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def too_long
|
|
138
|
+
located("a row start tag is longer than #{MAX_TAG_BYTES} bytes")
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def located(message)
|
|
142
|
+
return MalformedFileError.at(message, sheet: @sheet, row: @last_row) if @sheet
|
|
143
|
+
|
|
144
|
+
MalformedFileError.new("the sheet#{", row #{@last_row}" if @last_row}: #{message}", row: @last_row)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# The buffer is "<" and what may be the beginning of a row start tag's name: reads on until the name ends,
|
|
148
|
+
# scanning each chunk once, so that the next scan of the buffer can tell whether it is a row.
|
|
149
|
+
# @return [Boolean] false when the stream ends first
|
|
150
|
+
# @raise [MalformedFileError] when the name alone is longer than MAX_TAG_BYTES
|
|
151
|
+
def name_end
|
|
152
|
+
loop do
|
|
153
|
+
from = @buffer.bytesize
|
|
154
|
+
raise long_name if from > MAX_TAG_BYTES
|
|
155
|
+
return false unless read_more
|
|
156
|
+
return true if @buffer.byteindex(NAME_END, from)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ox"
|
|
4
|
+
require_relative "../../configuration"
|
|
5
|
+
require_relative "../../errors"
|
|
6
|
+
require_relative "../ox_errors"
|
|
7
|
+
require_relative "escapes"
|
|
8
|
+
require_relative "local_name"
|
|
9
|
+
require_relative "../references"
|
|
10
|
+
|
|
11
|
+
module Calcsheet
|
|
12
|
+
module Readers
|
|
13
|
+
module Xlsx
|
|
14
|
+
# The shared strings table of an XLSX file (+xl/sharedStrings.xml+), kept compactly: the UTF-8 bytes of all
|
|
15
|
+
# strings in one binary buffer and the end offset of each string packed as 64-bit integers, so a million
|
|
16
|
+
# strings cost about their text plus 8 bytes each instead of a Ruby String (40+ bytes) apiece.
|
|
17
|
+
#
|
|
18
|
+
# Rich text runs are joined, phonetic runs (+rPh+) are skipped, whitespace is kept exactly, and the
|
|
19
|
+
# +_xHHHH_+ escapes OOXML uses for control characters are decoded.
|
|
20
|
+
#
|
|
21
|
+
# The offsets hold +size+ + 1 values: 0, then the end of each string, so string +i+ is the bytes from offset +i+
|
|
22
|
+
# to offset +i+ + 1 of the buffer. The strings are tagged UTF-8 as Ox hands them over, not validated: the reader
|
|
23
|
+
# of a cell checks the one it takes (SheetParser), so that a table is not checked in full for the strings a read
|
|
24
|
+
# never uses.
|
|
25
|
+
#
|
|
26
|
+
# @api private
|
|
27
|
+
class SharedStrings
|
|
28
|
+
# The pack directive of an offset: an unsigned 64-bit integer, native byte order (the table never leaves the
|
|
29
|
+
# process).
|
|
30
|
+
OFFSET_DIRECTIVE = "Q"
|
|
31
|
+
OFFSET_BYTES = 8
|
|
32
|
+
# The Builder joins the text in chunks of about this many bytes, and packs the offsets this many at a time.
|
|
33
|
+
CHUNK_BYTES = 1_048_576
|
|
34
|
+
OFFSETS_PER_CHUNK = 65_536
|
|
35
|
+
|
|
36
|
+
# @param io [#readpartial] the shared strings XML, for example +ZipContainer#stream+
|
|
37
|
+
# @param max_cell_bytes [Integer, nil] the longest string, as the XML parser hands its text over; nil for no
|
|
38
|
+
# limit
|
|
39
|
+
# @return [SharedStrings]
|
|
40
|
+
# @raise [MalformedFileError] for damaged or empty XML (the parser or stream error is the +cause+) and for a
|
|
41
|
+
# string longer than +max_cell_bytes+
|
|
42
|
+
# @raise [Calcsheet::Error] the error the stream itself raises, such as a ZipBombError (Ox wraps it in a
|
|
43
|
+
# +TypeError+; it is raised again as itself)
|
|
44
|
+
def self.load(io, max_cell_bytes: Configuration::DEFAULTS[:max_cell_bytes])
|
|
45
|
+
builder = Builder.new(max_cell_bytes)
|
|
46
|
+
begin
|
|
47
|
+
# convert_special: false, so that the references reach the builder as written (see References).
|
|
48
|
+
Ox.sax_parse(builder, io, skip: :skip_none, convert_special: false, symbolize: true)
|
|
49
|
+
rescue *OxErrors::RAISED => error
|
|
50
|
+
OxErrors.raise_translated(error) { MalformedFileError.new("the shared strings cannot be parsed: #{_1}") }
|
|
51
|
+
end
|
|
52
|
+
builder.result
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# @return [Integer] the number of strings
|
|
56
|
+
attr_reader :size
|
|
57
|
+
|
|
58
|
+
# @api private
|
|
59
|
+
# @return [Integer] the bytes of the decoded text of all strings (a test seam: the readers do not use it)
|
|
60
|
+
attr_reader :bytesize
|
|
61
|
+
|
|
62
|
+
# @api private
|
|
63
|
+
def initialize(buffer, offsets, size)
|
|
64
|
+
@buffer = buffer
|
|
65
|
+
@offsets = offsets
|
|
66
|
+
@size = size
|
|
67
|
+
@bytesize = buffer.bytesize
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @param index [Integer] the 0-based position of the string, the +v+ of a +t="s"+ cell
|
|
71
|
+
# @return [String] a new frozen String tagged UTF-8 (not validated)
|
|
72
|
+
# @raise [MalformedFileError] when the index is outside the table
|
|
73
|
+
def [](index)
|
|
74
|
+
unless index.is_a?(Integer) && index >= 0 && index < @size
|
|
75
|
+
raise MalformedFileError, "shared string #{index.inspect} is outside the table of #{@size} strings"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
unchecked(index)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @api private
|
|
82
|
+
# The string at an index the caller checked, as SheetParser does: without the checks of #[].
|
|
83
|
+
# @param index [Integer] 0 to +size+ - 1
|
|
84
|
+
# @return [String] a new frozen String tagged UTF-8 (not validated)
|
|
85
|
+
def unchecked(index)
|
|
86
|
+
position = index * OFFSET_BYTES
|
|
87
|
+
from = @offsets.unpack1(OFFSET_DIRECTIVE, offset: position)
|
|
88
|
+
to = @offsets.unpack1(OFFSET_DIRECTIVE, offset: position + OFFSET_BYTES)
|
|
89
|
+
@buffer.byteslice(from, to - from).force_encoding(Encoding::UTF_8).freeze
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Collects the strings while Ox streams the XML. Text and offsets go to chunks that are joined once at the
|
|
93
|
+
# end, so neither buffer keeps the spare capacity of a growing String.
|
|
94
|
+
# @api private
|
|
95
|
+
class Builder < Ox::Sax
|
|
96
|
+
include LocalName::Memo
|
|
97
|
+
|
|
98
|
+
# +@chunk+ collects the text of the strings, +@offsets+ the end offset of each (+@total+, the bytes so far);
|
|
99
|
+
# both move to +@chunks+ and +@offset_chunks+ when they reach CHUNK_BYTES and OFFSETS_PER_CHUNK. +@item+ is
|
|
100
|
+
# the string of the open +si+ (nil outside one), +@in_text+ whether a text belongs to it, +@phonetic+ the
|
|
101
|
+
# depth of +rPh+ runs, +@count+ the strings so far and +@any_element+ whether the document had an element.
|
|
102
|
+
def initialize(max_cell_bytes)
|
|
103
|
+
super()
|
|
104
|
+
start_memo
|
|
105
|
+
@max_cell_bytes = max_cell_bytes
|
|
106
|
+
@chunks = []
|
|
107
|
+
@chunk = +""
|
|
108
|
+
@offsets = []
|
|
109
|
+
@offset_chunks = []
|
|
110
|
+
@total = 0
|
|
111
|
+
@count = 0
|
|
112
|
+
@item = nil
|
|
113
|
+
@in_text = false
|
|
114
|
+
@phonetic = 0
|
|
115
|
+
@any_element = false
|
|
116
|
+
@offsets << 0
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Ox reports damaged XML here instead of raising.
|
|
120
|
+
def error(message, line, column)
|
|
121
|
+
raise MalformedFileError.new("the shared strings cannot be parsed: #{message} (line #{line})"),
|
|
122
|
+
cause: OxErrors.parse_error(message, line, column)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Ox's callbacks (start_element, end_element, text, cdata) collect the strings.
|
|
126
|
+
def start_element(name)
|
|
127
|
+
@any_element = true
|
|
128
|
+
case @local[name] || local_name(name)
|
|
129
|
+
when :si then @item = +""
|
|
130
|
+
when :rPh then @phonetic += 1 if @item
|
|
131
|
+
when :t then @in_text = true if @item && @phonetic.zero?
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def end_element(name)
|
|
136
|
+
case @local[name] || local_name(name)
|
|
137
|
+
when :si then finish_item if @item
|
|
138
|
+
when :rPh then @phonetic -= 1 if @item && @phonetic.positive?
|
|
139
|
+
when :t then @in_text = false
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def text(value)
|
|
144
|
+
return unless @in_text
|
|
145
|
+
|
|
146
|
+
value = References.line_ends(value) if value.include?("\r")
|
|
147
|
+
append(value.include?("&") ? references(value) : value)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def cdata(value)
|
|
151
|
+
append(value.include?("\r") ? References.line_ends(value) : value) if @in_text
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# @return [SharedStrings]
|
|
155
|
+
# @raise [MalformedFileError] when the document has no element
|
|
156
|
+
def result
|
|
157
|
+
raise MalformedFileError, "the shared strings document is empty" unless @any_element
|
|
158
|
+
|
|
159
|
+
flush_chunk
|
|
160
|
+
flush_offsets
|
|
161
|
+
SharedStrings.new(@chunks.join.force_encoding(Encoding::BINARY), @offset_chunks.join, @count)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
private
|
|
165
|
+
|
|
166
|
+
def references(value)
|
|
167
|
+
References.decode(value)
|
|
168
|
+
rescue References::Invalid => error
|
|
169
|
+
raise MalformedFileError, "the shared strings: shared string #{@count} holds #{error.message}"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Measured before it is joined: a string longer than max_cell_bytes is never built.
|
|
173
|
+
def append(value)
|
|
174
|
+
if @max_cell_bytes && @item.bytesize + value.bytesize > @max_cell_bytes
|
|
175
|
+
raise MalformedFileError, "the shared strings: shared string #{@count} is longer than max_cell_bytes " \
|
|
176
|
+
"(#{@max_cell_bytes} bytes)"
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
@item << value
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def finish_item
|
|
183
|
+
Escapes.decode!(@item)
|
|
184
|
+
@chunk << @item
|
|
185
|
+
@total += @item.bytesize
|
|
186
|
+
@offsets << @total
|
|
187
|
+
@count += 1
|
|
188
|
+
@item = nil
|
|
189
|
+
@in_text = false
|
|
190
|
+
@phonetic = 0
|
|
191
|
+
flush_chunk if @chunk.bytesize >= CHUNK_BYTES
|
|
192
|
+
flush_offsets if @offsets.size >= OFFSETS_PER_CHUNK
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def flush_chunk
|
|
196
|
+
@chunks << @chunk
|
|
197
|
+
@chunk = +""
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def flush_offsets
|
|
201
|
+
@offset_chunks << @offsets.pack("#{OFFSET_DIRECTIVE}*")
|
|
202
|
+
@offsets.clear
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
private_constant :Builder
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|