calcsheet 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +9 -0
  3. data/CHANGELOG.md +62 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +921 -0
  6. data/lib/calcsheet/book.rb +164 -0
  7. data/lib/calcsheet/cast.rb +578 -0
  8. data/lib/calcsheet/column.rb +117 -0
  9. data/lib/calcsheet/column_letter.rb +34 -0
  10. data/lib/calcsheet/configuration.rb +186 -0
  11. data/lib/calcsheet/decimal_text.rb +19 -0
  12. data/lib/calcsheet/entry_name.rb +28 -0
  13. data/lib/calcsheet/errors.rb +186 -0
  14. data/lib/calcsheet/excel_serial.rb +65 -0
  15. data/lib/calcsheet/format_detector.rb +109 -0
  16. data/lib/calcsheet/formula_protection.rb +31 -0
  17. data/lib/calcsheet/grid.rb +20 -0
  18. data/lib/calcsheet/header_keys.rb +66 -0
  19. data/lib/calcsheet/limited_inflater.rb +326 -0
  20. data/lib/calcsheet/merge_check.rb +41 -0
  21. data/lib/calcsheet/odf.rb +45 -0
  22. data/lib/calcsheet/option_validation.rb +160 -0
  23. data/lib/calcsheet/options.rb +10 -0
  24. data/lib/calcsheet/raw_row.rb +18 -0
  25. data/lib/calcsheet/raw_types.rb +15 -0
  26. data/lib/calcsheet/readers/csv_dialect.rb +264 -0
  27. data/lib/calcsheet/readers/csv_input.rb +128 -0
  28. data/lib/calcsheet/readers/csv_reader.rb +239 -0
  29. data/lib/calcsheet/readers/ods/cell_formats.rb +115 -0
  30. data/lib/calcsheet/readers/ods/content_parser.rb +594 -0
  31. data/lib/calcsheet/readers/ods/count.rb +30 -0
  32. data/lib/calcsheet/readers/ods/cursor.rb +121 -0
  33. data/lib/calcsheet/readers/ods/patterns.rb +70 -0
  34. data/lib/calcsheet/readers/ods/row_seek.rb +157 -0
  35. data/lib/calcsheet/readers/ods/scanner.rb +302 -0
  36. data/lib/calcsheet/readers/ods/style_scan.rb +108 -0
  37. data/lib/calcsheet/readers/ods/tag_text.rb +76 -0
  38. data/lib/calcsheet/readers/ods_reader.rb +71 -0
  39. data/lib/calcsheet/readers/ox_errors.rb +40 -0
  40. data/lib/calcsheet/readers/references.rb +62 -0
  41. data/lib/calcsheet/readers/row_batches.rb +45 -0
  42. data/lib/calcsheet/readers/sax_sheet.rb +108 -0
  43. data/lib/calcsheet/readers/stream_remainder.rb +66 -0
  44. data/lib/calcsheet/readers/tag_scan.rb +119 -0
  45. data/lib/calcsheet/readers/xlsx/escapes.rb +36 -0
  46. data/lib/calcsheet/readers/xlsx/local_name.rb +54 -0
  47. data/lib/calcsheet/readers/xlsx/number_format.rb +89 -0
  48. data/lib/calcsheet/readers/xlsx/row_skipper.rb +162 -0
  49. data/lib/calcsheet/readers/xlsx/shared_strings.rb +209 -0
  50. data/lib/calcsheet/readers/xlsx/sheet_parser.rb +435 -0
  51. data/lib/calcsheet/readers/xlsx/styles.rb +156 -0
  52. data/lib/calcsheet/readers/xlsx/workbook_meta.rb +228 -0
  53. data/lib/calcsheet/readers/xlsx_reader.rb +122 -0
  54. data/lib/calcsheet/reading.rb +281 -0
  55. data/lib/calcsheet/row.rb +110 -0
  56. data/lib/calcsheet/sample.rb +10 -0
  57. data/lib/calcsheet/schema.rb +140 -0
  58. data/lib/calcsheet/sheet_info.rb +21 -0
  59. data/lib/calcsheet/sheet_names.rb +114 -0
  60. data/lib/calcsheet/sheet_selection.rb +58 -0
  61. data/lib/calcsheet/source.rb +263 -0
  62. data/lib/calcsheet/stream_body.rb +81 -0
  63. data/lib/calcsheet/time_zone.rb +96 -0
  64. data/lib/calcsheet/untrusted.rb +32 -0
  65. data/lib/calcsheet/version.rb +6 -0
  66. data/lib/calcsheet/write_book.rb +202 -0
  67. data/lib/calcsheet/write_sheet.rb +282 -0
  68. data/lib/calcsheet/writers/csv_writer.rb +138 -0
  69. data/lib/calcsheet/writers/ods_writer.rb +437 -0
  70. data/lib/calcsheet/writers/xlsx_writer.rb +350 -0
  71. data/lib/calcsheet/writers/xml_text.rb +35 -0
  72. data/lib/calcsheet/writers/zip_buffered.rb +55 -0
  73. data/lib/calcsheet/zip_container.rb +190 -0
  74. data/lib/calcsheet/zip_directory.rb +202 -0
  75. data/lib/calcsheet/zip_end_record.rb +117 -0
  76. data/lib/calcsheet.rb +295 -0
  77. metadata +200 -0
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ module Readers
5
+ # What the byte scanners (Xlsx::RowSkipper, Ods::Cursor) share: they search a buffer of the inflated XML for the
6
+ # opening of the tags they want, pass over comments, CDATA sections and processing instructions, and keep only
7
+ # what a tag cut by the end of a chunk needs. The including class keeps the binary +@buffer+ and the stream +@io+,
8
+ # and defines +located(message)+, the MalformedFileError of a message at its position.
9
+ #
10
+ # @api private
11
+ module TagScan
12
+ # The bytes read from the stream at a time.
13
+ CHUNK_BYTES = 65_536
14
+ # The longest tag a scanner buffers: a wanted start tag, or the name of one cut by a chunk boundary.
15
+ MAX_TAG_BYTES = 65_536
16
+ # A start or end tag from "<" to ">", with ">" allowed inside quoted attribute values.
17
+ TAG = /\G<(?>[^>"']*)(?>(?:"[^"]*"|'[^']*')(?>[^>"']*))*>/n
18
+ # A tag without quoted values or another "<", such as an end tag.
19
+ PLAIN_TAG = /\G<[^<>"']*>/n
20
+ # The bytes after "<" that open a processing instruction, and a comment, CDATA section or declaration.
21
+ QUESTION_MARK = 0x3F
22
+ EXCLAMATION_MARK = 0x21
23
+ # What ends the markup that starts with "<?", "<!--" and "<![CDATA[", and the length of each opening.
24
+ INSTRUCTION = [ "?>", 2 ].freeze
25
+ COMMENT = [ "-->", 4 ].freeze
26
+ CDATA = [ "]]>", 9 ].freeze
27
+ COMMENT_OPENING = "<!--".b.freeze
28
+ CDATA_OPENING = "<![CDATA[".b.freeze
29
+ # The "<!" markup whose text is passed over, with its terminator and opening length; another "<!" (a DOCTYPE)
30
+ # is passed over as a tag.
31
+ DECLARATIONS = [ [ COMMENT_OPENING, COMMENT ].freeze, [ CDATA_OPENING, CDATA ].freeze ].freeze
32
+
33
+ private
34
+
35
+ # @return [Array(String, Integer), Symbol, nil] the terminator and opening length of the markup at +start+;
36
+ # :incomplete when the buffer ends inside what may be an opening, :other for another "<!" declaration, nil
37
+ # for a tag
38
+ def markup_at(start)
39
+ case @buffer.getbyte(start + 1)
40
+ when QUESTION_MARK then INSTRUCTION
41
+ when EXCLAMATION_MARK
42
+ DECLARATIONS.each do |opening, markup|
43
+ piece = @buffer.byteslice(start, opening.bytesize)
44
+ return markup if piece == opening
45
+ return :incomplete if piece.bytesize < opening.bytesize && opening.start_with?(piece)
46
+ end
47
+ :other
48
+ end
49
+ end
50
+
51
+ # Whether the "<" at +start+ (a wanted tag or a markup opening) is not preceded by a complete tag: the search
52
+ # passes over the other tags without reading them, so the "<" before +start+, read with TAG's quoted values,
53
+ # must end before +start+. Otherwise +start+ is inside an attribute value, where a "<" is not XML but Ox would
54
+ # read it as text, and the scan cannot tell what the parser will see. +from+, where the search started, is
55
+ # always at a tag boundary (after a tag or markup, or at a "<" kept by #partial_tag), so the "<" before +start+
56
+ # is a tag's.
57
+ def enclosing_tag?(from, start)
58
+ return false if start == from
59
+
60
+ last = @buffer.byterindex("<", start - 1)
61
+ return false if last.nil? || last < from
62
+ # The common case without a MatchData: a tag without quotes (an end tag) that ends before the next "<".
63
+ return false if PLAIN_TAG.match?(@buffer, last)
64
+
65
+ match = TAG.match(@buffer, last)
66
+ !(match && match.end(0) <= start)
67
+ end
68
+
69
+ def inside_attribute
70
+ located("a \"<\" inside a tag, which XML does not allow")
71
+ end
72
+
73
+ def long_name
74
+ located("an element name is longer than #{MAX_TAG_BYTES} bytes")
75
+ end
76
+
77
+ # Reads past the markup at +start+, dropping its text as it goes.
78
+ # @return [Integer, nil] the position after its terminator; nil when the stream ends first
79
+ def skip_markup(start, terminator, opening)
80
+ from = start + opening
81
+ loop do
82
+ found = @buffer.byteindex(terminator, from)
83
+ return found + terminator.bytesize if found
84
+
85
+ # Keep only what could be the beginning of a terminator split between chunks.
86
+ discard([ from, @buffer.bytesize - terminator.bytesize + 1 ].max)
87
+ from = 0
88
+ return unless read_more
89
+ end
90
+ end
91
+
92
+ # Where a wanted tag cut by the end of the buffer may begin: the last "<" after +from+ when only name bytes
93
+ # (+partial_name+, anchored at it) follow it; the end of the buffer otherwise. A tag whose name has ended is not
94
+ # wanted (the search would have found it), so it is passed over unbuffered, whatever its length: the scan then
95
+ # resumes inside it, which is safe for well-formed XML, where no "<" occurs inside a tag (see #enclosing_tag?).
96
+ # @raise [MalformedFileError] for a last "<" inside the tag before it
97
+ def partial_tag(from, partial_name)
98
+ last = @buffer.byterindex("<")
99
+ return @buffer.bytesize if last.nil? || last < from
100
+ raise inside_attribute if enclosing_tag?(from, last)
101
+
102
+ @buffer.match?(partial_name, last) ? last : @buffer.bytesize
103
+ end
104
+
105
+ def discard(position)
106
+ @buffer = @buffer.byteslice(position, @buffer.bytesize - position)
107
+ end
108
+
109
+ # Appends the next chunk of the stream to the buffer.
110
+ # @return [Boolean] false at the end of the stream
111
+ def read_more
112
+ @buffer << @io.readpartial(CHUNK_BYTES).b
113
+ true
114
+ rescue EOFError
115
+ false
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ module Readers
5
+ module Xlsx
6
+ # The +_xHHHH_+ escapes OOXML uses in strings for characters XML cannot carry (control characters): HHHH is a
7
+ # UTF-16 code unit, so a character outside the BMP is a pair of escapes.
8
+ #
9
+ # @api private
10
+ module Escapes
11
+ # A surrogate pair of escapes (high, then low), or a single one.
12
+ PATTERN = /_x(D[89AB]\h\h)__x(D[C-F]\h\h)_|_x(\h{4})_/
13
+
14
+ module_function
15
+
16
+ # Decodes the escapes of +text+ in place; a lone surrogate stays as text, since it is not a character.
17
+ # @param text [String] a mutable UTF-8 string
18
+ # @return [String] +text+
19
+ def decode!(text)
20
+ text.gsub!(PATTERN) { decode(Regexp.last_match) } if text.include?("_x")
21
+ text
22
+ end
23
+
24
+ def decode(match)
25
+ if match[1]
26
+ ((match[1].hex - 0xD800) * 0x400 + (match[2].hex - 0xDC00) + 0x10000).chr(Encoding::UTF_8)
27
+ else
28
+ code = match[3].hex
29
+ code.between?(0xD800, 0xDFFF) ? match[0] : code.chr(Encoding::UTF_8)
30
+ end
31
+ end
32
+ private_class_method :decode
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ module Readers
5
+ module Xlsx
6
+ # Element and attribute names without their namespace prefix, because generators use other prefixes (+x:sheet+,
7
+ # +rel:id+). Read by bytes: a name may be ":" alone or end with ":" (an empty local name), which Ox's
8
+ # +strip_namespace+ refuses as damaged XML or drops, so the handlers that accept such names strip it here.
9
+ #
10
+ # @api private
11
+ module LocalName
12
+ module_function
13
+
14
+ # @param name [String, Symbol]
15
+ # @return [String] "" for a name that ends with ":"
16
+ def of(name)
17
+ name = name.to_s
18
+ colon = name.byterindex(":")
19
+ colon ? name.byteslice(colon + 1, name.bytesize - colon - 1) : name
20
+ end
21
+
22
+ # @param name [Symbol] a name as Ox symbolizes it
23
+ # @return [Symbol] +name+ itself when it has no prefix, so that the common name costs no String
24
+ def symbol(name)
25
+ text = name.name
26
+ colon = text.byterindex(":") or return name
27
+ text.byteslice(colon + 1, text.bytesize - colon - 1).to_sym
28
+ end
29
+
30
+ # The memo of local names of a SAX handler (+@local+): every name is noted the first time it comes, so that a
31
+ # document costs one Hash lookup per name, but only the first MEMO_ENTRIES names, so that the memo stays bounded
32
+ # whatever names a document holds. The handler reads +@local[name] || local_name(name)+. (Seeding the memo with
33
+ # the expected names was measured: no faster for unprefixed names, slower for prefixed ones, since Ruby looks
34
+ # up a Hash of up to 8 entries linearly.)
35
+ module Memo
36
+ # The names a memo keeps.
37
+ MEMO_ENTRIES = 64
38
+
39
+ private
40
+
41
+ def start_memo
42
+ @local = {}
43
+ end
44
+
45
+ def local_name(name)
46
+ local = LocalName.symbol(name)
47
+ @local[name] = local if @local.size < MEMO_ENTRIES
48
+ local
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Calcsheet
4
+ module Readers
5
+ module Xlsx
6
+ # Decides from a cell style's number format how a number is read: a plain number, a date, a datetime, a
7
+ # time or duration, or a zero-padded number. XLSX stores all of them as numbers; only the format tells them apart.
8
+ #
9
+ # @api private
10
+ module NumberFormat
11
+ # The built-in ids that are not general. Ids 14-17 and 22 are the dates and datetimes Excel defines, 18-21 and
12
+ # 45-47 are times and durations, and 27-36 and 50-58 are the date formats that depend on the locale (East Asian).
13
+ BUILT_IN = [
14
+ [ 14..17, :date ], [ 18..21, :time ], [ 22..22, :datetime ], [ 27..36, :date ], [ 45..47, :time ], [ 50..58, :date ]
15
+ ].each_with_object({}) { |(ids, kind), table| ids.each { |id| table[id] = kind } }.freeze
16
+
17
+ # The pieces of a format code: a quoted text, a backslash escape, spacing (_x) or fill (*x), a bracket section,
18
+ # a section separator or any other character.
19
+ ATOM = /"[^"]*"?|\\.|[_*].|\[[^\]]*\]?|;|./m
20
+ # A bracket of elapsed time: [h], [mm], [ss].
21
+ ELAPSED = /\A\[(?:h+|m+|s+)\]\z/i
22
+ # The AM/PM markers, read as a "p" (a time).
23
+ AM_PM = %r{am/pm|a/p}i
24
+ # A plain zero-padding format: two zeros or more and nothing else.
25
+ ONLY_ZEROS = /\A0{2,}\z/
26
+ # The letter runs that make a format a date (with an "m" that is a month).
27
+ DATE_LETTERS = %w[y d].freeze
28
+
29
+ class << self
30
+ # @param id [Integer] the +numFmtId+ of the cell style
31
+ # @param code [String, nil] the +formatCode+ of a custom format with that id; it wins over the built-in table
32
+ # @return [Array(Symbol, Integer | nil)] +:general+, +:date+, +:datetime+ or +:time+, and the width of a
33
+ # zero-padding format (+00000+ gives +[:general, 5]+) or nil
34
+ def classify(id, code = nil)
35
+ return [ BUILT_IN.fetch(id, :general), nil ] if code.nil?
36
+
37
+ atoms = first_section(code)
38
+ return [ :general, atoms.join.length ] if atoms.join.match?(ONLY_ZEROS)
39
+
40
+ [ kind_of(atoms), nil ]
41
+ end
42
+
43
+ private
44
+
45
+ # The pieces of the code before its first unquoted ";".
46
+ def first_section(code)
47
+ code.scan(ATOM).take_while { _1 != ";" }
48
+ end
49
+
50
+ # The date and time letters left once quotes, escapes, spacing, fill and brackets are gone; an elapsed time
51
+ # bracket ([h], [mm], [ss]) stays as an h or an s.
52
+ def letters(atoms)
53
+ atoms.map { |atom| literal(atom) }.join.gsub(AM_PM, "p").downcase
54
+ end
55
+
56
+ def literal(atom)
57
+ case atom
58
+ when /\A["\\_*]/ then ""
59
+ when /\A\[/ then elapsed(atom)
60
+ else atom
61
+ end
62
+ end
63
+
64
+ def elapsed(atom)
65
+ return "" unless atom.match?(ELAPSED)
66
+
67
+ atom.downcase.start_with?("[s") ? "s" : "h"
68
+ end
69
+
70
+ def kind_of(atoms)
71
+ letters = letters(atoms)
72
+ runs = letters.scan(/y+|m+|d+|h+|s+/).map { _1[0] }
73
+ months = runs.each_index.select { |i| runs[i] == "m" && !minutes?(runs, i) }
74
+ date = months.any? || runs.intersect?(DATE_LETTERS)
75
+ time = letters.include?("p") || runs.intersect?(%w[h s]) || runs.count("m") > months.size
76
+ return :datetime if date && time
77
+
78
+ date ? :date : (time ? :time : :general)
79
+ end
80
+
81
+ # An m is minutes when the run before it is hours or the run after it is seconds; otherwise a month.
82
+ def minutes?(runs, index)
83
+ (index.positive? && runs[index - 1] == "h") || runs[index + 1] == "s"
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,162 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../../errors"
4
+ require_relative "../stream_remainder"
5
+ require_relative "../tag_scan"
6
+
7
+ module Calcsheet
8
+ module Readers
9
+ module Xlsx
10
+ # Finds where a row starts in the inflated XML of a sheet without parsing it (+start_at:+): it scans the
11
+ # bytes for +<row+ start tags, with or without a namespace prefix, and reads their +r+ attribute. Rows before
12
+ # the wanted one cost a regexp scan, not a SAX callback per element. Comments, CDATA sections and processing
13
+ # instructions are passed over, so a +<row+ in their text is not taken for a row; a "<" inside the tag before a
14
+ # row start tag or a markup opening (in an attribute value: not XML, but Ox reads it as text) usually raises
15
+ # MalformedFileError (a tag cut by a chunk boundary may be passed over unchecked; ADR 008: the XML before the
16
+ # wanted row is assumed well-formed). An unescaped ">" inside an attribute value of a row start tag (legal XML
17
+ # that no known generator writes) would end the tag early.
18
+ #
19
+ # Every byte is scanned once and the buffer stays bounded: a row start tag may be at most MAX_TAG_BYTES long,
20
+ # other tags are passed over unbuffered whatever their length, and the text of a comment or CDATA section is
21
+ # dropped as it is read.
22
+ #
23
+ # rest = RowSkipper.new(stream).skip_to(900) # => an IO beginning with '<row r="900"' (or a later row)
24
+ #
25
+ # @api private
26
+ class RowSkipper
27
+ include TagScan
28
+
29
+ # A start tag named row, with any prefix: "<row ", "<x:row>", "<row/>"; not "<rowBreaks" or "</row>"; or "<!"
30
+ # and "<?", which may open a comment, a CDATA section or a processing instruction. One character class for
31
+ # these keeps the scan as fast as one for row tags alone.
32
+ MARKUP = /<(?:[!?]|(?:[A-Za-z_][\w.-]*:)?row[\s>\/])/n
33
+ # The +r+ attribute of a row start tag.
34
+ NUMBER = /\sr\s*=\s*(?:"(\d+)"|'(\d+)')/n
35
+ # What may still become a row start tag or a markup opening at the end of the buffer: "<" and the name read so
36
+ # far, whatever the length of its namespace prefix; and the byte that ends such a name.
37
+ PARTIAL_NAME = /\G<[\w.:-]*\z/n
38
+ NAME_END = /[^\w.:-]/n
39
+
40
+ # The state of a scan: +@buffer+ holds the bytes read and not yet discarded (TagScan), +@last_row+ the number of
41
+ # the last row passed over (for errors), +@unnumbered+ whether a row without +r+ stopped the scan.
42
+ #
43
+ # @param io [#readpartial] the inflated sheet XML from its start; read forward only, never closed
44
+ # @param sheet [String, nil] the sheet name, for errors
45
+ def initialize(io, sheet: nil)
46
+ @io = io
47
+ @sheet = sheet
48
+ @unnumbered = false
49
+ @last_row = nil
50
+ @buffer = String.new(encoding: Encoding::BINARY)
51
+ end
52
+
53
+ # @return [Boolean] whether #skip_to met a row without +r+, so rows cannot be found by number
54
+ def unnumbered?
55
+ @unnumbered
56
+ end
57
+
58
+ # Discards the bytes before the start tag of the first row numbered +row+ or later.
59
+ #
60
+ # @param row [Integer] a row number
61
+ # @param prefix [String] binary bytes the rest of the stream is handed out after
62
+ # @param bound [LimitedInflater::TokenBound, nil] for the StreamRemainder
63
+ # @return [StreamRemainder, nil] +prefix+ and the rest of the stream from that start tag; nil when no such row
64
+ # exists, or when a row without +r+ comes first (then #unnumbered? is true and the caller must parse from the
65
+ # start)
66
+ # @raise [MalformedFileError] for a row start tag longer than MAX_TAG_BYTES
67
+ def skip_to(row, prefix: "".b, bound: nil)
68
+ from = 0
69
+ loop do
70
+ start = @buffer.byteindex(MARKUP, from)
71
+ if start.nil?
72
+ discard(partial_tag(from, PARTIAL_NAME))
73
+ return unless @buffer.match?(PARTIAL_NAME, 0) ? name_end : read_more
74
+
75
+ from = 0
76
+ elsif enclosing_tag?(from, start)
77
+ raise inside_attribute
78
+ elsif (markup = markup_at(start))
79
+ case markup
80
+ when :other then from = start + 1
81
+ when :incomplete
82
+ discard(start)
83
+ return unless read_more
84
+
85
+ from = 0
86
+ else from = skip_markup(start, *markup) or return
87
+ end
88
+ else
89
+ close = @buffer.byteindex(">", start)
90
+ unless close
91
+ discard(start)
92
+ start = 0
93
+ close = tag_end or return
94
+ end
95
+ raise too_long if close - start >= MAX_TAG_BYTES
96
+
97
+ number = row_number(@buffer.byteslice(start, close - start))
98
+ return unnumbered! if number.nil?
99
+ if number >= row
100
+ return StreamRemainder.new(prefix + @buffer.byteslice(start, @buffer.bytesize - start), @io, bound: bound)
101
+ end
102
+
103
+ @last_row = number
104
+ from = close + 1
105
+ end
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ def row_number(tag)
112
+ match = NUMBER.match(tag) or return
113
+ (match[1] || match[2]).to_i
114
+ end
115
+
116
+ def unnumbered!
117
+ @unnumbered = true
118
+ nil
119
+ end
120
+
121
+ # The buffer begins with a row start tag whose ">" has not been read: reads on until it, scanning each
122
+ # chunk once.
123
+ # @return [Integer, nil] the position of the ">"; nil when the stream ends first
124
+ # @raise [MalformedFileError] when the tag is already longer than MAX_TAG_BYTES (the caller checks the tag
125
+ # that ends at the position returned)
126
+ def tag_end
127
+ loop do
128
+ from = @buffer.bytesize
129
+ raise too_long if from >= MAX_TAG_BYTES
130
+ return unless read_more
131
+
132
+ close = @buffer.byteindex(">", from)
133
+ return close if close
134
+ end
135
+ end
136
+
137
+ def too_long
138
+ located("a row start tag is longer than #{MAX_TAG_BYTES} bytes")
139
+ end
140
+
141
+ def located(message)
142
+ return MalformedFileError.at(message, sheet: @sheet, row: @last_row) if @sheet
143
+
144
+ MalformedFileError.new("the sheet#{", row #{@last_row}" if @last_row}: #{message}", row: @last_row)
145
+ end
146
+
147
+ # The buffer is "<" and what may be the beginning of a row start tag's name: reads on until the name ends,
148
+ # scanning each chunk once, so that the next scan of the buffer can tell whether it is a row.
149
+ # @return [Boolean] false when the stream ends first
150
+ # @raise [MalformedFileError] when the name alone is longer than MAX_TAG_BYTES
151
+ def name_end
152
+ loop do
153
+ from = @buffer.bytesize
154
+ raise long_name if from > MAX_TAG_BYTES
155
+ return false unless read_more
156
+ return true if @buffer.byteindex(NAME_END, from)
157
+ end
158
+ end
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ox"
4
+ require_relative "../../configuration"
5
+ require_relative "../../errors"
6
+ require_relative "../ox_errors"
7
+ require_relative "escapes"
8
+ require_relative "local_name"
9
+ require_relative "../references"
10
+
11
+ module Calcsheet
12
+ module Readers
13
+ module Xlsx
14
+ # The shared strings table of an XLSX file (+xl/sharedStrings.xml+), kept compactly: the UTF-8 bytes of all
15
+ # strings in one binary buffer and the end offset of each string packed as 64-bit integers, so a million
16
+ # strings cost about their text plus 8 bytes each instead of a Ruby String (40+ bytes) apiece.
17
+ #
18
+ # Rich text runs are joined, phonetic runs (+rPh+) are skipped, whitespace is kept exactly, and the
19
+ # +_xHHHH_+ escapes OOXML uses for control characters are decoded.
20
+ #
21
+ # The offsets hold +size+ + 1 values: 0, then the end of each string, so string +i+ is the bytes from offset +i+
22
+ # to offset +i+ + 1 of the buffer. The strings are tagged UTF-8 as Ox hands them over, not validated: the reader
23
+ # of a cell checks the one it takes (SheetParser), so that a table is not checked in full for the strings a read
24
+ # never uses.
25
+ #
26
+ # @api private
27
+ class SharedStrings
28
+ # The pack directive of an offset: an unsigned 64-bit integer, native byte order (the table never leaves the
29
+ # process).
30
+ OFFSET_DIRECTIVE = "Q"
31
+ OFFSET_BYTES = 8
32
+ # The Builder joins the text in chunks of about this many bytes, and packs the offsets this many at a time.
33
+ CHUNK_BYTES = 1_048_576
34
+ OFFSETS_PER_CHUNK = 65_536
35
+
36
+ # @param io [#readpartial] the shared strings XML, for example +ZipContainer#stream+
37
+ # @param max_cell_bytes [Integer, nil] the longest string, as the XML parser hands its text over; nil for no
38
+ # limit
39
+ # @return [SharedStrings]
40
+ # @raise [MalformedFileError] for damaged or empty XML (the parser or stream error is the +cause+) and for a
41
+ # string longer than +max_cell_bytes+
42
+ # @raise [Calcsheet::Error] the error the stream itself raises, such as a ZipBombError (Ox wraps it in a
43
+ # +TypeError+; it is raised again as itself)
44
+ def self.load(io, max_cell_bytes: Configuration::DEFAULTS[:max_cell_bytes])
45
+ builder = Builder.new(max_cell_bytes)
46
+ begin
47
+ # convert_special: false, so that the references reach the builder as written (see References).
48
+ Ox.sax_parse(builder, io, skip: :skip_none, convert_special: false, symbolize: true)
49
+ rescue *OxErrors::RAISED => error
50
+ OxErrors.raise_translated(error) { MalformedFileError.new("the shared strings cannot be parsed: #{_1}") }
51
+ end
52
+ builder.result
53
+ end
54
+
55
+ # @return [Integer] the number of strings
56
+ attr_reader :size
57
+
58
+ # @api private
59
+ # @return [Integer] the bytes of the decoded text of all strings (a test seam: the readers do not use it)
60
+ attr_reader :bytesize
61
+
62
+ # @api private
63
+ def initialize(buffer, offsets, size)
64
+ @buffer = buffer
65
+ @offsets = offsets
66
+ @size = size
67
+ @bytesize = buffer.bytesize
68
+ end
69
+
70
+ # @param index [Integer] the 0-based position of the string, the +v+ of a +t="s"+ cell
71
+ # @return [String] a new frozen String tagged UTF-8 (not validated)
72
+ # @raise [MalformedFileError] when the index is outside the table
73
+ def [](index)
74
+ unless index.is_a?(Integer) && index >= 0 && index < @size
75
+ raise MalformedFileError, "shared string #{index.inspect} is outside the table of #{@size} strings"
76
+ end
77
+
78
+ unchecked(index)
79
+ end
80
+
81
+ # @api private
82
+ # The string at an index the caller checked, as SheetParser does: without the checks of #[].
83
+ # @param index [Integer] 0 to +size+ - 1
84
+ # @return [String] a new frozen String tagged UTF-8 (not validated)
85
+ def unchecked(index)
86
+ position = index * OFFSET_BYTES
87
+ from = @offsets.unpack1(OFFSET_DIRECTIVE, offset: position)
88
+ to = @offsets.unpack1(OFFSET_DIRECTIVE, offset: position + OFFSET_BYTES)
89
+ @buffer.byteslice(from, to - from).force_encoding(Encoding::UTF_8).freeze
90
+ end
91
+
92
+ # Collects the strings while Ox streams the XML. Text and offsets go to chunks that are joined once at the
93
+ # end, so neither buffer keeps the spare capacity of a growing String.
94
+ # @api private
95
+ class Builder < Ox::Sax
96
+ include LocalName::Memo
97
+
98
+ # +@chunk+ collects the text of the strings, +@offsets+ the end offset of each (+@total+, the bytes so far);
99
+ # both move to +@chunks+ and +@offset_chunks+ when they reach CHUNK_BYTES and OFFSETS_PER_CHUNK. +@item+ is
100
+ # the string of the open +si+ (nil outside one), +@in_text+ whether a text belongs to it, +@phonetic+ the
101
+ # depth of +rPh+ runs, +@count+ the strings so far and +@any_element+ whether the document had an element.
102
+ def initialize(max_cell_bytes)
103
+ super()
104
+ start_memo
105
+ @max_cell_bytes = max_cell_bytes
106
+ @chunks = []
107
+ @chunk = +""
108
+ @offsets = []
109
+ @offset_chunks = []
110
+ @total = 0
111
+ @count = 0
112
+ @item = nil
113
+ @in_text = false
114
+ @phonetic = 0
115
+ @any_element = false
116
+ @offsets << 0
117
+ end
118
+
119
+ # Ox reports damaged XML here instead of raising.
120
+ def error(message, line, column)
121
+ raise MalformedFileError.new("the shared strings cannot be parsed: #{message} (line #{line})"),
122
+ cause: OxErrors.parse_error(message, line, column)
123
+ end
124
+
125
+ # Ox's callbacks (start_element, end_element, text, cdata) collect the strings.
126
+ def start_element(name)
127
+ @any_element = true
128
+ case @local[name] || local_name(name)
129
+ when :si then @item = +""
130
+ when :rPh then @phonetic += 1 if @item
131
+ when :t then @in_text = true if @item && @phonetic.zero?
132
+ end
133
+ end
134
+
135
+ def end_element(name)
136
+ case @local[name] || local_name(name)
137
+ when :si then finish_item if @item
138
+ when :rPh then @phonetic -= 1 if @item && @phonetic.positive?
139
+ when :t then @in_text = false
140
+ end
141
+ end
142
+
143
+ def text(value)
144
+ return unless @in_text
145
+
146
+ value = References.line_ends(value) if value.include?("\r")
147
+ append(value.include?("&") ? references(value) : value)
148
+ end
149
+
150
+ def cdata(value)
151
+ append(value.include?("\r") ? References.line_ends(value) : value) if @in_text
152
+ end
153
+
154
+ # @return [SharedStrings]
155
+ # @raise [MalformedFileError] when the document has no element
156
+ def result
157
+ raise MalformedFileError, "the shared strings document is empty" unless @any_element
158
+
159
+ flush_chunk
160
+ flush_offsets
161
+ SharedStrings.new(@chunks.join.force_encoding(Encoding::BINARY), @offset_chunks.join, @count)
162
+ end
163
+
164
+ private
165
+
166
+ def references(value)
167
+ References.decode(value)
168
+ rescue References::Invalid => error
169
+ raise MalformedFileError, "the shared strings: shared string #{@count} holds #{error.message}"
170
+ end
171
+
172
+ # Measured before it is joined: a string longer than max_cell_bytes is never built.
173
+ def append(value)
174
+ if @max_cell_bytes && @item.bytesize + value.bytesize > @max_cell_bytes
175
+ raise MalformedFileError, "the shared strings: shared string #{@count} is longer than max_cell_bytes " \
176
+ "(#{@max_cell_bytes} bytes)"
177
+ end
178
+
179
+ @item << value
180
+ end
181
+
182
+ def finish_item
183
+ Escapes.decode!(@item)
184
+ @chunk << @item
185
+ @total += @item.bytesize
186
+ @offsets << @total
187
+ @count += 1
188
+ @item = nil
189
+ @in_text = false
190
+ @phonetic = 0
191
+ flush_chunk if @chunk.bytesize >= CHUNK_BYTES
192
+ flush_offsets if @offsets.size >= OFFSETS_PER_CHUNK
193
+ end
194
+
195
+ def flush_chunk
196
+ @chunks << @chunk
197
+ @chunk = +""
198
+ end
199
+
200
+ def flush_offsets
201
+ @offset_chunks << @offsets.pack("#{OFFSET_DIRECTIVE}*")
202
+ @offsets.clear
203
+ end
204
+ end
205
+ private_constant :Builder
206
+ end
207
+ end
208
+ end
209
+ end