calcsheet 0.1.0

data/lib/calcsheet/readers/xlsx/sheet_parser.rb

@@ -0,0 +1,435 @@
+# frozen_string_literal: true
+
+require "ox"
+require_relative "../../cast"
+require_relative "../../configuration"
+require_relative "../../errors"
+require_relative "../../grid"
+require_relative "../../untrusted"
+require_relative "../sax_sheet"
+require_relative "escapes"
+require_relative "../references"
+
+module Calcsheet
+  module Readers
+    module Xlsx
+      # Streams the XML of one sheet with Ox's SAX parser and hands each non-blank row to a block as a RawRow, cast
+      # with a Cast. Parsing stops at the end of +sheetData+, so what follows it (merged ranges,
+      # hyperlinks...) is never read.
+      #
+      # Rows are numbered by their +r+ attribute, counted on from the previous row when it is absent; cells are
+      # placed by the column letters of their +r+, after the previous cell when it is absent. Element names
+      # are read without their namespace prefix. Since the file is untrusted, a row numbered outside 1..1048576 or
+      # not after the previous row, and a cell beyond column XFD, raise MalformedFileError: they would otherwise
+      # drop rows, yield them out of order, or build rows of any length.
+      #
+      # The text of a cell is at most +max_cell_bytes+ long, and the value of a number, a date, a boolean, an error or
+      # a shared string index at most Cast::MAX_NUMBER_BYTES: longer ones raise MalformedFileError before they are
+      # joined or cast.
+      #
+      # The work per cell is a few callbacks with no Hash, no regexp compilation and no String allocated beyond the
+      # ones Ox hands over; everything else is kept in instance variables reset per row and cell.
+      #
+      # @api private
+      class SheetParser < Ox::Sax
+        include SaxSheet
+
+        # convert_special: false, so that the references in text reach #text as written (see References).
+        # skip: :skip_none hands every text over as written, whitespace included: Ox reports a whitespace-only text
+        # once, and drops it only before the end tag of an element with children, where no cell text is (with
+        # :skip_off it would report the whitespace of a childless +<t> </t>+ twice, doubling it).
+        OPTIONS = { skip: :skip_none, convert_special: false, symbolize: true, strip_namespace: true }.freeze
+        # The bytes of "A" and "Z".
+        LETTER_A = 65
+        LETTER_Z = 90
+        # The bit that makes an ASCII letter lowercase.
+        CASE_BIT = 32
+        # A reference of more than three letters is beyond the last column.
+        MAX_LETTERS = 3
+        # A row's +r+: digits without leading zero, at most seven (Grid::MAX_ROWS has seven).
+        ROW_NUMBER = /\A[1-9]\d{0,6}\z/
+        # A cell style index: digits only, at most ten of them (cellXfs never holds more styles).
+        STYLE_INDEX = /\A\d{1,10}\z/
+        # The most style indices whose kind #style_kind memoizes; beyond them a style is checked at each cell.
+        MAX_STYLE_KINDS = 4_096
+        # The byte of "0": an index of more than one digit starting with it is not canonical, and is not memoized.
+        ZERO = 48
+
+        # Parses +io+ and yields the rows numbered +start_row+ or later.
+        #
+        # @param io [#readpartial] the sheet XML from its start, or from a row start tag inside a +sheetData+
+        #   element (see RowSkipper)
+        # @param sheet [String] the sheet name, for errors
+        # @param cast [Cast]
+        # @param types [#[], nil] the declared type per column index (see Reading)
+        # @param styles [WorkbookMeta] gives +style_kind(index)+
+        # @param strings [#call] returns the SharedStrings, called at the first +t="s"+ cell
+        # @param start_row [Integer]
+        # @param max_cell_bytes [Integer, nil] the longest text of a cell; nil for no limit
+        # @yieldparam row [RawRow]
+        # @raise [MalformedFileError] for damaged XML (the parser error is the +cause+), a shared string outside the
+        #   table, a text too long, or another error of the stream (kept as +cause+)
+        # @raise [Calcsheet::Error] an error of the stream itself (a zip limit)
+        def self.parse(io, sheet:, cast:, types:, styles:, strings:, start_row: 1,
+                       max_cell_bytes: Configuration::DEFAULTS[:max_cell_bytes], &block)
+          new(sheet: sheet, cast: cast, types: types, styles: styles, strings: strings, start_row: start_row,
+              max_cell_bytes: max_cell_bytes, on_row: block).parse(io)
+        end
+
+        # @api private
+        #
+        # The state of the parse, in instance variables so that a callback allocates nothing:
+        #
+        # - set once: +@sheet+, +@cast+, +@types+, +@styles+ (the WorkbookMeta), +@strings+ (the lambda that loads the
+        #   shared strings) and +@table+ (the SharedStrings, loaded at the first +t="s"+ cell), +@start_row+,
+        #   +@on_row+ (the caller's block), +@max_cell_bytes+, +@style_kinds+ (the memo of #style_kind);
+        # - across rows: +@last_number+ (the number of the last row read, 0 before the first) and +@yielding+ (the
+        #   caller's block runs; see SaxSheet#parse);
+        # - the element whose attributes come: +@element+;
+        # - per row (#begin_row): +@in_row+, +@row_number+ (its +r+, nil without one), +@values+ (sparse, see
+        #   SaxSheet#trim_row!), +@errors+, +@last+ (the last non-blank column, -1 for none) and +@next_column+ (the
+        #   column of a cell without +r+);
+        # - per cell (+c+): +@in_cell+ (inside a row), +@column+ (from its +r+, nil without one), +@cell_type+ (its
+        #   +t+), +@style+ (its +s+) and +@text+ (the text joined so far, nil before any);
+        # - per text: +@capture+ (the text of +v+, or of a +t+ of the inline string outside phonetic runs, is the
+        #   cell's), +@inline+ (inside the cell's +is+), +@phonetic+ (the depth of +rPh+ runs), +@bounded+ and
+        #   +@text_limit+ (the most bytes the text may join to: Cast::MAX_NUMBER_BYTES for a bounded value,
+        #   +@max_cell_bytes+ for text).
+        def initialize(sheet:, cast:, types:, styles:, strings:, start_row:, max_cell_bytes:, on_row:)
+          super()
+          @max_cell_bytes = max_cell_bytes
+          @text_limit = nil
+          @bounded = false
+          @sheet = sheet
+          @cast = cast
+          @types = types
+          @styles = styles
+          @strings = strings
+          @table = nil
+          @start_row = start_row
+          @on_row = on_row
+          @yielding = false
+          @last_number = 0
+          @element = nil
+          @in_row = false
+          @in_cell = false
+          @capture = false
+          @inline = false
+          @phonetic = 0
+          # +[kind, pad]+ by the text of a valid, canonical cell style index (see #style_kind), so that each is checked
+          # once.
+          @style_kinds = {}
+        end
+
+        # @api private
+        # Ox's callbacks (start_element, end_element, attr, text, cdata) drive the state described at #initialize.
+        def start_element(name)
+          @element = name
+          case name
+          when :c
+            # A cell outside a row has no row to go to.
+            @in_cell = @in_row
+            @column = nil
+            @cell_type = nil
+            @style = nil
+            @text = nil
+            # A cell nested in the inline string of another one starts outside any text.
+            @capture = false
+            @inline = false
+          when :v
+            @capture = @in_cell
+            @bounded = bounded_value?
+            @text_limit = @bounded ? Cast::MAX_NUMBER_BYTES : @max_cell_bytes
+          when :t
+            @capture = @inline && @phonetic.zero?
+            @bounded = false
+            @text_limit = @max_cell_bytes
+          when :row then begin_row
+          when :is then @inline = @in_cell
+          when :rPh then @phonetic += 1
+          end
+        end
+
+        # @api private
+        def end_element(name)
+          case name
+          when :v, :t then @capture = false
+          when :c
+            finish_cell if @in_cell
+            # The text ends with its cell, also for a cell nested in an inline string: no later text is joined to
+            # it, and @text (a shared string, or the raw value of a cast error) is never changed afterwards.
+            @text = nil
+            @capture = false
+            @inline = false
+          when :row then finish_row if @in_row
+          when :is then @inline = false
+          when :rPh then @phonetic -= 1 if @phonetic.positive?
+          when :sheetData then throw STOP
+          end
+        end
+
+        # @api private
+        def attr(name, value)
+          case @element
+          when :c
+            case name
+            # A reference is ASCII: only another one needs the UTF-8 check, which costs a method call per cell.
+            when :r then @column = column_of(value.ascii_only? ? value : attribute_utf8(value))
+            when :t then @cell_type = value
+            when :s then @style = value
+            end
+          when :row
+            @row_number = row_number(value) if name == :r
+          end
+        end
+
+        # The text of the cell, joined from its pieces (rich text runs, CDATA sections) and measured before each join.
+        # The pieces are appended to the first one, a String Ox allocated for this callback (+value+ copies it only
+        # if it were frozen), so a cell of many runs is not copied once per run. The line ends of every piece are read
+        # as XML reads them, and the references of a text piece decoded; a CDATA section keeps its references.
+        def text(value)
+          return unless @capture
+
+          value = References.line_ends(value) if value.include?("\r")
+          join(value.include?("&") ? references(value) : value)
+        end
+
+        # @api private
+        def cdata(value)
+          join(value.include?("\r") ? References.line_ends(value) : value) if @capture
+        end
+
+        private
+
+        def join(value)
+          if @text
+            raise text_too_long if @text_limit && @text.bytesize + value.bytesize > @text_limit
+
+            @text << value
+          else
+            raise text_too_long if @text_limit && value.bytesize > @text_limit
+
+            @text = +value
+          end
+        end
+
+        def references(value)
+          References.decode(value)
+        rescue References::Invalid => error
+          raise malformed("the text holds #{error.message}", column_index: current_column)
+        end
+
+        # Whether the value of the cell is a number, a date, a boolean, an error or a shared string index: a text
+        # bounded by Cast::MAX_NUMBER_BYTES rather than max_cell_bytes.
+        def bounded_value?
+          case @cell_type
+          when nil, "n", "s", "b", "e", "d" then true
+          else false
+          end
+        end
+
+        def text_too_long
+          message =
+            if @bounded then "the value is longer than #{Cast::MAX_NUMBER_BYTES} bytes"
+            else "the text is longer than max_cell_bytes (#{@max_cell_bytes} bytes)"
+            end
+          malformed(message, column_index: current_column)
+        end
+
+        def begin_row
+          @in_row = true
+          @row_number = nil
+          @values = []
+          @errors = nil
+          @last = -1
+          @next_column = 0
+        end
+
+        def finish_row
+          @in_row = false
+          number = @row_number || counted_row
+          @last_number = number
+          return if @last.negative? || number < @start_row
+
+          trim_row!
+          yield_row(number, @values, @errors)
+        end
+
+        def finish_cell
+          @in_cell = false
+          column = @column || @next_column
+          raise beyond_last_column if column > Grid::LAST_COLUMN_INDEX
+
+          @next_column = column + 1
+          return if @text.nil?
+
+          value = cell_value(column)
+          @values[column] = value unless value.nil?
+        rescue Cast::Failure => failure
+          (@errors ||= []) << [ column, error_text(failure), failure.reason ]
+        end
+
+        # The text of a failed cast: the cell's text, or for a number cell declared :integer that holds a fraction the
+        # number as read (Cast#integer_number), not the full print of a double that Excel writes.
+        def error_text(failure)
+          failure.reason == :not_integral && (@cell_type.nil? || @cell_type == "n") ? failure.value : @text
+        end
+
+        # The cell's value, cast; before the cast, moves @last to the cell when its raw content is not blank. A
+        # string cell leaves its text in @text, for the error of a failed cast.
+        def cell_value(column)
+          type = @types && @types[column]
+          case @cell_type
+          when nil, "n"
+            @last = column if column > @last
+            kind, pad = @style ? @style_kinds[@style] || style_kind(@style, column) : @styles.style_kind(0)
+            @cast.number(utf8(@text, column), kind, type, pad: pad)
+          when "s" then string_value(@text = shared_string(column), column, type)
+          when "inlineStr", "str" then string_value(Escapes.decode!(utf8(@text, column)), column, type)
+          when "b"
+            @last = column if column > @last
+            utf8(@text, column)
+            @cast.boolean(boolean(@text), type)
+          when "e"
+            utf8(@text, column)
+            error_value(column)
+          when "d"
+            @last = column if column > @last
+            kind, = @style ? @style_kinds[@style] || style_kind(@style, column) : @styles.style_kind(0)
+            @cast.iso_date(utf8(@text, column), type, kind)
+          else
+            attribute_utf8(@cell_type)
+            string_value(utf8(@text, column), column, type)
+          end
+        end
+
+        # The +[kind, pad]+ of a cell style index, checked where the style decides the value: digits only, one of the
+        # cell styles (0 also in a workbook without styles). A negative or huge index would pick another style or
+        # overflow. The kind is memoized by the canonical text of the index (no leading zero), for at most
+        # MAX_STYLE_KINDS indices: zero-padded variants of the same index would otherwise each take an entry.
+        def style_kind(style, column)
+          attribute_utf8(style)
+          count = @styles.style_count
+          index = STYLE_INDEX.match?(style) ? style.to_i : -1
+          unless index.zero? || index.between?(1, count - 1)
+            raise malformed("the cell style #{Untrusted.show(style)} is not one of the #{count} cell styles",
+                            column_index: column)
+          end
+
+          kind = @styles.style_kind(index)
+          canonical = style.getbyte(0) != ZERO || style.bytesize == 1
+          @style_kinds[style] = kind if canonical && @style_kinds.size < MAX_STYLE_KINDS
+          kind
+        end
+
+        # The value of an attribute the parser reads, as UTF-8 (see #utf8).
+        def attribute_utf8(value, row: current_row)
+          value.force_encoding(Encoding::UTF_8) unless value.encoding == Encoding::UTF_8
+          return value if value.valid_encoding?
+
+          raise malformed("an attribute value is not valid UTF-8", row: row)
+        end
+
+        def string_value(text, column, type)
+          return if text.empty?
+
+          @last = column if column > @last && !blank?(text)
+          @cast.text(text, type)
+        end
+
+        def error_value(column)
+          @last = column if column > @last
+          (@errors ||= []) << [ column, @text, :cell_error ]
+          nil
+        end
+
+        def shared_string(column)
+          @table ||= @strings.call
+          index = Integer(@text, 10, exception: false)
+          unless index && index >= 0 && index < @table.size
+            raise malformed("shared string #{Untrusted.show(@text)} is outside the table of #{@table.size} strings", column_index: column)
+          end
+
+          text = @table.unchecked(index)
+          return text if text.valid_encoding?
+
+          raise malformed("the text is not valid UTF-8", column_index: column)
+        end
+
+        # Ox tags text as UTF-8 only when the document declares it; UTF-8 is also the default of XML. Ox does not
+        # check the bytes.
+        def utf8(text, column)
+          text.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
+          return text if text.valid_encoding?
+
+          raise malformed("the text is not valid UTF-8", column_index: column)
+        end
+
+        # The 0-based column of a cell reference ("AB12" -> 27); nil without letters.
+        def column_of(ref)
+          first = ref.getbyte(0)
+          second = ref.getbyte(1)
+          # The common case: one capital letter and the row digits.
+          return first - LETTER_A if second && second < LETTER_A && first >= LETTER_A && first <= LETTER_Z
+
+          index = 0
+          position = 0
+          while (byte = ref.getbyte(position))
+            byte -= CASE_BIT if byte > LETTER_Z
+            break unless byte >= LETTER_A && byte <= LETTER_Z
+            raise beyond_last_column(ref) if position == MAX_LETTERS
+
+            index = index * 26 + byte - 64
+            position += 1
+          end
+          return if position.zero?
+          raise beyond_last_column(ref) if index > Grid::MAX_COLUMNS
+
+          index - 1
+        end
+
+        def beyond_last_column(ref = nil)
+          reference = ref ? " (#{Untrusted.show(ref)})" : ""
+          malformed("a cell is beyond the last column #{Grid::LAST_COLUMN}#{reference}")
+        end
+
+        # The number of the row given by its +r+: digits only, 1 to Grid::MAX_ROWS, after the previous row.
+        def row_number(ref)
+          number = ROW_NUMBER.match?(attribute_utf8(ref, row: last_row)) ? ref.to_i : 0
+          unless number.between?(1, Grid::MAX_ROWS)
+            raise malformed("the next row has the invalid number #{Untrusted.show(ref)} (expected 1 to #{Grid::MAX_ROWS})",
+                            row: last_row)
+          end
+          if number <= @last_number
+            raise malformed("the next row is numbered #{number}, but rows must be numbered in increasing order",
+                            row: last_row)
+          end
+
+          number
+        end
+
+        # A row without +r+ is numbered after the previous one.
+        def counted_row
+          number = @last_number + 1
+          return number if number <= Grid::MAX_ROWS
+
+          raise malformed("the next row would be numbered #{number}, beyond the last row #{Grid::MAX_ROWS}", row: last_row)
+        end
+
+        # The row being read when it is known, else the last row read.
+        def current_row
+          @in_row ? (@row_number || @last_number + 1) : last_row
+        end
+
+        def last_row
+          @last_number.positive? ? @last_number : nil
+        end
+
+        # The column of the cell being read: of its reference, else after the previous cell.
+        def current_column
+          @column || @next_column
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/xlsx/styles.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "ox"
+require_relative "../../errors"
+require_relative "../ox_errors"
+require_relative "local_name"
+require_relative "number_format"
+
+module Calcsheet
+  module Readers
+    module Xlsx
+      # How each cell style of +xl/styles.xml+ reads a number: streamed with Ox's SAX parser, keeping only the
+      # custom number formats (+numFmts+, id => code) and the +numFmtId+ of each +cellXfs+ entry. Fonts,
+      # fills, borders and the rest pass by without being kept, so a styles part costs its formats, not its size.
+      #
+      # Since the file is untrusted, more than MAX_CELL_FORMATS cellXfs entries, more than MAX_NUMBER_FORMATS numFmts,
+      # a format code longer than MAX_CODE_BYTES or not valid UTF-8 raise MalformedFileError. Elements and attributes
+      # are matched by local name.
+      #
+      # @api private
+      class Styles < Ox::Sax
+        include LocalName::Memo
+
+        PART = "xl/styles.xml"
+        # The cellXfs entries, numFmts and format code bytes a styles part may have.
+        MAX_CELL_FORMATS = 1_000_000
+        MAX_NUMBER_FORMATS = 100_000
+        MAX_CODE_BYTES = 1_024
+        # More digits are no numFmtId (an xsd:unsignedInt): such an id is UNKNOWN_ID. A numFmt of it defines nothing,
+        # and an xf of it is general, so that two different over-long ids never share a format code.
+        #
+        # Any other id is read with String#to_i: a negative one ("-5") pairs only with a numFmt of the same id, and is
+        # general without one, as no built-in format has it; "-1" is UNKNOWN_ID itself, general. A value without
+        # leading digits reads as 0, like a missing numFmtId (see #add_cell_format).
+        MAX_ID_DIGITS = 10
+        UNKNOWN_ID = -1
+        # The attribute values and the codes are all the part gives: whitespace between elements is dropped.
+        OPTIONS = { skip: :skip_white, convert_special: true, symbolize: true }.freeze
+
+        # @param io [#readpartial] the styles XML, for example +ZipContainer#stream+
+        # @return [Array<Array(Symbol, Integer | nil)>] +[kind, pad]+ per cell style index (see NumberFormat)
+        # @raise [MalformedFileError] for damaged XML (the parser error is the +cause+), a root that is not a
+        #   styleSheet, or a bound exceeded
+        # @raise [Calcsheet::Error] an error of the stream itself (a zip limit)
+        def self.load(io)
+          styles = new
+          begin
+            Ox.sax_parse(styles, io, **OPTIONS)
+          rescue *OxErrors::RAISED => error
+            OxErrors.raise_translated(error) { MalformedFileError.new("#{PART} cannot be parsed: #{_1}") }
+          end
+          styles.result
+        end
+
+        # @api private
+        def initialize
+          super
+          start_memo
+          # The local name of the root element, which must be styleSheet.
+          @root = nil
+          # The local names of the open elements, from the root.
+          @path = []
+          # The element whose attributes come.
+          @element = nil
+          # The code of each custom number format id, and how many numFmt elements came.
+          @codes = {}
+          @number_formats = 0
+          # The numFmtId of each cell format, by cell style index.
+          @ids = []
+          # The first top-level cellXfs: 0 before it, 1 inside it, 2 after it.
+          @cell_xfs = 0
+          # The numFmtId and formatCode of the numFmt or xf being read.
+          @id = nil
+          @code = nil
+        end
+
+        # Only the numFmt children of the top-level numFmts and the xf children of the first top-level cellXfs count:
+        # dxf and cellStyleXfs hold elements of the same names.
+        def start_element(name)
+          local = @local[name] || local_name(name)
+          @root ||= local
+          @cell_xfs = 1 if local == :cellXfs && @path.size == 1 && @cell_xfs.zero?
+          @path << local
+          @element = local
+          return unless local == :numFmt || local == :xf
+
+          @id = nil
+          @code = nil
+        end
+
+        # @api private
+        def end_element(_name)
+          local = @path.pop
+          if @path.size == 2
+            add_number_format if local == :numFmt && @path.last == :numFmts
+            add_cell_format if local == :xf && @path.last == :cellXfs && @cell_xfs == 1
+          elsif @path.size == 1 && local == :cellXfs && @cell_xfs == 1
+            @cell_xfs = 2
+          end
+          @element = nil
+        end
+
+        # @api private
+        def attr(name, value)
+          case @element
+          when :numFmt, :xf
+            case @local[name] || local_name(name)
+            when :numFmtId then @id = number_format_id(value)
+            when :formatCode then @code = value if @element == :numFmt
+            end
+          end
+        end
+
+        # Ox reports damaged XML here instead of raising.
+        def error(message, line, column)
+          raise MalformedFileError.new("#{PART} cannot be parsed: #{message}"), cause: OxErrors.parse_error(message, line, column)
+        end
+
+        # @api private
+        def result
+          raise MalformedFileError, "#{PART} is not a styleSheet part" unless @root == :styleSheet
+
+          kinds = {}
+          @ids.map { |id| kinds[id] ||= NumberFormat.classify(id, @codes[id]).freeze }
+        end
+
+        private
+
+        def number_format_id(value)
+          value.bytesize <= MAX_ID_DIGITS ? value.to_i : UNKNOWN_ID
+        end
+
+        def add_number_format
+          if (@number_formats += 1) > MAX_NUMBER_FORMATS
+            raise MalformedFileError, "#{PART} has more than #{MAX_NUMBER_FORMATS} number formats (numFmts)"
+          end
+
+          # Ox tags text as UTF-8 only when the document declares it; UTF-8 is also the default of XML.
+          code = @code.to_s.dup.force_encoding(Encoding::UTF_8)
+          raise MalformedFileError, "#{PART} has a number format code longer than #{MAX_CODE_BYTES} bytes" if code.bytesize > MAX_CODE_BYTES
+          raise MalformedFileError, "#{PART} has a number format code that is not valid UTF-8" unless code.valid_encoding?
+
+          # A numFmt without numFmtId (which the schema requires) is keyed as 0, as an xf without one reads.
+          @codes[@id || 0] = code unless @id == UNKNOWN_ID
+        end
+
+        # An xf without numFmtId reads as id 0, the General format.
+        def add_cell_format
+          raise MalformedFileError, "#{PART} has more than #{MAX_CELL_FORMATS} cell formats (cellXfs)" if @ids.size == MAX_CELL_FORMATS
+
+          @ids << (@id || 0)
+        end
+      end
+    end
+  end
+end