calcsheet 0.1.0

data/lib/calcsheet/readers/ods/style_scan.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "../../errors"
+require_relative "count"
+require_relative "tag_text"
+
+module Calcsheet
+  module Readers
+    module Ods
+      # Passes over the styles of an ODS entry with a Cursor, up to its first table or its end (the automatic styles
+      # of content.xml, the common styles of styles.xml), noting the table styles that hide a sheet and the cell and
+      # data styles that decide how a value reads (CellFormats): a date style that shows a time or none, and a plain
+      # zero-padding number style (+00000+).
+      #
+      # @api private
+      class StyleScan
+        include TagText
+
+        # The fields of a date style that show a time: a date value under it is a datetime.
+        TIME_FIELDS = %w[hours minutes seconds am-pm].freeze
+
+        # @param cursor [Cursor] at the start of the styles
+        # @param patterns [Patterns] of the entry's prefixes
+        # @param formats [CellFormats] where the cell and data styles go
+        # @param hidden [Hash{String => true}] where the names of the table styles with +table:display="false"+ go
+        # @param max_bytes [Integer] the most bytes of the entry the styles may take
+        def initialize(cursor, patterns, formats, hidden, max_bytes:)
+          @cursor = cursor
+          @patterns = patterns
+          @formats = formats
+          @hidden = hidden
+          @max_bytes = max_bytes
+        end
+
+        # @yieldreturn [String] the message of the error raised beyond +max_bytes+
+        # @return [String, nil] the first table start tag
+        # @raise [MalformedFileError] beyond +max_bytes+, or for an error of the cursor
+        def scan
+          style = nil
+          while (tag = @cursor.next_tag(@patterns.head))
+            raise MalformedFileError, yield if @cursor.position > @max_bytes
+
+            name = element_name(tag)
+            case name
+            when @patterns.table then return tag
+            when @patterns.style
+              closing = closing?(tag)
+              style = closing || self_closing?(tag) ? nil : table_style(tag)
+              cell_style(tag) unless closing
+            when @patterns.table_properties
+              @hidden[style] = true if style && attribute(tag, @patterns.display) == "false"
+            else data_style(tag, name)
+            end
+          end
+          nil
+        end
+
+        private
+
+        # Notes a cell style with its data style and its parent.
+        def cell_style(tag)
+          return unless attribute(tag, @patterns.family) == "table-cell"
+
+          name = attribute(tag, @patterns.style_name) or return
+          @formats.cell_style(name, attribute(tag, @patterns.data_style_name), attribute(tag, @patterns.parent_style_name))
+        end
+
+        # Follows the elements of the data styles: number:date-style and number:number-style with their fields; a
+        # style:map inside a number style makes it conditional.
+        def data_style(tag, name)
+          closing = closing?(tag)
+          local = name.delete_prefix(@patterns.number_qualifier)
+          case local
+          when "date-style", "number-style"
+            return @formats.finish if closing
+
+            @formats.start(local == "date-style" ? :date : :number, attribute(tag, @patterns.style_name))
+            @formats.finish if self_closing?(tag)
+          else
+            return if closing || @formats.current.nil?
+
+            if @formats.current == :date
+              @formats.time! if TIME_FIELDS.include?(local)
+            elsif local == "number" && @formats.digits.nil? && @formats.plain?
+              @formats.number!(zero_padding(tag))
+            else
+              @formats.not_plain!
+            end
+          end
+        end
+
+        # @return [Integer, nil] the minimum integer digits of a number:number without decimals or grouping
+        def zero_padding(tag)
+          decimals = attribute(tag, @patterns.decimal_places)
+          return if (decimals && decimals != "0") || attribute(tag, @patterns.grouping) == "true"
+
+          count = Count.parse(attribute(tag, @patterns.min_integer_digits))
+          count if count && count >= 2
+        end
+
+        # @return [String, nil] the name of a style of the table family
+        def table_style(tag)
+          attribute(tag, @patterns.family) == "table" ? attribute(tag, @patterns.style_name) : nil
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/ods/tag_text.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "../references"
+
+module Calcsheet
+  module Readers
+    module Ods
+      # What the byte scanners of an ODS entry (Scanner, StyleScan, RowSeek) read of a tag they found: its name,
+      # whether it closes an element or is empty, and the decoded value of an attribute. Included, its methods are
+      # private helpers.
+      #
+      # @api private
+      module TagText
+        # The qualified name of a start or end tag.
+        NAME = %r{\A</?([^\s/>]+)}n
+        # A reference in an attribute value: a predefined entity, a decimal or a hexadecimal character reference.
+        ENTITY = /&(?:(lt|gt|amp|quot|apos)|#([0-9]{1,7})|#x(\h{1,6}));/n
+        # The predefined entities of XML.
+        ENTITIES = { "lt" => "<", "gt" => ">", "amp" => "&", "quot" => '"', "apos" => "'" }.freeze
+        # The byte of "/".
+        SLASH = 0x2F
+
+        module_function
+
+        # @return [String, nil] the qualified name of the element of +tag+
+        def element_name(tag)
+          tag[NAME, 1]
+        end
+
+        # Whether +tag+ is an end tag ("</...").
+        def closing?(tag)
+          tag.getbyte(1) == SLASH
+        end
+
+        # Whether +tag+ is an empty-element tag ("<.../>").
+        def self_closing?(tag)
+          tag.getbyte(-2) == SLASH
+        end
+
+        # @param pattern [Regexp] the attribute, as Patterns builds it
+        # @return [String, nil] the decoded UTF-8 value of the attribute
+        def attribute(tag, pattern)
+          match = pattern.match(tag) or return
+          decode(match[1] || match[2])
+        end
+
+        # @param pattern [Regexp] the attribute, as Patterns builds it
+        # @return [String, nil] the value of the attribute decoded as the cells are (References), tagged UTF-8 but not
+        #   validated
+        # @raise [References::Invalid] for an invalid reference, which #attribute would keep as written
+        def strict_attribute(tag, pattern)
+          match = pattern.match(tag) or return
+          raw = match[1] || match[2]
+          (raw.include?("&") ? References.decode(raw) : raw.dup).force_encoding(Encoding::UTF_8)
+        end
+
+        # @return [String] +raw+ with its references decoded, tagged UTF-8 but not validated (the caller checks what
+        #   it keeps, such as a style name)
+        def decode(raw)
+          text = raw.include?("&") ? raw.gsub(ENTITY) { entity(Regexp.last_match) } : raw.dup
+          text.force_encoding(Encoding::UTF_8)
+        end
+
+        # A character reference outside Unicode or of a surrogate is left as written.
+        def entity(match)
+          return ENTITIES.fetch(match[1]).b if match[1]
+
+          code = match[2] ? match[2].to_i : match[3].to_i(16)
+          return match[0] if code.zero? || code > 0x10FFFF || code.between?(0xD800, 0xDFFF)
+
+          [ code ].pack("U").b
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/ods_reader.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "../zip_container"
+require_relative "ods/scanner"
+require_relative "ods/content_parser"
+require_relative "row_batches"
+
+module Calcsheet
+  module Readers
+    # The ODS reader backend (see Reading for the contract): the tables of +content.xml+. Every sheet lives in
+    # that one entry, so each read inflates it from the start: Scanner finds the table, and the first row wanted,
+    # by scanning bytes (sampling, +start_at:+), and ContentParser parses from there to the end of the table only.
+    # Sampling and the header check of merged sheets therefore parse the first rows of each sheet and nothing else.
+    # Every pass counts against +max_uncompressed_size+, and +max_sheets+ bounds the tables, hence the
+    # passes a merged read can make. The rows of a repeated row or cell share their value objects (see ContentParser).
+    #
+    # @api private
+    class OdsReader
+      include RowBatches
+
+      # The entry that holds every table, and the one of the common styles.
+      CONTENT = "content.xml"
+      STYLES = "styles.xml"
+
+      # @return [Boolean] true: the zip central directory needs random access
+      def self.needs_seek?
+        true
+      end
+
+      # @param source [Source] a seekable source
+      # @param options [Options] the book options; the zip limits come from them
+      # @raise [MalformedFileError] when the archive cannot be read or has no content.xml
+      # @raise [ZipBombError] when a zip limit is exceeded
+      def initialize(source, options)
+        @zip = ZipContainer.open(source, options)
+        raise MalformedFileError, "the spreadsheet has no #{CONTENT}" unless @zip.entry?(CONTENT)
+
+        @scanner = Ods::Scanner.new(@zip, entry: CONTENT, styles: (STYLES if @zip.entry?(STYLES)), max_sheets: options.max_sheets)
+        @max_cell_bytes = options.max_cell_bytes
+      end
+
+      # Scans the whole content.xml the first time.
+      # @return [Array<SheetInfo>]
+      # @raise [MalformedFileError] for a damaged content.xml
+      # @raise [TooManySheetsError] for more tables than +max_sheets+
+      def sheets
+        @scanner.sheets
+      end
+
+      # Holds nothing between reads: each read closes its own stream.
+      def close
+        nil
+      end
+
+      private
+
+      def each_row(index, cast, types, start_row, &block)
+        sheet = sheets.fetch(index)
+        io = @scanner.io_at_sheet(index, row: start_row) or return
+        begin
+          Ods::ContentParser.parse(io, sheet: sheet.name, prefixes: @scanner.prefixes, cast: cast, types: types,
+                                       first_row: io.first_row, start_row: start_row, formats: @scanner.cell_formats,
+                                       max_cell_bytes: @max_cell_bytes, &block)
+        ensure
+          io.close
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/ox_errors.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "ox"
+require_relative "../errors"
+
+module Calcsheet
+  module Readers
+    # What the Ox parsers of the readers raise instead of Ox's own errors.
+    #
+    # @api private
+    module OxErrors
+      # What Ox raises: an Ox::Error for damaged XML, a TypeError caused by what the IO raised, and an EncodingError
+      # for an element or attribute name that is not valid UTF-8.
+      RAISED = [ TypeError, Ox::Error, EncodingError ].freeze
+
+      module_function
+
+      # Raises the error of the stream itself when it is a Calcsheet::Error (a zip limit), else the
+      # MalformedFileError the block builds from the message of the original error (the IO's, else Ox's), with that
+      # error as +cause+.
+      #
+      # @param error [Exception] one of RAISED
+      # @yieldparam message [String]
+      # @yieldreturn [MalformedFileError]
+      def raise_translated(error)
+        cause = error.cause
+        raise cause if cause.is_a?(Calcsheet::Error)
+
+        original = cause || error
+        raise yield(original.message), cause: original
+      end
+
+      # @return [Ox::ParseError] the +cause+ of the MalformedFileError of an error Ox reports to a SAX handler's
+      #   +error+ callback instead of raising it
+      def parse_error(message, line, column)
+        Ox::ParseError.new("#{message} at line #{line}, column #{column}")
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/references.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Readers
+    # The XML references of a text (+&+, +A+, +A+), decoded by the gem instead of Ox for the XLSX and
+    # ODS parsers, and the line ends Ox hands over as written (#line_ends): Ox truncates a text at +�+ and writes a reference beyond U+10FFFF as another character, so that
+    # a crafted cell would read as a shorter or different text. A reference to NUL, a surrogate or a code point beyond
+    # U+10FFFF, an unknown entity and an "&" that starts no reference raise Invalid.
+    #
+    # @api private
+    module References
+      # Raised for a text that is not valid XML; the parser turns it into a MalformedFileError at its cell.
+      class Invalid < StandardError; end
+
+      # Leading zeros are allowed ("&#00065;"); the digits after them are bounded, which keeps the codes small.
+      REFERENCE = /&(?:(lt|gt|amp|quot|apos)|#0*([0-9]{1,7})|#x0*(\h{1,6}));/n
+      # The predefined entities of XML.
+      ENTITIES = { "lt" => "<", "gt" => ">", "amp" => "&", "quot" => '"', "apos" => "'" }.transform_values { _1.b.freeze }.freeze
+      # The code points a reference may name: up to U+10FFFF, not a surrogate, not NUL.
+      MAX_CODE_POINT = 0x10FFFF
+      SURROGATES = (0xD800..0xDFFF)
+
+      # The line ends that XML reads as LF (XML 1.0, 2.11): CR LF and a CR alone.
+      LINE_END = /\r\n?/n
+
+      module_function
+
+      # Reads the line ends of a text as XML does; called before #decode, so that a referenced CR (+&#13;+) stays.
+      # @param text [String] text holding a CR
+      # @return [String] a new String in the encoding of +text+
+      def line_ends(text)
+        text.b.gsub(LINE_END, "\n").force_encoding(text.encoding)
+      end
+
+      # Works on the bytes, so that a text that is not valid UTF-8 is decoded too (and refused later by its reader).
+      # @param text [String] text holding an "&"
+      # @return [String] a new String in the encoding of +text+, with the references decoded
+      # @raise [Invalid]
+      def decode(text)
+        count = 0
+        bytes = text.b
+        decoded = bytes.gsub(REFERENCE) do
+          count += 1
+          match = Regexp.last_match
+          match[1] ? ENTITIES.fetch(match[1]) : character(match[2] ? match[2].to_i : match[3].hex, match[0])
+        end
+        raise Invalid, "an \"&\" that starts no character reference" unless count == bytes.count("&")
+
+        decoded.force_encoding(text.encoding)
+      end
+
+      def character(code, reference)
+        if code.zero? || code > MAX_CODE_POINT || SURROGATES.cover?(code)
+          raise Invalid, "the invalid character reference #{reference}"
+        end
+
+        code.chr(Encoding::UTF_8).b
+      end
+      private_class_method :character
+    end
+  end
+end

data/lib/calcsheet/readers/row_batches.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Readers
+    # The +first_rows+ and +each_raw_batch+ of the backend contract (see Reading) for a reader that streams the rows of
+    # a sheet with a private +each_row(sheet_index, cast, types, start_row) { |row| ... }+; a throw or a break out of
+    # the block must stop the stream and release it.
+    #
+    # @api private
+    module RowBatches
+      # Thrown out of +each_row+ once #first_rows has its rows; the reader's +ensure+ releases the stream.
+      ENOUGH_ROWS = :calcsheet_enough_rows
+
+      # @return [Array<RawRow>] the first +count+ non-blank rows of the sheet
+      def first_rows(sheet_index, count, cast:, types: nil)
+        rows = []
+        return rows unless count.positive?
+
+        # The throw leaves through the reader's parser (Ox re-raises it, see SaxSheet#parse) and the +ensure+ of its
+        # +each_row+, which closes the stream.
+        catch(ENOUGH_ROWS) do
+          each_row(sheet_index, cast, types, 1) do |row|
+            rows << row
+            throw ENOUGH_ROWS if rows.size == count
+          end
+        end
+        rows
+      end
+
+      # @yieldparam batch [Array<RawRow>] at most +size+ rows numbered +start_row+ or later, a new Array each time
+      # @return [void]
+      def each_raw_batch(sheet_index, size:, cast:, types:, start_row: 1)
+        batch = []
+        each_row(sheet_index, cast, types, start_row) do |row|
+          batch << row
+          next if batch.size < size
+
+          yield batch
+          batch = []
+        end
+        yield batch unless batch.empty?
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/sax_sheet.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "ox"
+require_relative "../cast"
+require_relative "../errors"
+require_relative "../limited_inflater"
+require_relative "../raw_row"
+require_relative "../reading"
+require_relative "ox_errors"
+
+module Calcsheet
+  module Readers
+    # What the SAX parsers of one sheet (Xlsx::SheetParser, Ods::ContentParser) share: parsing until they throw STOP,
+    # the translation of parser errors, errors located at the sheet, row and column, and the end of a row.
+    #
+    # The including class defines OPTIONS (for Ox.sax_parse), +current_row+ and +current_column+ (where an error is
+    # located), and keeps +@sheet+ (the sheet name), +@on_row+ (the block rows are yielded to), +@values+, +@errors+
+    # (+[column, raw, reason]+ per failed cell, or nil), +@last+ (the last non-blank column, -1 for none) and
+    # +@in_cell+. +@yielding+ is true while the caller's block runs.
+    #
+    # @api private
+    module SaxSheet
+      # Thrown by the handler where the part it parses ends, so that the rest is never read.
+      STOP = :calcsheet_end_of_sheet
+
+      # @api private
+      # The throw of STOP, like an exception of a callback, unwinds through Ox.sax_parse: Ox runs the parse under
+      # rb_protect, frees its buffers and re-raises (rb_jump_tag), so nothing of the parser is left behind and the
+      # caller's +ensure+ closes the stream.
+      def parse(io)
+        catch(STOP) { Ox.sax_parse(self, io, **self.class::OPTIONS) }
+        nil
+      rescue *OxErrors::RAISED => error
+        # A TypeError of the caller's block, or one without a cause, is not the stream's.
+        raise if @yielding || (error.is_a?(TypeError) && error.cause.nil?)
+        raise token_too_long(error.cause), cause: error.cause if error.cause.is_a?(LimitedInflater::TokenTooLong)
+
+        OxErrors.raise_translated(error) { malformed("the sheet cannot be read: #{_1}") }
+      end
+
+      # Ox reports damaged XML here instead of raising.
+      # @raise [MalformedFileError] located at the row and column being read
+      def error(message, line, column)
+        raise malformed(message), cause: OxErrors.parse_error(message, line, column)
+      end
+
+      private
+
+      # Drops what follows the last non-blank cell: the values (a failed cast leaves the last non-blank cell nil,
+      # whitespace after it is trimmed) and, like CSV, the cast failures of the whitespace after it. +@errors+ is nil
+      # when none is left, else sorted by column.
+      #
+      # +@values+ is sparse: a cell sets only its own position, so the Array is as long as the last cell that got a
+      # value, nil before it where cells were empty or missing. A cell whose cast failed, or an error cell, sets
+      # nothing: when it is the last non-blank one and no whitespace text after it got a value, the Array is shorter
+      # than +@last+ + 1, and assigning nil at +@last+ extends it to that cell. Otherwise the values after +@last+
+      # (whitespace texts) are popped.
+      def trim_row!
+        if @values.size <= @last
+          @values[@last] = nil
+        else
+          @values.pop while @values.size > @last + 1
+        end
+        return unless @errors
+
+        @errors.reject! { |error| error[0] > @last }
+        if @errors.empty? then @errors = nil
+        elsif @errors.size > 1 then @errors.sort_by!(&:first)
+        end
+      end
+
+      # Hands a row to the caller's block; its errors are the block's own: +@yielding+ tells #parse not to translate
+      # them.
+      def yield_row(number, values, errors)
+        row = RawRow.new(number: number, values: values, errors: errors)
+        @yielding = true
+        @on_row.call(row)
+        @yielding = false
+      end
+
+      # Whitespace only, without a regexp for the common text that starts with a printable ASCII character.
+      def blank?(text)
+        byte = text.getbyte(0)
+        !(byte > 32 && byte < 128) && !Reading::NON_BLANK.match?(text)
+      end
+
+      # The texts of a boolean value in XLSX ("1", "0") and ODS ("true", "false"); both formats accept both. A case of
+      # String literals, which Ruby dispatches by hash, costs less per cell than a Hash or Array lookup.
+      def boolean(text)
+        case text
+        when "1", "true" then true
+        when "0", "false" then false
+        else raise Cast::Failure.new(:invalid_boolean, text)
+        end
+      end
+
+      # The zip stream refused a text or a tag too long for max_cell_bytes: located like the parser's own errors.
+      def token_too_long(error)
+        malformed("a text or a tag is longer than max_cell_bytes (#{error.max_cell_bytes} bytes) allows",
+                  column_index: (current_column if @in_cell))
+      end
+
+      def malformed(message, column_index: nil, row: current_row)
+        MalformedFileError.at(message, sheet: @sheet, row: row, column_index: column_index)
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/stream_remainder.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "tag_scan"
+
+module Calcsheet
+  module Readers
+    # The rest of a stream after a byte scanner (Xlsx::RowSkipper, Ods::Scanner) found where the parser starts: the
+    # bytes already read (+head+), then the stream. Answers +readpartial+ (what Ox uses), +read+ and +eof?+; never
+    # hands out more bytes than asked.
+    #
+    # @api private
+    class StreamRemainder
+      # @param head [String] binary bytes handed out first
+      # @param io [#readpartial] the rest; never closed here
+      # @param bound [LimitedInflater::TokenBound, nil] checks every piece handed out, when +io+ is a stream
+      #   opened without the bound because a byte scanner read it first (ZipContainer#token_bound)
+      def initialize(head, io, bound: nil)
+        @head = head
+        @io = io
+        @bound = bound
+      end
+
+      # @raise [EOFError] at the end
+      # @raise [LimitedInflater::TokenTooLong] when the bound is exceeded
+      def readpartial(length, outbuf = nil)
+        data = @head.empty? ? @io.readpartial(length) : take(length)
+        @bound&.check(data)
+        outbuf ? outbuf.replace(data) : data
+      end
+
+      # @return [String, nil] up to +length+ bytes (all with nil); nil at the end for a positive +length+
+      # @raise [LimitedInflater::TokenTooLong] when the bound is exceeded
+      def read(length = nil, outbuf = nil)
+        data = String.new(encoding: Encoding::BINARY)
+        begin
+          data << readpartial(length ? length - data.bytesize : TagScan::CHUNK_BYTES) while length.nil? || data.bytesize < length
+        rescue EOFError
+          nil
+        end
+        data = nil if data.empty? && length&.positive?
+        outbuf && data ? outbuf.replace(data) : data
+      end
+
+      # Reads ahead one chunk into the head to find out.
+      # @return [Boolean] whether nothing is left
+      def eof?
+        return false unless @head.empty?
+
+        @head = @io.readpartial(TagScan::CHUNK_BYTES).b
+        false
+      rescue EOFError
+        true
+      end
+
+      private
+
+      def take(length)
+        return @head.tap { @head = String.new(encoding: Encoding::BINARY) } if @head.bytesize <= length
+
+        piece = @head.byteslice(0, length)
+        @head = @head.byteslice(length, @head.bytesize - length)
+        piece
+      end
+    end
+  end
+end