calcsheet 0.1.0

data/lib/calcsheet/writers/ods_writer.rb

@@ -0,0 +1,437 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+require "date"
+require "stringio" # zip_kit 6.3 uses StringIO without requiring it
+require "zip_kit"
+require "zlib"
+require_relative "../decimal_text"
+require_relative "../errors"
+require_relative "../grid"
+require_relative "../odf"
+require_relative "xml_text"
+require_relative "zip_buffered"
+
+module Calcsheet
+  module Writers
+    # ODS writer backend, see the contract in WriteBook. The +mimetype+ entry comes first, stored, without data
+    # descriptor (LibreOffice refuses one that has a descriptor) and without extra field (ODF packages §3.3: magic
+    # sniffing finds the media type at offset 38). Every sheet is a table of +content.xml+, one
+    # deflated zip entry streamed with zip_kit as rows arrive, built as strings in a buffer of about BUFFER_SIZE
+    # bytes: memory stays flat and the output may be non-seekable. The parts known only at the end, +styles.xml+
+    # (the styles the sheets used), +settings.xml+ (the frozen headers, by sheet name) and the manifest, follow it.
+    #
+    # Styles: +office:automatic-styles+ must precede the tables in +content.xml+, before the sheets are known, so
+    # the cell styles are common styles of +styles.xml+ instead, written at the end with those the sheets used:
+    # ODF lets a cell's +table:style-name+ name a common style, and LibreOffice applies it. They are the bold header,
+    # a number style per decimal +scale+ (the full value is stored, the style shows +scale+ decimals), dates,
+    # datetimes and booleans. LibreOffice ignores a column's common style, though, so the column widths are
+    # automatic styles, declared for every whole width from 1 to 255 characters (MAX_WIDTH, the most Excel allows):
+    # a width is rounded to whole characters and kept in that range, and any sheet can use any width.
+    #
+    # A :decimal cell with a +scale+ shows its value rounded half up to +scale+ decimals in its paragraph, as its number
+    # style does; +office:value+ keeps the full value.
+    #
+    # Text cells are +office:value-type="string"+, never evaluated, so "+39..." and "=..." stay verbatim.
+    # A paragraph collapses white space (ODF 1.3 part 3, §6.1.2), and the ODS reader too: a run of spaces after a
+    # character is written as one space and a +text:s+ for the rest, other spaces (leading, after a tab or a line
+    # break) as +text:s+, tabs as +text:tab+ and line feeds as +text:line-break+. A carriage return has no element:
+    # text that holds one also carries its exact value in +office:string-value+, which LibreOffice and the reader
+    # prefer to the paragraph. So does text with more than Odf::MAX_SPACES spaces, more than the reader takes from the
+    # +text:s+ of a cell; a +text:s+ counts at most Odf::MAX_SPACES spaces.
+    #
+    # @api private
+    class OdsWriter
+      include ZipBuffered
+
+      # The private method of zip_kit's ZipWriter that ZipWriter below overrides.
+      OVERRIDDEN_ZIP_METHOD = :timestamp_extra_for_local_file_header
+
+      # The override relies on a private method of zip_kit (the gemspec allows any 6.x from 6.3): fail on the first ODS
+      # write, with a clear message, rather than write a mimetype entry LibreOffice refuses. Not when the gem loads:
+      # a zip_kit change must not break the CSV and XLSX users.
+      # @raise [Calcsheet::Error] when +base+ does not define the method
+      def self.check_zip_writer!(base)
+        return if base.private_method_defined?(OVERRIDDEN_ZIP_METHOD) || base.method_defined?(OVERRIDDEN_ZIP_METHOD)
+
+        raise Calcsheet::Error, "zip_kit #{ZipKit::VERSION} has no ZipWriter##{OVERRIDDEN_ZIP_METHOD}, which the ODS writer " \
+                                "overrides to write the mimetype entry without an extra field; use a zip_kit release that has it"
+      end
+
+      # zip_kit's writer, except that the local header of the +mimetype+ entry has no extra field: zip_kit adds an
+      # extended timestamp to every entry.
+      class ZipWriter < ZipKit::ZipWriter
+        def write_local_file_header(filename:, **)
+          @bare = filename == Odf::MIMETYPE_ENTRY
+          super
+        ensure
+          @bare = false
+        end
+
+        private
+
+        def timestamp_extra_for_local_file_header(mtime)
+          @bare ? "" : super
+        end
+      end
+
+      # The widest column style, in characters (Column::MAX_WIDTH).
+      MAX_WIDTH = 255
+      # The ODF version the documents declare.
+      VERSION = "1.3"
+      DECLARATION = %(<?xml version="1.0" encoding="UTF-8"?>\n)
+      # The namespaces the root elements declare. LibreOffice resolves the prefix of the "ooo:view-settings" name:
+      # without ooo, it ignores the view settings.
+      NAMESPACES = Odf::NAMESPACES.slice(:office, :style, :text, :table, :fo, :number, :config, :ooo).freeze
+      MANIFEST_NS = "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"
+      # The namespace declarations and version of every root element.
+      ROOT_ATTRIBUTES = "#{NAMESPACES.map { |prefix, uri| %(xmlns:#{prefix}="#{uri}") }.join(" ")} office:version=\"#{VERSION}\"".freeze
+      # A width of one character, as LibreOffice converts the widths of XLSX columns.
+      INCHES_PER_CHARACTER = 0.0875
+      # The automatic column styles co1 to co255, one per whole width.
+      COLUMN_STYLES = (1..MAX_WIDTH).map do |width|
+        %(<style:style style:name="co#{width}" style:family="table-column"><style:table-column-properties ) +
+          %(style:column-width="#{format("%.4f", width * INCHES_PER_CHARACTER)}in"/></style:style>)
+      end.join.freeze
+
+      # Common cell styles by key: their name, data style and properties.
+      HEADER_STYLE = "Header"
+      DATE_STYLE = "Date"
+      DATETIME_STYLE = "Datetime"
+      BOOLEAN_STYLE = "Boolean"
+      # The text properties of the header style.
+      BOLD = '<style:text-properties fo:font-weight="bold" style:font-weight-asian="bold" style:font-weight-complex="bold"/>'
+      # The fields of the date and datetime data styles: yyyy-mm-dd and hh:mm:ss.
+      DATE_PARTS = '<number:year number:style="long"/><number:text>-</number:text><number:month number:style="long"/>' \
+                   '<number:text>-</number:text><number:day number:style="long"/>'
+      TIME_PARTS = '<number:text> </number:text><number:hours number:style="long"/><number:text>:</number:text>' \
+                   '<number:minutes number:style="long"/><number:text>:</number:text><number:seconds number:style="long"/>'
+
+      # The view settings of a sheet whose first row is frozen, as LibreOffice writes them.
+      FROZEN_HEADER = {
+        "HorizontalSplitMode" => [ "short", 0 ], "VerticalSplitMode" => [ "short", 2 ],
+        "HorizontalSplitPosition" => [ "int", 0 ], "VerticalSplitPosition" => [ "int", 1 ],
+        "ActiveSplitRange" => [ "short", 2 ], "PositionLeft" => [ "int", 0 ], "PositionRight" => [ "int", 0 ],
+        "PositionTop" => [ "int", 0 ], "PositionBottom" => [ "int", 1 ]
+      }.map { |name, (type, value)| %(<config:config-item config:name="#{name}" config:type="#{type}">#{value}</config:config-item>) }
+        .join.freeze
+
+      # The pieces of the cells: an empty cell, the end of a cell with its paragraph, the end of a value attribute
+      # and the start of the paragraph, and the start tags of text cells up to their attributes.
+      EMPTY_CELL = "<table:table-cell/>"
+      CELL_END = "</text:p></table:table-cell>"
+      VALUE_END = '"><text:p>'
+      STRING_START = '<table:table-cell office:value-type="string"'
+      HEADER_START = %(<table:table-cell table:style-name="#{HEADER_STYLE}" office:value-type="string").freeze
+      # The byte of the "T" in "YYYY-MM-DDThh:mm:ss", a space in the text shown.
+      TIME_SEPARATOR = 10
+      # The byte of a space.
+      SPACE = 32
+      # Text a paragraph would change: a tab, a line feed, a leading space or two spaces.
+      PARAGRAPH_SPACE = /[\t\n]|\A | {2}/
+      # A tab, a line feed, a run of spaces after another character, or one at the start or after a tab or line feed.
+      SPACE_TOKENS = /(\t)|(\n)|(?<=[^\t\n])( +)|( +)/
+      # What an attribute value would turn into spaces, as character references.
+      ATTRIBUTE_BREAKS = /[\t\n]/
+      ATTRIBUTE_SPACE = { "\t" => "&#9;", "\n" => "&#10;" }.freeze
+      # A carriage return, alone or before a line feed: one line break in the paragraph.
+      LINE_ENDING = /\r\n?/
+
+      # @return [Integer] rows per sheet, header included
+      def self.max_rows_per_sheet
+        Grid::MAX_ROWS
+      end
+
+      # @param io [#<<] the output, never closed here
+      # @param _options [Options] as returned by WriteBook.prepare_options; ODS has no format options
+      # @raise [Calcsheet::Error] when zip_kit has no longer the method the zip writer overrides
+      def initialize(io, _options)
+        self.class.check_zip_writer!(ZipKit::ZipWriter)
+        @zip = ZipKit::Streamer.new(io, writer: ZipWriter.new)
+        @zip.add_stored_entry(filename: Odf::MIMETYPE_ENTRY, size: Odf::MIMETYPE.bytesize, crc32: Zlib.crc32(Odf::MIMETYPE),
+                              use_data_descriptor: false)
+        @zip << Odf::MIMETYPE
+        @entry = @zip.write_deflated_file("content.xml")
+        @buffer = new_buffer
+        @buffer << content_start
+        @sheets = []
+        @styles = {}
+      end
+
+      # Starts the sheet's table in content.xml, with its columns and header row.
+      # @raise [ArgumentError] for more columns than ODS holds
+      def start_sheet(name, columns, header:)
+        check_columns(name, columns, "ODS")
+        @sheets << [ name, header ]
+        @name = name
+        @columns = columns
+        @types = columns.map(&:type)
+        @row_number = 0
+        prepare_cells(columns)
+        @buffer << %(<table:table table:name="#{XmlText.escape(name)}">) << table_columns(columns)
+        write_header(columns) if header
+      end
+
+      # Writes one row of checked values (see the contract in WriteBook), consecutive empty cells as one repeated cell.
+      # @raise [WriteError] for a year outside 1..9999; nothing of the row is written
+      def row(values)
+        mark = @buffer.bytesize
+        @buffer << "<table:table-row>"
+        empty = 0
+        index = 0
+        while index < values.size
+          value = values[index]
+          if value.nil?
+            empty += 1
+          else
+            if empty.positive?
+              @buffer << empty_cells(empty)
+              empty = 0
+            end
+            cell(value, index)
+          end
+          index += 1
+        end
+        @buffer << EMPTY_CELL if empty == values.size
+        @buffer << "</table:table-row>"
+        @row_number += 1
+        flush if @buffer.bytesize >= BUFFER_SIZE
+      rescue StandardError
+        # Nothing of a rejected row is written (a WriteError of a date beyond the range).
+        rollback(mark)
+        raise
+      end
+
+      # A table needs a row: a sheet without any gets an empty one.
+      def end_sheet
+        @buffer << "<table:table-row>#{EMPTY_CELL}</table:table-row>" if @row_number.zero?
+        @buffer << "</table:table>"
+        flush
+      end
+
+      # Ends content.xml and writes the parts that depend on the sheets, and the zip central directory. A spreadsheet
+      # needs a table: a book without any gets an empty one.
+      def finish
+        empty_book_sheet if @sheets.empty?
+        @buffer << "</office:spreadsheet></office:body></office:document-content>"
+        flush
+        @entry.close
+        write_parts("styles.xml" => styles_xml, "settings.xml" => settings_xml, "META-INF/manifest.xml" => manifest_xml)
+        @zip.close
+      end
+
+      # @api private
+      # The first value of a checked row that #row would refuse, without writing anything (see
+      # XlsxWriter#row_problem).
+      # @return [Array(Integer, String), nil] the column index and the reason
+      def row_problem(values)
+        values.each_with_index do |value, index|
+          next if value.nil? || !%i[date datetime].include?(@types[index]) || (value.year >= 1 && value.year <= 9999)
+
+          return [ index, year_reason(value) ]
+        end
+        nil
+      end
+
+      private
+
+      def content_start
+        %(#{DECLARATION}<office:document-content #{ROOT_ATTRIBUTES}><office:automatic-styles>#{COLUMN_STYLES}) +
+          "</office:automatic-styles><office:body><office:spreadsheet>"
+      end
+
+      # One table-column per column, consecutive ones of the same width merged; a table needs one.
+      def table_columns(columns)
+        styles = columns.map { |column| column.width && %( table:style-name="co#{column.width.round.clamp(1, MAX_WIDTH)}") }
+        return "<table:table-column/>" if styles.empty?
+
+        styles.chunk_while { _1 == _2 }.map do |run|
+          repeated = run.size > 1 ? %( table:number-columns-repeated="#{run.size}") : ""
+          "<table:table-column#{run.first}#{repeated}/>"
+        end.join
+      end
+
+      # The constant parts of each column's cells: the start up to the value, the part between the value and its text,
+      # and the whole cells of booleans.
+      def prepare_cells(columns)
+        @openings = columns.map do |column|
+          style = cell_style(column)
+          attributes = style ? %( table:style-name="#{style}") : ""
+          case column.type
+          when :integer, :decimal then %(<table:table-cell#{attributes} office:value-type="float" office:value=")
+          when :date, :datetime then %(<table:table-cell#{attributes} office:value-type="date" office:date-value=")
+          when :boolean then %(<table:table-cell#{attributes} office:value-type="boolean" office:boolean-value=")
+          end
+        end
+        @scales = columns.map { _1.type == :decimal ? _1.scale : nil }
+        @trues = @openings.map { %(#{_1}true#{VALUE_END}TRUE#{CELL_END}) if _1 }
+        @falses = @openings.map { %(#{_1}false#{VALUE_END}FALSE#{CELL_END}) if _1 }
+      end
+
+      # The common style of a column's data cells, registered for styles.xml at its first use.
+      def cell_style(column)
+        case column.type
+        when :decimal then column.scale && use_style("Decimal#{column.scale}")
+        when :date then use_style(DATE_STYLE)
+        when :datetime then use_style(DATETIME_STYLE)
+        when :boolean then use_style(BOOLEAN_STYLE)
+        end
+      end
+
+      def use_style(name)
+        @styles[name] = true
+        name
+      end
+
+      def empty_cells(count)
+        count == 1 ? EMPTY_CELL : %(<table:table-cell table:number-columns-repeated="#{count}"/>)
+      end
+
+      def write_header(columns)
+        use_style(HEADER_STYLE)
+        @buffer << "<table:table-row>"
+        columns.each { text_cell(HEADER_START, _1.header) }
+        @buffer << "</table:table-row>"
+        @row_number = 1
+      end
+
+      def cell(value, index)
+        case @types[index]
+        when :string then text_cell(STRING_START, value)
+        when :integer then number(value.to_s, index)
+        when :decimal
+          text = DecimalText.plain(value)
+          scale = @scales[index]
+          @buffer << @openings[index] << text << VALUE_END << (scale ? shown(value, text, scale) : text) << CELL_END
+        when :date
+          check_year(value, index)
+          text = value.iso8601
+          @buffer << @openings[index] << text << VALUE_END << text << CELL_END
+        when :datetime
+          check_year(value, index)
+          text = value.strftime("%Y-%m-%dT%H:%M:%S")
+          @buffer << @openings[index] << text << VALUE_END
+          text.setbyte(TIME_SEPARATOR, SPACE)
+          @buffer << text << CELL_END
+        else @buffer << (value ? @trues[index] : @falses[index]) # :boolean
+        end
+      end
+
+      def number(text, index)
+        @buffer << @openings[index] << text << VALUE_END << text << CELL_END
+      end
+
+      # The decimal rounded half up to +scale+ decimals, as its number style shows it; never "-0.00".
+      # @param text [String] the value written, digits with at most one "."
+      def shown(value, text, scale)
+        if value.is_a?(BigDecimal)
+          rounded = value.round(scale, BigDecimal::ROUND_HALF_UP)
+          rounded = rounded.abs if rounded.zero?
+          text = rounded.to_s("F")
+        end
+        dot = text.index(".")
+        return dot ? text[0, dot] : text if scale.zero?
+
+        missing = dot ? scale - (text.bytesize - dot - 1) : scale
+        "#{text}#{"." unless dot}#{"0" * missing}"
+      end
+
+      # Appends a text cell from +start+, its start tag up to the attributes. A carriage return has no element in a
+      # paragraph: text that holds one is also written exactly in office:string-value, and with line feeds instead in
+      # the paragraph; so is text with more spaces than the reader takes from a paragraph (counted generously).
+      def text_cell(start, text)
+        @buffer << start
+        # Only a text of more than Odf::MAX_SPACES bytes can hold more spaces: the others are not counted.
+        if text.include?("\r") || (text.bytesize > Odf::MAX_SPACES && text.count(" ") > Odf::MAX_SPACES)
+          @buffer << ' office:string-value="' << XmlText.escape(text).gsub(ATTRIBUTE_BREAKS, ATTRIBUTE_SPACE) << '"'
+          text = text.gsub(LINE_ENDING, "\n")
+        end
+        @buffer << "><text:p>" << paragraph(text) << CELL_END
+      end
+
+      # Escaped text whose spaces, tabs and line feeds survive the white-space collapsing of a paragraph.
+      def paragraph(text)
+        text = XmlText.escape(text)
+        return text unless PARAGRAPH_SPACE.match?(text)
+
+        text.gsub(SPACE_TOKENS) do
+          if $1 then "<text:tab/>"
+          elsif $2 then "<text:line-break/>"
+          elsif $3 then $3.size == 1 ? " " : " #{spaces($3.size - 1)}"
+          else spaces($4.size)
+          end
+        end
+      end
+
+      def spaces(count)
+        return "<text:s/>" if count == 1
+        return %(<text:s text:c="#{count}"/>) if count <= Odf::MAX_SPACES
+
+        full, rest = count.divmod(Odf::MAX_SPACES)
+        (%(<text:s text:c="#{Odf::MAX_SPACES}"/>) * full) + (rest.zero? ? "" : spaces(rest))
+      end
+
+      # @raise [WriteError] the year is before 1 or after 9999: xsd:date has no year 0, the readers read four digits
+      def check_year(value, index)
+        return if value.year >= 1 && value.year <= 9999
+
+        raise WriteError.at(year_reason(value), sheet: @name, row: @row_number + 1, column_index: index, header: @columns[index].header)
+      end
+
+      def year_reason(value)
+        "ODS dates are written from 0001-01-01 to 9999-12-31, got #{value.inspect}"
+      end
+
+      def styles_xml
+        data_styles = []
+        cell_styles = @styles.keys.map do |name|
+          case name
+          when HEADER_STYLE then %(<style:style style:name="#{name}" style:family="table-cell">#{BOLD}</style:style>)
+          else
+            data_styles << data_style(name)
+            %(<style:style style:name="#{name}" style:family="table-cell" style:data-style-name="N#{name}"/>)
+          end
+        end
+        %(#{DECLARATION}<office:document-styles #{ROOT_ATTRIBUTES}><office:styles>#{data_styles.join}#{cell_styles.join}) +
+          "</office:styles></office:document-styles>"
+      end
+
+      def data_style(name)
+        case name
+        when DATE_STYLE then %(<number:date-style style:name="N#{name}">#{DATE_PARTS}</number:date-style>)
+        when DATETIME_STYLE then %(<number:date-style style:name="N#{name}">#{DATE_PARTS}#{TIME_PARTS}</number:date-style>)
+        when BOOLEAN_STYLE then %(<number:boolean-style style:name="N#{name}"><number:boolean/></number:boolean-style>)
+        else
+          scale = name.delete_prefix("Decimal")
+          %(<number:number-style style:name="N#{name}"><number:number number:decimal-places="#{scale}" ) +
+            %(number:min-decimal-places="#{scale}" number:min-integer-digits="1"/></number:number-style>)
+        end
+      end
+
+      # The view settings LibreOffice reads: the first sheet active, and the first row frozen in each sheet with a
+      # header. A Tables map needs an entry, so it is left out when no sheet has a header.
+      def settings_xml
+        frozen = @sheets.filter_map do |name, header|
+          %(<config:config-item-map-entry config:name="#{XmlText.escape(name)}">#{FROZEN_HEADER}</config:config-item-map-entry>) if header
+        end
+        tables = frozen.empty? ? "" : %(<config:config-item-map-named config:name="Tables">#{frozen.join}</config:config-item-map-named>)
+        %(#{DECLARATION}<office:document-settings #{ROOT_ATTRIBUTES}><office:settings>) +
+          '<config:config-item-set config:name="ooo:view-settings"><config:config-item-map-indexed config:name="Views">' \
+          '<config:config-item-map-entry><config:config-item config:name="ViewId" config:type="string">view1</config:config-item>' +
+          tables +
+          %(<config:config-item config:name="ActiveTable" config:type="string">#{XmlText.escape(@sheets.first.first)}</config:config-item>) +
+          "</config:config-item-map-entry></config:config-item-map-indexed></config:config-item-set></office:settings>" \
+          "</office:document-settings>"
+      end
+
+      def manifest_xml
+        entries = %w[content.xml styles.xml settings.xml].map do |path|
+          %(<manifest:file-entry manifest:full-path="#{path}" manifest:media-type="text/xml"/>)
+        end
+        %(#{DECLARATION}<manifest:manifest xmlns:manifest="#{MANIFEST_NS}" manifest:version="#{VERSION}">) +
+          %(<manifest:file-entry manifest:full-path="/" manifest:version="#{VERSION}" manifest:media-type="#{Odf::MIMETYPE}"/>) +
+          "#{entries.join}</manifest:manifest>"
+      end
+    end
+  end
+end