calcsheet 0.1.0

data/lib/calcsheet/readers/csv_reader.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+require "csv"
+require "stringio"
+require_relative "../cast"
+require_relative "../errors"
+require_relative "../formula_protection"
+require_relative "../raw_row"
+require_relative "../reading"
+require_relative "../sheet_info"
+require_relative "csv_dialect"
+require_relative "csv_input"
+require_relative "row_batches"
+
+module Calcsheet
+  # The reader backends registered in Calcsheet.backends (CsvReader, XlsxReader, OdsReader), which implement the
+  # contract of Reading, and what they share: RowBatches, the SAX handling of SaxSheet and OxErrors, and the byte
+  # scanners of TagScan and StreamRemainder. Xlsx and Ods hold the parts of each format.
+  #
+  # @api private
+  module Readers
+    # The CSV reader backend (see Reading for the contract): one sheet, named after the file without its extension
+    # ("csv" for an IO without a path), parsed by the stdlib CSV in the detected dialect.
+    #
+    # Rows are numbered by CSV record, counting the header and blank lines, and +line+ is the physical line where
+    # the record starts. A seekable source is read again from its start whenever a request needs a record
+    # already parsed or another dialect; a non-seekable stream is read directly, forward only, so it can be read
+    # once. The options +encoding+, +col_sep+ and +quote_char+ of the read force the dialect; nil detects it.
+    #
+    # @api private
+    class CsvReader
+      include RowBatches
+
+      # The message of a second read of a stream read directly.
+      READ_TWICE = "a non-seekable CSV stream cannot be read twice; pass a file or seekable IO"
+      # The line break that does not belong to each line ending: CSV keeps it in an unquoted field. A stray one in a
+      # CRLF file CSV refuses itself, with the message UNQUOTED_BREAK starts.
+      STRAY_BREAKS = { "\n" => "\r", "\r" => "\n" }.freeze
+      UNQUOTED_BREAK = "Unquoted fields do not allow new line"
+
+      # @return [Boolean] false: a stream is read directly (see Source)
+      def self.needs_seek?
+        false
+      end
+
+      # @param source [Source]
+      # @param options [Options] the book options (+max_cell_bytes+); the read options come with each request's Cast
+      def initialize(source, options)
+        @source = source
+        @max_cell_bytes = options.max_cell_bytes
+        @name = sheet_name(source.path)
+        # The detection sample cannot be given back to a stream: it is replayed before the rest of it, once (#input
+        # consumes it; a stream is read once, see #restart).
+        @sample = source.io.read(CsvDialect::SAMPLE_BYTES).to_s.b unless source.seekable?
+        # The dialect options of the last restart, and the dialect they gave.
+        @forced = nil
+        @dialect = nil
+        # The parser and the Enumerator over its records (see #shift); nil when the next request starts again.
+        @csv = nil
+        @rows = nil
+        # Whether a stream has been read, which it can be only once.
+        @started = false
+        # The records parsed, and the physical line where the next one starts.
+        @records = 0
+        @line = 1
+      end
+
+      # The file name without its directories and extension; an upload's name may hold Windows separators.
+      # @return [String] "csv" without a name
+      def sheet_name(path)
+        name = path && File.basename(path.split(/[\\\/]/).last.to_s, ".*")
+        name.nil? || name.empty? ? "csv" : name
+      end
+
+      # @return [Array<SheetInfo>]
+      def sheets
+        [ SheetInfo.new(name: @name, index: 0) ]
+      end
+
+      # Drops the parser and the sample; the book closes the source.
+      def close
+        @csv = nil
+        @rows = nil
+        @sample = nil
+      end
+
+      private
+
+      # Yields the non-blank rows numbered +start+ or later (see RowBatches; a CSV has one sheet). The counters are
+      # updated before each yield, so that a caller breaking out leaves the parser on the next unread record.
+      def each_row(_sheet_index, cast, types, start)
+        forced = forced_dialect(cast.options)
+        # The parser goes on from where the last request stopped only when that is before +start+ and the dialect is
+        # the same: a request for a record already parsed (the header after a sample, a second read), another
+        # forced dialect or a parser that failed starts again from the first record.
+        restart(forced) if @csv.nil? || start <= @records || forced != @forced
+        strip = cast.options.strip_formula_prefix
+        while (fields = shift)
+          number = @records += 1
+          line = @line
+          raw = @csv.line
+          @line += line_breaks(raw)
+          @stray = STRAY_BREAKS.fetch(@row_sep = @csv.row_sep, false) if @stray.nil?
+          mixed_line_endings(number, line, @row_sep) if @stray && raw.include?(@stray) && stray_break?(raw)
+          next if number < start
+
+          last = fields.size - 1
+          last -= 1 while last >= 0 && blank_field?(fields[last], number, last)
+          next if last.negative?
+
+          fields = fields.first(last + 1) if last + 1 < fields.size
+          yield raw_row(number, line, fields, cast, types, strip)
+        end
+      end
+
+      # Parses from the first record again: a seekable source from its start, a stream from its replayed sample, which
+      # is possible once. The dialect is detected again only when the forced options changed.
+      # @raise [ArgumentError] READ_TWICE for a stream already read
+      def restart(forced)
+        raise ArgumentError, READ_TWICE if @started && !@source.seekable?
+
+        unless @dialect && forced == @forced
+          io = @source.seekable? ? @source.io : StringIO.new(@sample)
+          @dialect = CsvDialect.detect(io, seekable: @source.seekable?, **forced)
+          @forced = forced
+        end
+        @started = true
+        @csv = CSV.new(input, col_sep: @dialect.col_sep, quote_char: @dialect.quote_char, row_sep: @dialect.row_sep)
+        @stray = nil
+        @rows = nil
+        @records = 0
+        @line = 1
+      end
+
+      # The text the parser reads: the source after the byte order mark, or the sample (consumed here) and the rest of
+      # the stream.
+      def input
+        bom = @dialect.bom_size
+        return CsvInput.new(@source.io.tap { _1.seek(bom, IO::SEEK_SET) }, @dialect.encoding) if @source.seekable?
+
+        prefix = @sample.byteslice(bom, @sample.bytesize - bom)
+        @sample = nil
+        CsvInput.new(@source.io, @dialect.encoding, prefix: prefix)
+      end
+
+      # The next record, nil at the end. Records are pulled from an Enumerator over CSV#each, not CSV#shift: in csv
+      # 3.3.x each shift abandons a parse and leaks one scanner position, so memory grows with the records read. The
+      # parser is created, and reads its first bytes, by the first call, inside the error handling below.
+      def shift
+        (@rows ||= @csv.each).next
+      rescue StopIteration
+        nil
+      rescue CSV::MalformedCSVError, EncodingError => error
+        row_sep = @csv.row_sep
+        @csv = @rows = nil
+        record = @records + 1
+        mixed_line_endings(record, @line, row_sep) if error.message.start_with?(UNQUOTED_BREAK)
+        raise MalformedFileError.new("CSV record #{record} (line #{@line}): #{error.message}", sheet: @name, row: record)
+      rescue StandardError
+        # A parser that failed (a size limit, an IO error) cannot continue: the next request starts again.
+        @csv = @rows = nil
+        raise
+      end
+
+      # Physical lines a record ends: "\r\n", "\n" and "\r" each end one, inside quoted fields too.
+      def line_breaks(raw)
+        breaks = raw.count("\n")
+        position = 0
+        while (cr = raw.byteindex("\r", position))
+          breaks += 1 unless raw.getbyte(cr + 1) == 0x0A
+          position = cr + 1
+        end
+        breaks
+      end
+
+      # Whether the stray line break of a record is outside its quoted fields: the parts between quote characters
+      # alternate outside and inside, a doubled quote giving an empty inside part.
+      def stray_break?(raw)
+        raw.split(@dialect.quote_char, -1).each_slice(2).any? { |outside, _inside| outside.include?(@stray) }
+      end
+
+      # Records end with the first line ending of the file; another one outside quoted fields would leave a CR in
+      # a value or merge two records.
+      def mixed_line_endings(record, line, row_sep)
+        @csv = @rows = nil
+        raise MalformedFileError.new("CSV record #{record} (line #{line}): mixed line endings: the file's lines end " \
+                                     "with #{row_sep.inspect}, this record has another line ending outside " \
+                                     "quoted fields", sheet: @name, row: record)
+      end
+
+      # The dialect options of a read (checked by OptionValidation, except whether the encoding can be read): nil
+      # detects.
+      def forced_dialect(options)
+        { encoding: options.encoding && readable_encoding(options.encoding), col_sep: options.col_sep,
+          quote_char: options.quote_char }
+      end
+
+      # The encoding named by +value+, when it can be transcoded to UTF-8.
+      def readable_encoding(value)
+        encoding = Encoding.find(value)
+        Encoding::Converter.new(encoding, Encoding::UTF_8) unless encoding == Encoding::UTF_8
+        encoding
+      rescue ArgumentError, Encoding::ConverterNotFoundError => error
+        raise ArgumentError, "encoding: cannot read #{value.inspect} (#{error.message})"
+      end
+
+      def raw_row(number, line, fields, cast, types, strip)
+        errors = nil
+        # Array.new with a block rather than each_with_index.map, which allocates an Enumerator per row.
+        values = Array.new(fields.size) do |index|
+          field = fields[index]
+          next if field.nil? || field.empty?
+
+          check_size(field, number, index) if @max_cell_bytes && field.bytesize > @max_cell_bytes
+          field = FormulaProtection.unprotect(field) if strip
+          cast.text(field, types&.[](index))
+        rescue Cast::Failure => failure
+          (errors ||= []) << [ index, field, failure.reason ]
+          nil
+        end
+        RawRow.new(number: number, line: line, values: values, errors: errors)
+      end
+
+      # Every field is measured before a regexp sees it: here those from the end to the last non-blank one, the
+      # others in raw_row.
+      def blank_field?(field, number, index)
+        return true if field.nil?
+
+        check_size(field, number, index) if @max_cell_bytes && field.bytesize > @max_cell_bytes
+        !Reading::NON_BLANK.match?(field)
+      end
+
+      def check_size(field, number, index)
+        raise MalformedFileError.at("the field is longer than max_cell_bytes (#{@max_cell_bytes} bytes)",
+                                    sheet: @name, row: number, column_index: index)
+      end
+    end
+  end
+end

data/lib/calcsheet/readers/ods/cell_formats.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Readers
+    module Ods
+      # What the styles of an ODS document say about how a value reads, gathered by Scanner from the data
+      # styles and the cell styles of styles.xml and of the automatic styles of content.xml: a date style that shows a
+      # time or none, and a plain zero-padding number style (+00000+: one number:number without decimals or grouping,
+      # nothing else). The Scanner feeds the elements in document order; #to_h resolves each cell style through its
+      # parents.
+      #
+      # @api private
+      class CellFormats
+        # Parents followed from a cell style to the one that names its data style; a longer chain is not followed.
+        MAX_PARENTS = 8
+
+        # @return [Symbol, nil] :date or :number while inside a data style that matters, else nil
+        attr_reader :current
+        # @return [Integer, nil] the digits of the zero-padding number of the current number style
+        attr_reader :digits
+
+        # +@cells+ holds +[data style, parent]+ by cell style name, +@formats+ the +[kind, pad]+ by data style name.
+        def initialize
+          @cells = {}
+          @formats = {}
+          @current = nil
+        end
+
+        # @param name [String] a cell style
+        # @param data [String, nil] its data style
+        # @param parent [String, nil] its parent style
+        def cell_style(name, data, parent)
+          @cells[name] = [ data, parent ].freeze
+        end
+
+        # Starts a data style of +kind+ (:date or :number) named +name+.
+        def start(kind, name)
+          @current = kind
+          @name = name
+          @time = false
+          @digits = nil
+          @plain = true
+        end
+
+        # A time field of the current date style.
+        def time!
+          @time = true
+        end
+
+        # Whether the current number style may still be a plain zero-padding format.
+        def plain?
+          @plain
+        end
+
+        # The first number:number of the current number style: the digits of a zero-padding format, or nil.
+        def number!(digits)
+          @digits = digits
+          @plain = false if digits.nil?
+        end
+
+        # Another element in the current number style: it is not a plain zero-padding format.
+        def not_plain!
+          @plain = false
+        end
+
+        # Keeps the format of the data style that ends, when it has one (a date style; a number style that stayed a
+        # plain zero padding), and leaves it.
+        def finish
+          if @name && @current == :date
+            @formats[@name] = [ @time ? :datetime : :date, nil ].freeze
+          elsif @name && @current == :number && @plain && @digits
+            @formats[@name] = [ :general, @digits ].freeze
+          end
+          @current = nil
+        end
+
+        # @param other [CellFormats] styles that win over these (content.xml over styles.xml)
+        # @return [CellFormats]
+        def merge(other)
+          merged = CellFormats.new
+          merged.absorb(@cells.merge(other.cells), @formats.merge(other.formats))
+          merged
+        end
+
+        # @return [Hash{String => Array(Symbol, Integer | nil)}] +[kind, pad]+ (see Scanner#cell_formats) of each cell
+        #   style whose data style, its own or its nearest parent's, has one
+        def to_h
+          @cells.each_key.filter_map { |name| (format = resolve(name)) && [ name, format ] }.to_h.freeze
+        end
+
+        protected
+
+        attr_reader :cells, :formats
+
+        def absorb(cells, formats)
+          @cells = cells
+          @formats = formats
+        end
+
+        private
+
+        def resolve(name)
+          MAX_PARENTS.times do
+            data, parent = @cells[name]
+            return @formats[data] if data
+            return if parent.nil?
+
+            name = parent
+          end
+          nil
+        end
+      end
+    end
+  end
+end