calcsheet 0.1.0

data/lib/calcsheet/option_validation.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative "cast"
+require_relative "time_zone"
+
+module Calcsheet
+  # Checks the type and value of every option when Configuration#options resolves them, so that a wrong value fails
+  # at the call that gives it (Calcsheet.open, Calcsheet.write, Calcsheet.stream, a read with overrides) or, for a
+  # configured value, at the next Calcsheet.open, Calcsheet.write or Calcsheet.stream (an open book keeps its snapshot),
+  # naming the option, instead of in the middle of a read or a write. The README options reference describes the
+  # accepted values; each method below raises ArgumentError "key: must be ..., got ..." for another one.
+  #
+  # @api private
+  module OptionValidation
+    # Limits where nil means none.
+    LIMITS = %i[max_file_size max_rows max_sheets max_cell_bytes max_rows_per_sheet].freeze
+    # The zip-bomb limits: always on, so that a blank configuration value cannot switch them off.
+    ZIP_LIMITS = %i[max_uncompressed_size max_zip_entries].freeze
+    # The options that are true or false.
+    BOOLEANS = %i[strip_formula_prefix excel_it sanitize_sheet_names].freeze
+    # The decimal separators of text read as numbers and of CSV output.
+    DECIMAL_SEPARATORS = [ ".", "," ].freeze
+    # The thousands separators: none, ".", ",", a space, an apostrophe, the no-break space U+00A0 and the narrow
+    # no-break space U+202F.
+    THOUSANDS_SEPARATORS = [ nil, ".", ",", " ", "'", " ", " " ].freeze
+    # The line endings of CSV output.
+    ROW_SEPARATORS = [ "\n", "\r\n", "\r" ].freeze
+    # Characters a CSV separator or quote cannot be: line breaks, and the double quote for the separator.
+    LINE_BREAKS = [ "\n", "\r" ].freeze
+    # A strptime directive of a date format, or a "%" that ends it.
+    DIRECTIVE = /%(.)|%\z/m
+
+    module_function
+
+    # @param options [Hash{Symbol => Object}] every key of Configuration::DEFAULTS
+    # @return [Hash] +options+
+    # @raise [ArgumentError] naming the first invalid option
+    def check!(options)
+      LIMITS.each { |key| limit(key, options[key]) }
+      ZIP_LIMITS.each { |key| positive(key, options[key]) }
+      ratio(options[:max_compression_ratio])
+      BOOLEANS.each { |key| boolean(key, options[key]) }
+      non_negative(:compression_ratio_floor, options[:compression_ratio_floor], nil_allowed: true)
+      positive(:batch_size, options[:batch_size])
+      non_negative(:sample_rows, options[:sample_rows], nil_allowed: false)
+      one_of(:decimals, options[:decimals], %i[bigdecimal float])
+      TimeZone.wrap(options[:time_zone])
+      separators(options[:decimal_separator], options[:thousands_separator])
+      %i[date_formats datetime_formats].each { |key| formats(key, options[key]) }
+      %i[true_words false_words].each { |key| strings(key, options[key]) }
+      csv(options)
+      options
+    end
+
+    # A positive Integer or nil.
+    def limit(key, value)
+      return if value.nil? || (value.is_a?(Integer) && value.positive?)
+
+      invalid(key, "a positive Integer or nil", value)
+    end
+
+    # max_compression_ratio: a positive finite real number, never nil.
+    def ratio(value)
+      return if value.is_a?(Numeric) && value.real? && value.finite? && value.positive?
+
+      invalid(:max_compression_ratio, "a positive number", value)
+    end
+
+    # A positive Integer, never nil.
+    def positive(key, value)
+      invalid(key, "a positive Integer", value) unless value.is_a?(Integer) && value.positive?
+    end
+
+    # A non-negative Integer, or nil when +nil_allowed+.
+    def non_negative(key, value, nil_allowed:)
+      return if nil_allowed && value.nil?
+      return if value.is_a?(Integer) && !value.negative?
+
+      invalid(key, nil_allowed ? "a non-negative Integer or nil" : "a non-negative Integer", value)
+    end
+
+    # true or false, not truthy values.
+    def boolean(key, value)
+      invalid(key, "true or false", value) unless true.equal?(value) || false.equal?(value)
+    end
+
+    # A member of +allowed+.
+    def one_of(key, value, allowed)
+      invalid(key, "one of #{allowed.map(&:inspect).join(", ")}", value) unless allowed.include?(value)
+    end
+
+    # The decimal and thousands separators, which must differ.
+    def separators(decimal, thousands)
+      one_of(:decimal_separator, decimal, DECIMAL_SEPARATORS)
+      one_of(:thousands_separator, thousands, THOUSANDS_SEPARATORS)
+      return unless thousands == decimal
+
+      raise ArgumentError, "thousands_separator: must differ from decimal_separator, both are #{decimal.inspect}"
+    end
+
+    # strptime formats that use only the directives Cast reads (Cast::FORMAT_DIRECTIVES).
+    def formats(key, value)
+      strings(key, value)
+      value.each do |format|
+        format.scan(DIRECTIVE) do |(directive)|
+          next if Cast::FORMAT_DIRECTIVES.key?(directive)
+
+          raise ArgumentError, "#{key}: #{format.inspect} uses the unsupported directive %#{directive} " \
+                               "(supported: #{Cast::FORMAT_DIRECTIVES.keys.map { "%#{_1}" }.join(" ")})"
+        end
+      end
+    end
+
+    # An Array of non-empty Strings.
+    def strings(key, value)
+      return if value.is_a?(Array) && value.all? { _1.is_a?(String) && !_1.empty? }
+
+      invalid(key, "an Array of non-empty Strings", value)
+    end
+
+    # The CSV options: the dialect forced when reading, and the writing ones.
+    def csv(options)
+      encoding(options[:encoding])
+      col_sep, quote_char = options.values_at(:col_sep, :quote_char)
+      character(:col_sep, col_sep, LINE_BREAKS + [ "\"" ])
+      character(:quote_char, quote_char, LINE_BREAKS)
+      raise ArgumentError, "col_sep: and quote_char: must differ, both are #{col_sep.inspect}" if col_sep && col_sep == quote_char
+
+      one_of(:row_sep, options[:row_sep], ROW_SEPARATORS)
+      one_of(:bom, options[:bom], [ nil, true, false ])
+      %i[date_format datetime_format].each do |key|
+        invalid(key, "a String or nil", options[key]) unless options[key].nil? || options[key].is_a?(String)
+      end
+    end
+
+    # nil, or an Encoding or the name of one Ruby knows (whether it can be read is checked by the CSV reader).
+    def encoding(value)
+      return if value.nil?
+
+      invalid(:encoding, "an Encoding, a String or nil", value) unless value.is_a?(Encoding) || value.is_a?(String)
+      Encoding.find(value)
+    rescue ArgumentError => error
+      raise if error.message.start_with?("encoding:")
+
+      raise ArgumentError, "encoding: unknown encoding #{value.inspect}"
+    end
+
+    # nil, or one valid character not in +refused+.
+    def character(key, value, refused)
+      return if value.nil? || (value.is_a?(String) && value.length == 1 && value.valid_encoding? && !refused.include?(value))
+
+      invalid(key, "nil or one character other than #{refused.map(&:inspect).join(", ")}", value)
+    end
+
+    # @raise [ArgumentError] naming +key+, what it must be and the value given
+    def invalid(key, expected, value)
+      raise ArgumentError, "#{key}: must be #{expected}, got #{value.inspect}"
+    end
+  end
+end

data/lib/calcsheet/options.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+
+module Calcsheet
+  # Frozen value object holding the resolved options of one operation, built and checked by Configuration#options.
+  # Its members are the keys of Configuration::DEFAULTS, the single table to extend; each one is described in the
+  # options reference of the README. Book#options gives those of a book.
+  Options = Data.define(*Configuration::DEFAULTS.keys)
+end

data/lib/calcsheet/raw_row.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  # A non-blank row as a reader backend produces it (see Reading for the backend contract).
+  #
+  # @attr_reader number [Integer] the row number the user sees
+  # @attr_reader line [Integer, nil] the physical line where it starts (CSV); nil otherwise
+  # @attr_reader values [Array] in file order, already cast, with trailing blank cells trimmed
+  # @attr_reader errors [Array<Array(Integer, String, Symbol)>, nil] nil, or +[column_index, raw_value, reason]+ for
+  #   each cell whose cast failed (its value is nil), in column order
+  RawRow = Data.define(:number, :line, :values, :errors) do
+    # @param line [Integer, nil] default nil
+    # @param errors [Array, nil] default nil
+    def initialize(number:, values:, line: nil, errors: nil)
+      super
+    end
+  end
+end

data/lib/calcsheet/raw_types.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  # The declared types a backend applies, by source column (see Reading for the backend contract).
+  #
+  # @attr_reader by_index [Array<Symbol, nil>] the type of each column; nil entries and indices beyond it fall back to
+  #   +default+
+  # @attr_reader default [Symbol, nil] the type of every other column; nil for the native value
+  RawTypes = Data.define(:by_index, :default) do
+    # @return [Symbol, nil] the type of the column, nil for the native value
+    def [](index)
+      by_index[index] || default
+    end
+  end
+end

data/lib/calcsheet/readers/csv_dialect.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Calcsheet
+  module Readers
+    # Detects how a CSV file is written: its encoding, byte order mark, column separator and quote character.
+    # Only the encoding of a seekable source reads the whole file, and then only as a streaming byte-level
+    # UTF-8 check; the separator and the quote are decided on a sample.
+    #
+    # @api private
+    module CsvDialect
+      # The dialect of a CSV file.
+      #
+      # @attr_reader encoding [Encoding] the encoding of the file's bytes (UTF-8, UTF-16LE/BE, Windows-1252 or a
+      #   forced one)
+      # @attr_reader bom_size [Integer] the bytes of the byte order mark to skip, 0 when there is none
+      # @attr_reader col_sep [String] the column separator
+      # @attr_reader quote_char [String] the quote character
+      # @attr_reader row_sep [String, Symbol] the first line ending outside quoted fields ("\n", "\r\n" or "\r"), or
+      #   :auto when the sample has none and the parser decides
+      Dialect = Data.define(:encoding, :bom_size, :col_sep, :quote_char, :row_sep)
+
+      # The bytes of the sample that decides the separator, the quote, the line ending and a stream's encoding.
+      SAMPLE_BYTES = 64 * 1024
+      # The records (or lines) of the sample that decide.
+      SAMPLE_RECORDS = 50
+      # The bytes read at a time by the UTF-8 check of a whole file.
+      VALIDATION_CHUNK_SIZE = 64 * 1024
+      # The byte order marks and their encodings.
+      BOMS = [
+        [ "\xEF\xBB\xBF".b.freeze, Encoding::UTF_8 ],
+        [ "\xFF\xFE".b.freeze, Encoding::UTF_16LE ],
+        [ "\xFE\xFF".b.freeze, Encoding::UTF_16BE ]
+      ].freeze
+      # Candidates in tie-breaking order: the first one wins when several fit equally well.
+      SEPARATORS = [ ";", ",", "\t", "|" ].freeze
+      # The quote characters detection chooses from.
+      QUOTES = [ "\"", "'" ].freeze
+      # The separator of a sample that no candidate parses into records of more than one field.
+      DEFAULT_SEPARATOR = ","
+      # The quote character tried first, and kept unless the single quote is clearly better.
+      DEFAULT_QUOTE = "\""
+
+      module_function
+
+      # Detects the dialect of a CSV source. A seekable IO is read from its start and left at 0. A non-seekable
+      # one cannot be rewound: the sample (at most 64 KiB) is consumed, so the caller must buffer it or have
+      # copied the stream to a seekable one beforehand.
+      #
+      # @param io [IO] a binary-mode IO, positioned at 0 unless +seekable+ is true
+      # @param seekable [Boolean] whether the whole file may be validated and the IO rewound
+      # @param encoding [Encoding, String, nil] forces the encoding and skips its detection
+      # @param col_sep [String, nil] forces the separator
+      # @param quote_char [String, nil] forces the quote character
+      # @return [Dialect]
+      def detect(io, seekable:, encoding: nil, col_sep: nil, quote_char: nil)
+        io.seek(0, IO::SEEK_SET) if seekable
+        head = io.read(SAMPLE_BYTES).to_s.b
+        bom_encoding, bom_size = byte_order_mark(head)
+        encoding = encoding ? Encoding.find(encoding) : (bom_encoding || guess_encoding(io, head, seekable))
+        bom_size = 0 unless bom_encoding == encoding
+        text = sample_text(head, encoding, bom_size)
+        cut = head.bytesize >= SAMPLE_BYTES
+        col_sep, quote_char = detect_separator_and_quote(text, cut, col_sep, quote_char)
+        row_sep = row_separator(text, quote_char, cut)
+        Dialect.new(encoding: encoding, bom_size: bom_size, col_sep: col_sep, quote_char: quote_char, row_sep: row_sep)
+      ensure
+        io.seek(0, IO::SEEK_SET) if seekable
+      end
+
+      def byte_order_mark(head)
+        BOMS.each do |bom, encoding|
+          return [ encoding, bom.bytesize ] if head.start_with?(bom)
+        end
+        [ nil, 0 ]
+      end
+
+      # UTF-8 when the bytes are valid UTF-8, Windows-1252 otherwise. A seekable source is checked in full; a
+      # stream only on its sample.
+      def guess_encoding(io, head, seekable)
+        valid = seekable ? valid_utf8_file?(io) : valid_utf8_sample?(head)
+        valid ? Encoding::UTF_8 : Encoding::WINDOWS_1252
+      end
+
+      # Streams the file in fixed-size chunks read into one reused buffer, so that checking a large file allocates
+      # nothing per chunk. A multibyte sequence cut by a chunk boundary is not an error: the IO moves back to its
+      # start and the next chunk begins there.
+      def valid_utf8_file?(io)
+        io.seek(0, IO::SEEK_SET)
+        buffer = String.new(capacity: VALIDATION_CHUNK_SIZE)
+        while io.read(VALIDATION_CHUNK_SIZE, buffer) && !buffer.empty?
+          next if buffer.force_encoding(Encoding::UTF_8).valid_encoding?
+
+          body, tail = split_incomplete_tail(buffer)
+          return false unless body.force_encoding(Encoding::UTF_8).valid_encoding? && !tail.empty?
+          return false if io.eof?
+
+          io.seek(-tail.bytesize, IO::SEEK_CUR)
+        end
+        true
+      end
+
+      # The first 50 lines of the sample decide; a multibyte character cut by the end of a full sample is not
+      # an error.
+      def valid_utf8_sample?(head)
+        sample = first_lines(head)
+        sample = split_incomplete_tail(sample).first if head.bytesize >= SAMPLE_BYTES
+        utf8?(sample)
+      end
+
+      def first_lines(bytes)
+        position = 0
+        SAMPLE_RECORDS.times do
+          newline = bytes.index("\n", position)
+          return bytes unless newline
+
+          position = newline + 1
+        end
+        bytes.byteslice(0, position)
+      end
+
+      def utf8?(bytes)
+        bytes.dup.force_encoding(Encoding::UTF_8).valid_encoding?
+      end
+
+      # @api private
+      # Splits off a trailing UTF-8 lead byte with fewer continuation bytes than its sequence needs (CsvInput reads
+      # in chunks with it).
+      #
+      # @return [Array(String, String)] the complete part and the incomplete tail (empty when none)
+      def split_incomplete_tail(bytes)
+        size = bytes.bytesize
+        1.upto([ 3, size ].min) do |back|
+          byte = bytes.getbyte(size - back)
+          next if byte.between?(0x80, 0xBF)
+          return [ bytes.byteslice(0, size - back), bytes.byteslice(size - back, back) ] if sequence_length(byte) > back
+
+          break
+        end
+        [ bytes, "".b ]
+      end
+
+      def sequence_length(lead)
+        case lead
+        when 0xC2..0xDF then 2
+        when 0xE0..0xEF then 3
+        when 0xF0..0xF4 then 4
+        else 1
+        end
+      end
+
+      # The double quote is the default and wins ties: the single quote is chosen only when no separator parses the
+      # sample with a double quote (a failure that the sample's cut alone causes does not count) and the single
+      # quote parses it with consistent records of more than one field. A single quote is never inferred from
+      # records that a double quote reads as well, so that a value such as "Smith, John" in a one-column file stays
+      # one field.
+      def detect_separator_and_quote(text, cut, col_sep, quote_char)
+        return [ col_sep, quote_char ] if col_sep && quote_char
+
+        separators = col_sep ? [ col_sep ] : SEPARATORS
+        scores = scores(text, separators, quote_char || DEFAULT_QUOTE, cut)
+        best = best(scores)
+        if best.nil? && quote_char.nil?
+          best = best(scores(text, separators, "'", cut))
+          best = nil if best&.records&.zero?
+        end
+        best ||= scores.max_by(&:key)
+        return [ col_sep || DEFAULT_SEPARATOR, best.quote_char ] if best.records.zero?
+
+        [ best.col_sep, best.quote_char ]
+      end
+
+      # A candidate's parse of the sample. +records+ is the size of the largest group of the first SAMPLE_RECORDS
+      # records that have the same number of fields, more than one: the most consistent column count, not the most
+      # fields, so that a separator inside quoted fields does not mislead it. +failed+ is whether the parser rejected
+      # a record (see #field_counts).
+      #
+      # @attr col_sep [String] the candidate separator
+      # @attr quote_char [String] the candidate quote character
+      # @attr records [Integer] the size of the largest group of records with the same field count, above one
+      # @attr failed [Boolean] whether the parser rejected a record
+      Score = Struct.new(:col_sep, :quote_char, :records, :failed) do
+        # Ranking: the most consistent records, then the first separator in SEPARATORS.
+        # @return [Array(Integer, Integer)]
+        def key
+          [ records, -SEPARATORS.index(col_sep).to_i ]
+        end
+      end
+
+      # Each candidate parses the sample with the line ending that the quote character gives.
+      def scores(text, separators, quote_char, cut)
+        row_sep = row_separator(text, quote_char, cut)
+        separators.map { |col_sep| score(text, col_sep, quote_char, row_sep, cut) }
+      end
+
+      # The best candidate that parses the sample, nil when every one fails.
+      def best(scores)
+        scores.reject(&:failed).max_by(&:key)
+      end
+
+      def score(text, col_sep, quote_char, row_sep, cut)
+        counts, failed = field_counts(text, col_sep, quote_char, row_sep, cut)
+        records = counts.select { _1 > 1 }.tally.values.max || 0
+        Score.new(col_sep, quote_char, records, failed)
+      end
+
+      # Field count of each of the first 50 records, and whether the parser rejected a record. A quoted field
+      # still open at the end of a cut sample is not a failure: the cut made it, not the file.
+      def field_counts(text, col_sep, quote_char, row_sep, cut)
+        counts = []
+        csv = CSV.new(text, col_sep: col_sep, quote_char: quote_char, row_sep: row_sep)
+        while counts.size < SAMPLE_RECORDS && (row = csv.shift)
+          counts << row.size
+        end
+        [ counts, false ]
+      rescue CSV::MalformedCSVError => error
+        [ counts, !(cut && error.message.start_with?("Unclosed quoted field")) ]
+      end
+
+      # The first line ending outside quoted fields; :auto when there is none, or when it is a CR that ends a cut
+      # sample (the LF of a CRLF may follow beyond the cut).
+      def row_separator(text, quote_char, cut)
+        pattern = /[\r\n#{Regexp.escape(quote_char)}]/
+        quoted = false
+        position = 0
+        while (index = text.byteindex(pattern, position))
+          char = Regexp.last_match(0)
+          position = index + char.bytesize
+          if char == quote_char
+            quoted = !quoted
+          elsif !quoted
+            return "\n" if char == "\n"
+            return :auto if cut && position == text.bytesize
+
+            return text.getbyte(position) == 0x0A ? "\r\n" : "\r"
+          end
+        end
+        :auto
+      end
+
+      # The sample as UTF-8 text, without the BOM. A sample that is cut by its size limit loses its last,
+      # possibly incomplete line (and any character or code unit it cut).
+      def sample_text(head, encoding, bom_size)
+        text = to_utf8(head.byteslice(bom_size, head.bytesize - bom_size), encoding)
+        return text if head.bytesize < SAMPLE_BYTES
+
+        last_break = text.rindex(/[\r\n]/)
+        last_break ? text[0..last_break] : text
+      end
+
+      def to_utf8(bytes, encoding)
+        return bytes.dup.force_encoding(Encoding::UTF_8).scrub if encoding == Encoding::UTF_8
+
+        bytes = bytes.byteslice(0, bytes.bytesize & ~1) if [ Encoding::UTF_16LE, Encoding::UTF_16BE ].include?(encoding)
+        bytes.dup.force_encoding(encoding).encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      end
+
+      private_class_method :byte_order_mark, :guess_encoding, :valid_utf8_file?, :valid_utf8_sample?, :first_lines, :utf8?,
+                           :sequence_length, :detect_separator_and_quote, :scores, :best, :score, :field_counts,
+                           :row_separator, :sample_text, :to_utf8
+    end
+  end
+end

data/lib/calcsheet/readers/csv_input.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require_relative "csv_dialect"
+
+module Calcsheet
+  module Readers
+    # The bytes of a CSV file as UTF-8 text for the stdlib CSV parser. It replays a prefix already read (the
+    # detection sample of a stream), transcodes UTF-16 and Windows-1252, and checks UTF-8, reading the IO in chunks
+    # with +read+ only. It answers what CSV::Parser uses: +gets(separator, limit)+, +eof?+ and the encodings.
+    #
+    # An invalid or undefined byte sequence raises its EncodingError only once the text before it has been read,
+    # so the parser fails on the record that holds it.
+    #
+    # @api private
+    class CsvInput
+      # The bytes read from the IO at a time.
+      CHUNK_SIZE = 64 * 1024
+
+      # @param io [IO] binary bytes, positioned after the byte order mark
+      # @param encoding [Encoding] the encoding of the bytes
+      # @param prefix [String] bytes read from +io+ already, returned first
+      # @param chunk_size [Integer] bytes read from +io+ at a time
+      def initialize(io, encoding, prefix: "".b, chunk_size: CHUNK_SIZE)
+        @io = io
+        @chunk_size = chunk_size
+        @converter = Encoding::Converter.new(encoding, Encoding::UTF_8) unless encoding == Encoding::UTF_8
+        @bytes = prefix.b
+        @text = String.new(encoding: Encoding::UTF_8)
+        @position = 0
+        @error = nil
+        @eof = false
+      end
+
+      # @return [Encoding] UTF-8, the encoding of the text handed out
+      def external_encoding
+        Encoding::UTF_8
+      end
+
+      # @return [nil] no transcoding by the parser
+      def internal_encoding
+        nil
+      end
+
+      # Like IO#gets: the text up to and including +separator+ (nil: no separator), at most +limit+ bytes (nil: no
+      # limit) rounded up to a whole character; nil at the end.
+      # @raise [EncodingError] the next bytes are not valid in the encoding
+      def gets(separator = $/, limit = nil)
+        length = line_length(separator, limit)
+        length = line_length(separator, limit) while length.nil? && fill
+        length ||= @text.bytesize - @position
+        if length.zero?
+          raise @error if @error
+
+          return nil
+        end
+        take(length)
+      end
+
+      # @return [Boolean] whether no text is left; false while an encoding error waits to be raised by #gets
+      def eof?
+        nil while @position == @text.bytesize && fill
+        @position == @text.bytesize && @error.nil?
+      end
+      alias eof eof?
+
+      private
+
+      def line_length(separator, limit)
+        stop = separator && @text.byteindex(separator, @position)
+        if stop
+          length = stop - @position + separator.bytesize
+          return limit && limit < length ? limit : length
+        end
+        limit if limit && @text.bytesize - @position >= limit
+      end
+
+      def take(length)
+        length += 1 while @position + length < @text.bytesize && @text.getbyte(@position + length).between?(0x80, 0xBF)
+        chunk = @text.byteslice(@position, length)
+        @position += length
+        chunk
+      end
+
+      # Decodes more bytes into the text. Returns false when nothing more can come: the end, or an invalid sequence.
+      def fill
+        return false if @error || @eof
+
+        chunk = @io.read(@chunk_size)
+        # A duck-typed read(length) may answer "" instead of nil at the end.
+        chunk.nil? || chunk.empty? ? @eof = true : @bytes << chunk.b
+        decoded = @converter ? convert : validate
+        @text = @text.byteslice(@position, @text.bytesize - @position) if @position.positive?
+        @position = 0
+        @text << decoded
+        true
+      end
+
+      # Transcodes the bytes, keeping an incomplete trailing sequence in the converter until the end.
+      def convert
+        output = String.new(encoding: Encoding::UTF_8)
+        result = @converter.primitive_convert(@bytes, output, nil, nil, @eof ? 0 : Encoding::Converter::PARTIAL_INPUT)
+        unless result == :source_buffer_empty || result == :finished
+          @error = @converter.last_error
+          @bytes.clear
+        end
+        output
+      end
+
+      # The valid UTF-8 prefix of the bytes; an incomplete trailing character waits for the next chunk.
+      def validate
+        body, @bytes = @eof ? [ @bytes, "".b ] : CsvDialect.split_incomplete_tail(@bytes)
+        text = body.force_encoding(Encoding::UTF_8)
+        return text if text.valid_encoding?
+
+        valid = 0
+        invalid = text.each_char.find do |char|
+          next true unless char.valid_encoding?
+
+          valid += char.bytesize
+          false
+        end
+        @error = Encoding::InvalidByteSequenceError.new("invalid UTF-8 byte sequence #{invalid.b.inspect}")
+        @bytes = "".b
+        text.byteslice(0, valid)
+      end
+    end
+  end
+end