csv-utils 0.6.0 → 0.8.0

data/lib/csv_utils/csv_options.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module CSVUtils
-  # Detects the separators, byte order mark, encoding and column count of a CSV file from its first line.
+  # Detects the separators, byte order mark, encoding and column count of a CSV file from its header row
+  # and, for the encoding, its first {SAMPLE_SIZE} bytes or, with full_scan, all of them.
   #
   # @example
   #   options = CSVUtils::CSVOptions.new('data.csv')
@@ -10,12 +11,14 @@ module CSVUtils
     # @return [Hash{String => String}] byte order marks, as binary strings, to the encoding they indicate
     BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS
 
-    # @return [Array<String>] column separators to look for, the first one found in the line wins
+    # @return [Array<String>] column separators to look for; the one found most often outside quotes in the
+    #   header row wins, and on a tie the one listed first
     COL_SEPARATORS = [
       "\x02",
       "\t",
       '|',
-      ','
+      ',',
+      ';'
     ].freeze
 
     # @return [Array<String>] row separators to look for, the first one found in the line wins
@@ -25,26 +28,52 @@ module CSVUtils
       "\r"
     ].freeze
 
+    # @return [Hash{String => String}] UTF-16 and UTF-32 byte order marks, as binary strings, to the :encoding
+    #   option that decodes them to UTF-8
+    WIDE_ENCODINGS = {
+      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-32BE:UTF-8',
+      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-32LE:UTF-8',
+      (+"\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-16BE:UTF-8',
+      (+"\xFF\xFE").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-16LE:UTF-8'
+    }.freeze
+
+    # @return [Integer] bytes read to detect the encoding of a file without a byte order mark
+    SAMPLE_SIZE = 1024 * 1024
+
+    QUOTED = /"[^"]*"/n
+    # the header row: up to the first line break outside quotes
+    HEADER_ROW = /\A(?:[^"\r\n]|"[^"]*")*(?:\r\n|\n|\r)?/n
+    LINE_BREAK = /\r\n|\n|\r/n
+    private_constant :QUOTED, :HEADER_ROW, :LINE_BREAK
+
     # @return [Integer, nil] number of headers, nil without a column separator
     attr_reader :columns
     # @return [String, nil] the byte order mark the file starts with, as a binary string
     attr_reader :byte_order_mark
-    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32', from the byte order mark; 'UTF-8' without one
+    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32' from the byte order mark; without one, from the first
+    #   {SAMPLE_SIZE} bytes, or all of them with full_scan: 'UTF-8', 'Windows-1252' or 'ISO-8859-1'
+    #   (see {CharacterEncoding.detect})
     attr_reader :encoding
     # @return [String, nil] one of {COL_SEPARATORS}
     attr_reader :col_separator
     # @return [String, nil] one of {ROW_SEPARATORS}
     attr_reader :row_separator
 
-    # @param io [String, IO] path of the file, or an IO positioned at its first line
-    def initialize(io)
-      line = read_first_line(io)
-
-      @col_separator = auto_detect_col_sep(line)
-      @row_separator = auto_detect_row_sep(line)
-      @byte_order_mark = get_byte_order_mark(line)
-      @encoding = get_character_encoding(@byte_order_mark)
-      @columns = get_number_of_columns(line) if @col_separator
+    # @param io [String, IO] path of the file, or an IO positioned at its first line; up to {SAMPLE_SIZE}
+    #   bytes are read from it, or all of it with full_scan
+    # @param full_scan [Boolean] check the encoding of the whole file, in {SAMPLE_SIZE} chunks, for files whose
+    #   first bytes that aren't UTF-8 may come after the sample
+    def initialize(io, full_scan: false)
+      open_io(io) do |input|
+        sample = input.read(SAMPLE_SIZE) || ''
+        line = first_line(sample)
+
+        @col_separator = auto_detect_col_sep(line)
+        @row_separator = auto_detect_row_sep(line)
+        @byte_order_mark = get_byte_order_mark(line)
+        @encoding = get_character_encoding(@byte_order_mark, sample, full_scan ? input : nil)
+        @columns = get_number_of_columns(line) if @col_separator
+      end
     end
 
     # Whether both separators were found. An empty file, or a single line without a newline, isn't valid.
@@ -55,11 +84,30 @@ module CSVUtils
       true
     end
 
+    # Options for CSV.open, or any class in this library, with the detected separators and an :encoding that
+    # reads values as UTF-8 strings: 'bom|utf-8', 'Windows-1252:UTF-8', 'BOM|UTF-16LE:UTF-8' and so on.
+    # A separator that wasn't detected is left to CSV, and so is the row separator of UTF-16 and UTF-32 files,
+    # since it's detected from raw bytes.
+    # @example
+    #   CSV.open(path, 'rb', **options.to_csv_options)
+    #   CSVUtils::CSVSort.new(path, 'sorted.csv', true, options.to_csv_options).sort
+    # @return [Hash{Symbol => String}]
+    def to_csv_options
+      options = { encoding: encoding_option }
+      options[:col_sep] = col_separator if col_separator
+      options[:row_sep] = row_separator if row_separator && !wide_encoding?
+      options
+    end
+
     # @api private
-    # @param line [String]
-    # @return [String, nil]
+    # @param line [String] the header row
+    # @return [String, nil] the separator found most often outside quotes, nil when there's none
     def auto_detect_col_sep(line)
-      COL_SEPARATORS.detect { |sep| line.include?(sep) }
+      unquoted = line.b.gsub(QUOTED, '')
+      counts = COL_SEPARATORS.map { |sep| unquoted.count(sep) }
+      return if counts.max.zero?
+
+      COL_SEPARATORS[counts.index(counts.max)]
     end
 
     # @api private
@@ -96,9 +144,14 @@ module CSVUtils
 
     # @api private
     # @param bom [String, nil]
+    # @param sample [String] bytes from the start of the file
+    # @param rest [IO, nil] the rest of the file, to scan it all
     # @return [String]
-    def get_character_encoding(bom)
-      BYTE_ORDER_MARKS[bom] || 'UTF-8'
+    def get_character_encoding(bom, sample, rest = nil)
+      return BYTE_ORDER_MARKS[bom] if bom
+      return CharacterEncoding.detect_stream(chunks(sample, rest)) if rest
+
+      CharacterEncoding.detect(sample, truncated: sample.bytesize == SAMPLE_SIZE)
     end
 
     # @api private
@@ -110,10 +163,35 @@ module CSVUtils
 
     private
 
-    # an empty file reads as an empty line, which makes the options invalid
-    def read_first_line(io)
-      line = io.is_a?(String) ? File.open(io, 'rb', &:gets) : io.gets
-      line || ''
+    # an empty file reads as an empty sample, which makes the options invalid
+    def open_io(io, &)
+      io.is_a?(String) ? File.open(io, 'rb', &) : yield(io)
+    end
+
+    def chunks(sample, rest)
+      Enumerator.new do |chunks|
+        chunks << sample
+        while (chunk = rest.read(SAMPLE_SIZE))
+          chunks << chunk
+        end
+      end
+    end
+
+    # The header row, which a quoted line break doesn't end. A stray quote falls back to the first line.
+    def first_line(sample)
+      line = sample.b[HEADER_ROW]
+      return line if line.bytesize == sample.bytesize || line.end_with?("\n", "\r")
+
+      line_break = sample.b.match(LINE_BREAK)
+      line_break ? sample.b.byteslice(0, line_break.end(0)) : sample.b
+    end
+
+    def wide_encoding?
+      WIDE_ENCODINGS.key?(byte_order_mark)
+    end
+
+    def encoding_option
+      WIDE_ENCODINGS.fetch(byte_order_mark) { encoding == 'UTF-8' ? EncodingOptions::DEFAULT_ENCODING : "#{encoding}:UTF-8" }
     end
 
     def header_parse_options

data/lib/csv_utils/csv_report.rb

@@ -15,13 +15,14 @@ module CSVUtils
 
     # @param csv [String, CSV, #<<] path to write, opened in 'wb' mode by default, or a CSV to append to
     # @param headers [Array<String>, Class, nil] header row, or a {CSVRow} class to take the headers from
-    # @param csv_options [Hash] options passed to CSV.open for a path; :mode overrides the file mode
+    # @param csv_options [Hash] options passed to CSV.open for a path; :mode overrides the file mode and
+    #   :encoding, UTF-8 by default, is the encoding of the file written
     # @yieldparam report [CSVReport] when given, the block is passed to {#generate}
     def initialize(csv, headers = nil, csv_options = {}, &block)
       @csv =
         if csv.is_a?(String)
           @must_close = true
-          opts = csv_options.dup
+          opts = EncodingOptions.write(csv_options)
           mode = opts.delete(:mode) || 'wb'
           CSV.open(csv, mode, **opts)
         else

data/lib/csv_utils/csv_sort.rb

@@ -4,11 +4,20 @@ require 'fileutils'
 
 module CSVUtils
   # Sorts a CSV file too large to load into memory with an external merge sort: batches of rows are
-  # sorted into temporary part files next to the new file, which are then merged in pairs.
+  # sorted into temporary files, which are then merged up to {MERGE_WIDTH} at a time, so most files
+  # are sorted with a single merge pass.
   #
   # @example
   #   CSVUtils::CSVSort.new('input.csv', 'sorted.csv').sort { |a, b| a[0].to_i <=> b[0].to_i }
+  #   CSVUtils::CSVSort.new('input.csv', 'sorted.csv').sort_by { |row| row[0].to_i }
   class CSVSort
+    # @return [Integer] temporary files merged at once, which bounds the files open at the same time
+    MERGE_WIDTH = 64
+
+    ROW_KEY = proc { |row| row }
+    KEY_COMPARE = proc { |key1, key2| key1 <=> key2 }
+    private_constant :ROW_KEY, :KEY_COMPARE
+
     # @return [String] path of the file to sort
     attr_reader :csv_file
     # @return [String] path of the sorted file to write
@@ -23,134 +32,138 @@ module CSVUtils
     # @param csv_file [String] path of the file to sort
     # @param new_csv_file [String] path of the sorted file to write
     # @param has_headers [Boolean] whether the first row is a header row
-    # @param csv_options [Hash] options passed to CSV.open
+    # @param csv_options [Hash] options passed to CSV.open; the file is read with its :encoding, 'bom|utf-8' by
+    #   default, and the sorted file written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(csv_file, new_csv_file, has_headers = true, csv_options = {})
       @csv_file = csv_file
       @new_csv_file = new_csv_file
       @has_headers = has_headers
       @csv_options = csv_options
-      @csv_part_files = []
+      @temp_files = []
     end
 
-    # Writes the sorted file. Temporary files are removed even when sorting fails.
+    # Writes the file sorted by comparing rows. Temporary files are removed even when sorting fails.
     # @param batch_size [Integer] rows held in memory and sorted at a time
+    # @param tmp_dir [String, nil] directory for the temporary files; the sorted file's directory by default
     # @yieldparam row1 [Array<String>]
     # @yieldparam row2 [Array<String>]
     # @yieldreturn [Integer] negative, 0 or positive like <=>; without a block rows are compared as arrays
     # @return [void]
-    def sort(batch_size = 100_000, &block)
-      block ||= proc { |row1, row2| row1 <=> row2 }
-      @csv_part_files = []
-      create_sorted_csv_part_files(batch_size, &block)
-      merge_csv_part_files(&block)
-    ensure
-      delete_csv_part_files
+    def sort(batch_size = 100_000, tmp_dir: nil, &block)
+      compare = block || KEY_COMPARE
+      run_sort(batch_size, tmp_dir, ROW_KEY, compare) { |batch| batch.sort!(&compare) }
+    end
+
+    # Writes the file sorted by a key computed once per row, which is faster than {#sort} when
+    # the comparison would otherwise convert values (ex: to_i) on every comparison.
+    # @param batch_size [Integer] rows held in memory and sorted at a time
+    # @param tmp_dir [String, nil] directory for the temporary files; the sorted file's directory by default
+    # @yieldparam row [Array<String>]
+    # @yieldreturn [Comparable] the sort key; keys are compared with <=>, so use arrays for several columns
+    # @return [void]
+    def sort_by(batch_size = 100_000, tmp_dir: nil, &key)
+      run_sort(batch_size, tmp_dir, key, KEY_COMPARE) { |batch| batch.sort_by!(&key) }
     end
 
     private
 
-    # rubocop:disable-next Metrics/MethodLength
-    def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
-      src1 = CSV.open(src_csv_file1, 'rb', **csv_options)
-      begin
-        src2 = CSV.open(src_csv_file2, 'rb', **csv_options)
-        begin
-          dest = CSV.open(dest_csv_file, 'wb', **csv_options)
-          begin
-            if @headers
-              dest << @headers
-              src1.shift
-              src2.shift
-            end
-
-            row1 = src1.shift
-            row2 = src2.shift
-
-            append_row1_proc = proc do
-              dest << row1
-              row1 = src1.shift
-            end
-
-            append_row2_proc = proc do
-              dest << row2
-              row2 = src2.shift
-            end
-
-            while row1 || row2
-              if row1.nil?
-                append_row2_proc.call
-              elsif row2.nil?
-                append_row1_proc.call
-              elsif yield(row1, row2) <= 0
-                append_row1_proc.call
-              else
-                append_row2_proc.call
-              end
-            end
-          ensure
-            dest.close
-          end
-        ensure
-          src2.close
+    def run_sort(batch_size, tmp_dir, key, compare, &)
+      @tmp_dir = tmp_dir || File.dirname(new_csv_file)
+      @temp_files = []
+      part_files = create_sorted_part_files(batch_size, &)
+      part_files = merge_pass(part_files, key, compare) while part_files.size > 1
+      write_sorted_file(part_files.first)
+    ensure
+      delete_temp_files
+    end
+
+    def create_sorted_part_files(batch_size, &sort_batch)
+      CSV.open(csv_file, 'rb', **EncodingOptions.read(csv_options)) do |src|
+        @headers = (src.shift if has_headers)
+        src.each_slice(batch_size).map do |batch|
+          sorted = sort_batch.call(batch)
+          write_temp_file { |csv| sorted.each { |row| csv << row } }
         end
-      ensure
-        src1.close
       end
     end
 
-    def create_sorted_csv_part_files(batch_size, &block)
-      src = CSV.open(csv_file, 'rb', **csv_options)
-      begin
-        @headers = src.shift if has_headers
-
-        batch = []
-        create_batch_part_proc = proc do
-          batch.sort!(&block)
-          @csv_part_files << "#{new_csv_file}.part.#{@csv_part_files.size}"
-          CSV.open(@csv_part_files.last, 'wb', **csv_options) do |csv|
-            csv << @headers if @headers
-            batch.each { |row| csv << row }
-          end
-          batch = []
-        end
+    # merges consecutive groups of files, so each pass divides the number of files by MERGE_WIDTH
+    def merge_pass(files, key, compare)
+      files.each_slice(MERGE_WIDTH).map do |group|
+        next group.first if group.size == 1
+
+        merged = write_temp_file { |dest| merge_files(group, dest, key, compare) }
+        group.each { |file| File.unlink(file) }
+        merged
+      end
+    end
 
-        while (row = src.shift)
-          batch << row
-          create_batch_part_proc.call if batch.size >= batch_size
+    # keeps the next row of every file in a queue sorted by key, so each row written is the smallest left
+    def merge_files(files, dest, key, compare)
+      open_temp_files(files) do |sources|
+        queue = []
+        sources.each_with_index do |src, idx|
+          src.shift if @headers
+          enqueue(queue, src.shift, idx, key, compare)
         end
 
-        create_batch_part_proc.call if batch.size.positive?
-      ensure
-        src.close
+        until queue.empty?
+          _, row, idx = queue.shift
+          dest << row
+          enqueue(queue, sources[idx].shift, idx, key, compare)
+        end
       end
     end
 
-    def merge_csv_part_files(&)
-      file_merge_cnt = 0
+    # rows with equal keys keep the order of the files they came from
+    def enqueue(queue, row, idx, key, compare)
+      return unless row
 
-      while @csv_part_files.size > 1
-        file_merge_cnt += 1
-
-        # inputs stay in the list until merged, so a failed merge still cleans them up
-        csv_part_file1, csv_part_file2 = @csv_part_files.first(2)
-        @csv_part_files << "#{new_csv_file}.merge.#{file_merge_cnt}"
+      value = key.call(row)
+      pos = queue.bsearch_index do |other_value, _, other_idx|
+        result = compare.call(value, other_value)
+        result.negative? || (result.zero? && idx < other_idx)
+      end
+      queue.insert(pos || queue.size, [value, row, idx])
+    end
 
-        merge_sort_csv_files(csv_part_file1, csv_part_file2, @csv_part_files.last, &)
+    def open_temp_files(files)
+      sources = []
+      # not map: sources has to hold the files opened before one fails to open, so ensure closes them
+      # rubocop:disable-next Style/MapIntoArray
+      files.each { |file| sources << CSV.open(file, 'rb', **write_options) }
+      yield sources
+    ensure
+      sources.each(&:close)
+    end
 
-        @csv_part_files.shift(2).each { |file| File.unlink(file) }
+    # temporary files are written with the write options, so reading them back with those changes nothing
+    def write_temp_file
+      @temp_files << File.join(@tmp_dir, "#{File.basename(new_csv_file)}.#{@temp_files.size}.tmp")
+      CSV.open(@temp_files.last, 'wb', **write_options) do |csv|
+        csv << @headers if @headers
+        yield csv
       end
+      @temp_files.last
+    end
 
-      if @csv_part_files.size.positive?
-        FileUtils.mv(@csv_part_files.pop, new_csv_file)
-      else
-        FileUtils.cp(@csv_file, new_csv_file)
+    # a file without rows is written rather than copied, so it's in the same encoding as a sorted one
+    def write_sorted_file(sorted_file)
+      return FileUtils.mv(sorted_file, new_csv_file) if sorted_file
+
+      CSV.open(new_csv_file, 'wb', **write_options) do |csv|
+        csv << @headers if @headers
       end
     end
 
+    def write_options
+      EncodingOptions.write(csv_options)
+    end
+
     # removes the temporary files left behind when sorting fails part way through
-    def delete_csv_part_files
-      @csv_part_files.each { |file| FileUtils.rm_f(file) }
-      @csv_part_files = []
+    def delete_temp_files
+      @temp_files.each { |file| FileUtils.rm_f(file) }
+      @temp_files = []
     end
   end
 end

data/lib/csv_utils/csv_transformer.rb

@@ -19,10 +19,11 @@ module CSVUtils
 
     # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the transformer
     # @param dest_csv [String, CSV, #<<] path or CSV to write
-    # @param csv_options [Hash] options passed to CSV.open for paths
+    # @param csv_options [Hash] options passed to CSV.open for paths; the source is read with its :encoding,
+    #   'bom|utf-8' by default, and the output written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(src_csv, dest_csv, csv_options = {})
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
-      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', EncodingOptions.read(csv_options))
+      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', EncodingOptions.write(csv_options))
     end
 
     # Reads the first row as the headers. Without it, every row is data and the output has no header row.

data/lib/csv_utils/csv_wrapper.rb

@@ -54,6 +54,16 @@ module CSVUtils
       csv.shift
     end
 
+    # @return [String, nil] the raw text of the last row read, with its row separator
+    def line
+      csv.line
+    end
+
+    # @return [String] the row separator, detected from the file when it was :auto
+    def row_sep
+      csv.row_sep
+    end
+
     # Moves back to the first row.
     # @return [void]
     def rewind

data/lib/csv_utils/encoding_options.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # The one place the library decides how files are decoded and encoded. Every class reads with the
+  # :encoding CSV option, 'bom|utf-8' by default, so values are UTF-8 strings whatever the locale, and
+  # writes in the encoding the values were decoded to, so reading 'Windows-1252:UTF-8' writes UTF-8.
+  # File modes never carry an encoding: CSV raises when the mode and the :encoding option both have one.
+  #
+  # @example
+  #   CSV.open(path, 'rb', **CSVUtils::EncodingOptions.read(col_sep: "\t"))
+  module EncodingOptions
+    # @return [String] the :encoding option used for reading when none is given
+    DEFAULT_ENCODING = 'bom|utf-8'
+
+    # Options for reading: the given options with the default :encoding when they have none.
+    # @param csv_options [Hash]
+    # @return [Hash]
+    def self.read(csv_options)
+      { encoding: DEFAULT_ENCODING }.merge(csv_options)
+    end
+
+    # Options for writing values read with csv_options: the :encoding is the one values were decoded to.
+    # @param csv_options [Hash]
+    # @return [Hash]
+    def self.write(csv_options)
+      csv_options.merge(encoding: decoded_encoding(read(csv_options)[:encoding]))
+    end
+
+    # The encoding of strings read with an :encoding option: the internal encoding of 'external:internal',
+    # otherwise the external one, without a 'bom|' prefix.
+    # @example
+    #   CSVUtils::EncodingOptions.decoded_encoding('Windows-1252:UTF-8') # => 'UTF-8'
+    #   CSVUtils::EncodingOptions.decoded_encoding('bom|utf-8') # => 'utf-8'
+    # @param encoding [String, Encoding]
+    # @return [String]
+    def self.decoded_encoding(encoding)
+      encoding.to_s.split(':').last.sub(/\Abom\|/i, '')
+    end
+  end
+end

data/lib/csv_utils/error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Included in every exception this library raises, so `rescue CSVUtils::Error` catches all of them.
+  # Each exception also subclasses the error Ruby or CSV would raise, so existing rescues keep working.
+  module Error
+  end
+end

data/lib/csv_utils/header_not_found_error.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Raised when a header the caller named isn't in the file. A RuntimeError, as before this class existed.
+  class HeaderNotFoundError < RuntimeError
+    include Error
+
+    # @return [String] the header that wasn't found
+    attr_reader :header
+    # @return [Array<String>] the headers of the file
+    attr_reader :headers
+
+    # @param header [String]
+    # @param headers [Array<String>]
+    def initialize(header, headers)
+      @header = header
+      @headers = headers
+      super("header #{header} not found in #{headers}")
+    end
+  end
+end

data/lib/csv_utils/malformed_row_error.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # A CSV::MalformedCSVError with the physical line the bad row starts on, rather than CSV's row count,
+  # and the row read before it.
+  class MalformedRowError < CSV::MalformedCSVError
+    include Error
+
+    # @return [Array<String>, nil] the last row read successfully
+    attr_reader :prev_row
+
+    # @param message [String] CSV's message without its "in line N." suffix
+    # @param line_number [Integer] physical line the bad row starts on
+    # @param prev_row [Array<String>, nil]
+    def initialize(message, line_number, prev_row)
+      @prev_row = prev_row
+      super(message, line_number)
+    end
+  end
+end

data/lib/csv_utils/row_reader.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Reads rows from a CSV positioned at its first line, keeping the physical line each row starts on.
+  # CSV counts rows, so a quoted value with a line break would make its line numbers fall behind.
+  # The byte order mark is stripped from the first row, and rows CSV can't parse raise {MalformedRowError}.
+  #
+  # @example
+  #   reader = CSVUtils::RowReader.new(csv)
+  #   headers = reader.shift
+  #   while (row = reader.shift)
+  #     puts "#{reader.lineno}: #{row.inspect}"
+  #   end
+  class RowReader
+    # @return [Integer, nil] line the last row read starts on, counting the first line as 1
+    attr_reader :lineno
+    # @return [Array<String>, nil] the row read before the last one
+    attr_reader :prev_row
+
+    # @param csv [CSV, CSVWrapper] positioned at the first line
+    def initialize(csv)
+      @csv = csv
+      @last_line = 0
+      @row = nil
+      @prev_row = nil
+    end
+
+    # @return [Array<String>, nil] the next row, nil at the end
+    # @raise [MalformedRowError] for a row CSV can't parse, with the line it starts on
+    # @raise [CSV::InvalidEncodingError] for bytes that don't match the encoding; CSV raises it when it buffers
+    #   them, ahead of the rows before them, with the right line already
+    def shift
+      row = @csv.shift
+      @lineno = @last_line + 1
+      count_lines(row) if row
+      @prev_row = @row
+      @row = row
+    rescue CSV::InvalidEncodingError
+      raise
+    rescue CSV::MalformedCSVError => e
+      raise MalformedRowError.new(e.message.delete_suffix(" in line #{e.line_number}."), @last_line + 1, @row)
+    end
+
+    private
+
+    def count_lines(row)
+      row[0] = ByteOrderMark.strip(row[0]) if @last_line.zero? && row[0]
+      @last_line += @csv.line.count(@csv.row_sep[-1])
+    end
+  end
+end

data/lib/csv_utils/unsorted_file_error.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Raised by {CSVCompare} when a file isn't sorted the way its compare block expects.
+  class UnsortedFileError < RuntimeError
+    include Error
+
+    # @return [String] path of the file
+    attr_reader :file
+    # @return [Integer] physical line of the first record out of order
+    attr_reader :lineno
+
+    # @param file [String]
+    # @param lineno [Integer]
+    def initialize(file, lineno)
+      @file = file
+      @lineno = lineno
+      super("#{file} isn't sorted: the record on line #{lineno} sorts before the one above it")
+    end
+  end
+end

data/lib/csv_utils/version.rb

@@ -2,5 +2,5 @@
 
 module CSVUtils
   # Gem version, bumped by release-please
-  VERSION = '0.6.0'
+  VERSION = '0.8.0'
 end