csv-utils 0.7.0 → 0.8.0

data/bin/csv-validator

@@ -1,26 +1,26 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'csv'
 begin
   require 'rchardet'
 rescue LoadError
-  $stderr.puts 'gem install rchardet'
+  warn 'gem install rchardet'
   exit 1
 end
 
+# checks copies, so the row's values keep their bytes for the messages and the correction file
 def utf8?(str)
-  str
-    .force_encoding('utf-8')
-    .valid_encoding?
+  str.dup.force_encoding('utf-8').valid_encoding?
 end
 
 def convert_to_utf8(str, current_encoding)
   return nil unless current_encoding
 
-  str.force_encoding(current_encoding)
-  return nil unless str.valid_encoding?
+  converted = str.dup.force_encoding(current_encoding)
+  return nil unless converted.valid_encoding?
 
-  str.encode('utf-8')
+  converted.encode('utf-8')
 end
 
 def detect_encoding(col)
@@ -28,7 +28,7 @@ def detect_encoding(col)
 end
 
 def strip_bom!(col)
-  col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+  col.sub!("\xEF\xBB\xBF".b, '')
 end
 
 id_column_name = ARGV[1]
@@ -42,7 +42,7 @@ begin
 
   id_column_name ||= headers[0]
   unless headers.include?(id_column_name)
-    $stderr.puts("header #{id_column_name} not found in current set of headers")
+    warn("header #{id_column_name} not found in current set of headers")
     exit 1
   end
 
@@ -64,20 +64,18 @@ begin
   while (row = csv.shift)
     csv_lineno += 1
 
-    unless row.size == headers.size
-      $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
-    end
+    warn "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}" unless row.size == headers.size
 
     row.each_with_index do |col, idx|
       next if col.nil? || utf8?(col)
 
-      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: non UTF-8 characters found in #{col.inspect}"
+      warn "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: non UTF-8 characters found in #{col.inspect}"
       encoding = detect_encoding(col)
       if (col_utf8_encoded = convert_to_utf8(col, encoding))
         puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{encoding} \"#{col_utf8_encoded}\""
         out_proc.call [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
       else
-        $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
+        warn "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
       end
     end
   end

data/lib/csv-utils.rb

@@ -8,6 +8,11 @@ module CSVUtils
   autoload :ByteOrderMark, 'csv_utils/byte_order_mark'
   autoload :CharacterEncoding, 'csv_utils/character_encoding'
   autoload :EncodingOptions, 'csv_utils/encoding_options'
+  autoload :Error, 'csv_utils/error'
+  autoload :HeaderNotFoundError, 'csv_utils/header_not_found_error'
+  autoload :MalformedRowError, 'csv_utils/malformed_row_error'
+  autoload :RowReader, 'csv_utils/row_reader'
+  autoload :UnsortedFileError, 'csv_utils/unsorted_file_error'
   autoload :CSVCompare, 'csv_utils/csv_compare'
   autoload :CSVExtender, 'csv_utils/csv_extender'
   autoload :CSVIterator, 'csv_utils/csv_iterator'

data/lib/csv_utils/character_encoding.rb

@@ -1,21 +1,71 @@
 # frozen_string_literal: true
 
 module CSVUtils
-  # Guesses the character encoding of a file without a byte order mark from a sample of its bytes.
+  # Guesses the character encoding of a file without a byte order mark from its bytes.
   module CharacterEncoding
     # Bytes Windows-1252 leaves undefined; converting them to UTF-8 raises.
     # @return [Regexp]
     WINDOWS_1252_UNDEFINED = /[\x81\x8D\x8F\x90\x9D]/n
 
-    # 'UTF-8' when the sample is valid UTF-8. Otherwise 'Windows-1252', the usual encoding of Excel exports,
-    # or 'ISO-8859-1' when the sample has a byte Windows-1252 leaves undefined, since every ISO-8859-1 byte
+    # The last character of a chunk when it's multibyte, whole or cut off: a lead byte and continuation bytes.
+    TRAILING_MULTIBYTE_CHAR = /[\xC0-\xFF][\x80-\xBF]*\z/n
+    private_constant :TRAILING_MULTIBYTE_CHAR
+
+    # 'UTF-8' when the bytes are valid UTF-8. Otherwise 'Windows-1252', the usual encoding of Excel exports,
+    # or 'ISO-8859-1' when they have a byte Windows-1252 leaves undefined, since every ISO-8859-1 byte
     # converts to UTF-8.
-    # @param sample [String] bytes from the start of the file, ending on a whole character
+    # @param sample [String] bytes from the start of the file
+    # @param truncated [Boolean] whether the sample was cut off, so its last character may be incomplete
+    # @return [String]
+    def self.detect(sample, truncated: false)
+      scan([sample], truncated)
+    end
+
+    # Like {.detect}, for a file read in chunks, so it can check a whole file without holding it in memory.
+    # A character split between chunks is checked whole.
+    # @param chunks [Enumerable<String>] the file's bytes in order
+    # @return [String]
+    def self.detect_stream(chunks)
+      scan(chunks, false)
+    end
+
+    # @api private
+    # @param chunks [Enumerable<String>]
+    # @param truncated [Boolean] whether to ignore an incomplete character at the end
+    # @return [String]
+    def self.scan(chunks, truncated)
+      utf8 = true
+      undefined = false
+      carry = ''.b
+      chunks.each do |chunk|
+        bytes = carry + chunk.b
+        whole, carry = split_trailing_char(bytes)
+        utf8 &&= whole.force_encoding(Encoding::UTF_8).valid_encoding?
+        undefined ||= bytes.match?(WINDOWS_1252_UNDEFINED)
+        break unless utf8 || !undefined
+      end
+      utf8 &&= truncated || carry.force_encoding(Encoding::UTF_8).valid_encoding?
+      encoding_name(utf8, undefined)
+    end
+
+    # @api private
+    # @param bytes [String] binary
+    # @return [Array(String, String)] the bytes before their last multibyte character, and that character
+    def self.split_trailing_char(bytes)
+      tail_start = [bytes.bytesize - 4, 0].max
+      idx = bytes.byteslice(tail_start..) =~ TRAILING_MULTIBYTE_CHAR
+      return [bytes.dup, ''.b] unless idx
+
+      [bytes.byteslice(0, tail_start + idx), bytes.byteslice((tail_start + idx)..)]
+    end
+
+    # @api private
+    # @param utf8 [Boolean]
+    # @param undefined [Boolean]
     # @return [String]
-    def self.detect(sample)
-      bytes = sample.b
-      return 'UTF-8' if bytes.dup.force_encoding(Encoding::UTF_8).valid_encoding?
-      return 'ISO-8859-1' if bytes.match?(WINDOWS_1252_UNDEFINED)
+    def self.encoding_name(utf8, undefined)
+      return 'UTF-8' if utf8
+      return 'ISO-8859-1' if undefined
 
       'Windows-1252'
     end

data/lib/csv_utils/csv_compare.rb

@@ -37,15 +37,21 @@ module CSVUtils
 
     # Walks both files and yields each difference.
     # @param secondary_data_file [String] path of the file to bring in line with the primary file
+    # @param check_order [Boolean] raise when a file isn't sorted the way the compare block expects, by comparing
+    #   each record with the one before it in the same file. A file is only checked when the block compares its
+    #   first record equal to itself, so a block written for different headers in each file skips the check.
     # @yieldparam action [Symbol] :create (only in the primary file), :update (in both, with changed
     #   update_comparison_columns) or :delete (only in the secondary file)
-    # @yieldparam record [Hash{String => String}] the primary record for :create and :update,
-    #   the secondary record for :delete
+    # @yieldparam record [CSVIterator::RowWrapper] the primary record for :create and :update, the secondary
+    #   record for :delete; a Hash of header to value that knows the line it starts on
     # @return [void]
-    def compare(secondary_data_file, &)
-      open_csv(primary_data_file) do |src, src_headers|
-        open_csv(secondary_data_file) do |dest, dest_headers|
-          compare_records(src, src_headers, dest, dest_headers, &)
+    # @raise [HeaderNotFoundError] when an update_comparison_column isn't in one of the files
+    # @raise [UnsortedFileError] when check_order finds a record out of order
+    def compare(secondary_data_file, check_order: true, &)
+      @check_order = check_order
+      open_csv(primary_data_file) do |src|
+        open_csv(secondary_data_file) do |dest|
+          compare_records(src, dest, &)
         end
       end
     end
@@ -53,22 +59,22 @@ module CSVUtils
     private
 
     # walks both sorted files at once, advancing whichever side is behind
-    def compare_records(src, src_headers, dest, dest_headers)
-      src_record = next_record(src_headers, src)
-      dest_record = next_record(dest_headers, dest)
+    def compare_records(src, dest)
+      src_record = next_record(src)
+      dest_record = next_record(dest)
 
       while src_record || dest_record
         case record_action(src_record, dest_record)
         when :create
           yield :create, src_record
-          src_record = next_record(src_headers, src)
+          src_record = next_record(src, src_record)
         when :delete
           yield :delete, dest_record
-          dest_record = next_record(dest_headers, dest)
+          dest_record = next_record(dest, dest_record)
         else
           yield :update, src_record if update_row?(src_record, dest_record)
-          src_record = next_record(src_headers, src)
-          dest_record = next_record(dest_headers, dest)
+          src_record = next_record(src, src_record)
+          dest_record = next_record(dest, dest_record)
         end
       end
     end
@@ -87,23 +93,33 @@ module CSVUtils
       end
     end
 
+    # yields the file's path, reader, headers and whether its order is checked, decided at its first record
     def open_csv(file)
       csv = CSV.open(file, 'rb', **EncodingOptions.read(csv_options))
-      yield csv, read_headers(csv)
+      reader = RowReader.new(csv)
+      headers = reader.shift || []
+      check_update_comparison_columns(headers)
+      yield({ file: file, reader: reader, headers: headers, check_order: nil })
     ensure
       csv&.close
     end
 
-    def read_headers(csv)
-      headers = csv.shift || []
-      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
-      headers
+    def check_update_comparison_columns(headers)
+      missing = (update_comparison_columns || []) - headers
+      raise HeaderNotFoundError.new(missing.first, headers) unless missing.empty?
     end
 
-    def next_record(headers, csv)
-      return unless (row = csv.shift)
+    def next_record(source, previous_record = nil)
+      return unless (row = source[:reader].shift)
 
-      headers.zip(row).to_h
+      record = CSVIterator::RowWrapper.create(source[:headers], row, source[:reader].lineno)
+      source[:check_order] = @check_order && compare_proc.call(record, record).zero? if source[:check_order].nil?
+      check_order(source[:file], previous_record, record) if previous_record && source[:check_order]
+      record
+    end
+
+    def check_order(file, previous_record, record)
+      raise UnsortedFileError.new(file, record.lineno) if compare_proc.call(previous_record, record).positive?
     end
 
     def update_row?(src_record, dest_record)

data/lib/csv_utils/csv_iterator.rb

@@ -13,12 +13,10 @@ module CSVUtils
     # @return [Array<String>] the byte order marks stripped from the first header, as binary strings
     BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS.keys.freeze
 
-    # @return [Array<String>, nil] the raw row read before the current one, for debugging malformed rows
-    attr_reader :prev_row
-
     # A row as a hash of header to value that knows its line number in the file.
     class RowWrapper < Hash
-      # @return [Integer] line number of the row in the file, counting the header row as line 1
+      # @return [Integer] line the row starts on in the file, counting the header row as line 1. A quoted value
+      #   with a line break spans several lines, so this can differ from the row's position in the file.
       attr_accessor :lineno
 
       # @param headers [Array<String>]
@@ -52,9 +50,10 @@ module CSVUtils
     # @example
     #   CSVUtils::CSVIterator.auto_detect('export.tsv').each { |row| puts row['name'] }
     # @param path [String]
+    # @param full_scan [Boolean] check the encoding of the whole file, not only its first megabyte
     # @return [CSVIterator]
-    def self.auto_detect(path)
-      options = CSVOptions.new(path)
+    def self.auto_detect(path, full_scan: false)
+      options = CSVOptions.new(path, full_scan: full_scan)
       new(path, options.to_csv_options)
     end
 
@@ -68,10 +67,19 @@ module CSVUtils
       @csv_options = EncodingOptions.read(csv_options)
     end
 
+    # The raw row read before the current one (the header row for the first row), for debugging;
+    # after a full pass it's the last row.
+    # @return [Array<String>, nil]
+    def prev_row
+      @reader&.prev_row
+    end
+
     # Yields each row from the start of the file.
     # @param headers [Array<String>, nil] headers to use; by default the first row is read as the headers
     # @yieldparam row [RowWrapper]
     # @return [Enumerator, nil] an enumerator without a block
+    # @raise [MalformedRowError] for a row CSV can't parse, with the line it starts on and {#prev_row}
+    # @raise [CSV::InvalidEncodingError] for bytes that don't match the encoding, with their line
     def each(headers = nil, &)
       return enum_for(:each, headers) unless block_given?
 
@@ -89,11 +97,11 @@ module CSVUtils
     # @param value [String, nil] header whose value is the hash value
     # @yieldparam row [RowWrapper] used instead of value to compute the hash value
     # @return [Hash]
-    # @raise [RuntimeError] when key or value isn't a header
+    # @raise [HeaderNotFoundError] when key or value isn't a header
     def to_hash(key, value = nil, &)
       file_headers = headers
-      raise("header #{key} not found in #{file_headers}") unless file_headers.include?(key)
-      raise("headers #{value} not found in #{file_headers}") if value && !file_headers.include?(value)
+      raise HeaderNotFoundError.new(key, file_headers) unless file_headers.include?(key)
+      raise HeaderNotFoundError.new(value, file_headers) if value && !file_headers.include?(value)
 
       value_proc =
         if value
@@ -152,33 +160,16 @@ module CSVUtils
     end
 
     def each_row(csv, headers)
-      @prev_row = nil
-      row = shift_first_row(csv)
-      lineno = 1
-      unless headers
-        headers = row || []
-        row = csv.shift
-        lineno += 1
-      end
-
-      while row
-        yield RowWrapper.create(headers, row, lineno)
-        @prev_row = row
-        row = csv.shift
-        lineno += 1
+      @reader = RowReader.new(csv)
+      headers ||= @reader.shift || []
+      while (row = @reader.shift)
+        yield RowWrapper.create(headers, row, @reader.lineno)
       end
     end
 
     # an empty file has no headers, and an empty first header cell is nil
     def read_headers(csv)
-      shift_first_row(csv) || []
-    end
-
-    # the first row without the byte order mark, whether it's read as headers or as data
-    def shift_first_row(csv)
-      row = csv.shift
-      row[0] = ByteOrderMark.strip(row[0]) if row && row[0]
-      row
+      RowReader.new(csv).shift || []
     end
   end
 end

data/lib/csv_utils/csv_options.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module CSVUtils
-  # Detects the separators, byte order mark, encoding and column count of a CSV file from its first line.
+  # Detects the separators, byte order mark, encoding and column count of a CSV file from its header row
+  # and, for the encoding, its first {SAMPLE_SIZE} bytes or, with full_scan, all of them.
   #
   # @example
   #   options = CSVUtils::CSVOptions.new('data.csv')
@@ -10,12 +11,14 @@ module CSVUtils
     # @return [Hash{String => String}] byte order marks, as binary strings, to the encoding they indicate
     BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS
 
-    # @return [Array<String>] column separators to look for, the first one found in the line wins
+    # @return [Array<String>] column separators to look for; the one found most often outside quotes in the
+    #   header row wins, and on a tie the one listed first
     COL_SEPARATORS = [
       "\x02",
       "\t",
       '|',
-      ','
+      ',',
+      ';'
     ].freeze
 
     # @return [Array<String>] row separators to look for, the first one found in the line wins
@@ -37,12 +40,19 @@ module CSVUtils
     # @return [Integer] bytes read to detect the encoding of a file without a byte order mark
     SAMPLE_SIZE = 1024 * 1024
 
+    QUOTED = /"[^"]*"/n
+    # the header row: up to the first line break outside quotes
+    HEADER_ROW = /\A(?:[^"\r\n]|"[^"]*")*(?:\r\n|\n|\r)?/n
+    LINE_BREAK = /\r\n|\n|\r/n
+    private_constant :QUOTED, :HEADER_ROW, :LINE_BREAK
+
     # @return [Integer, nil] number of headers, nil without a column separator
     attr_reader :columns
     # @return [String, nil] the byte order mark the file starts with, as a binary string
     attr_reader :byte_order_mark
-    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32' from the byte order mark; without one, from the bytes of
-    #   the first {SAMPLE_SIZE} bytes: 'UTF-8', 'Windows-1252' or 'ISO-8859-1' (see {CharacterEncoding.detect})
+    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32' from the byte order mark; without one, from the first
+    #   {SAMPLE_SIZE} bytes, or all of them with full_scan: 'UTF-8', 'Windows-1252' or 'ISO-8859-1'
+    #   (see {CharacterEncoding.detect})
     attr_reader :encoding
     # @return [String, nil] one of {COL_SEPARATORS}
     attr_reader :col_separator
@@ -50,16 +60,20 @@ module CSVUtils
     attr_reader :row_separator
 
     # @param io [String, IO] path of the file, or an IO positioned at its first line; up to {SAMPLE_SIZE}
-    #   bytes are read from it
-    def initialize(io)
-      sample = read_sample(io)
-      line = first_line(sample)
-
-      @col_separator = auto_detect_col_sep(line)
-      @row_separator = auto_detect_row_sep(line)
-      @byte_order_mark = get_byte_order_mark(line)
-      @encoding = get_character_encoding(@byte_order_mark, sample)
-      @columns = get_number_of_columns(line) if @col_separator
+    #   bytes are read from it, or all of it with full_scan
+    # @param full_scan [Boolean] check the encoding of the whole file, in {SAMPLE_SIZE} chunks, for files whose
+    #   first bytes that aren't UTF-8 may come after the sample
+    def initialize(io, full_scan: false)
+      open_io(io) do |input|
+        sample = input.read(SAMPLE_SIZE) || ''
+        line = first_line(sample)
+
+        @col_separator = auto_detect_col_sep(line)
+        @row_separator = auto_detect_row_sep(line)
+        @byte_order_mark = get_byte_order_mark(line)
+        @encoding = get_character_encoding(@byte_order_mark, sample, full_scan ? input : nil)
+        @columns = get_number_of_columns(line) if @col_separator
+      end
     end
 
     # Whether both separators were found. An empty file, or a single line without a newline, isn't valid.
@@ -86,10 +100,14 @@ module CSVUtils
     end
 
     # @api private
-    # @param line [String]
-    # @return [String, nil]
+    # @param line [String] the header row
+    # @return [String, nil] the separator found most often outside quotes, nil when there's none
     def auto_detect_col_sep(line)
-      COL_SEPARATORS.detect { |sep| line.include?(sep) }
+      unquoted = line.b.gsub(QUOTED, '')
+      counts = COL_SEPARATORS.map { |sep| unquoted.count(sep) }
+      return if counts.max.zero?
+
+      COL_SEPARATORS[counts.index(counts.max)]
     end
 
     # @api private
@@ -127,9 +145,13 @@ module CSVUtils
     # @api private
     # @param bom [String, nil]
     # @param sample [String] bytes from the start of the file
+    # @param rest [IO, nil] the rest of the file, to scan it all
     # @return [String]
-    def get_character_encoding(bom, sample)
-      BYTE_ORDER_MARKS[bom] || CharacterEncoding.detect(sample)
+    def get_character_encoding(bom, sample, rest = nil)
+      return BYTE_ORDER_MARKS[bom] if bom
+      return CharacterEncoding.detect_stream(chunks(sample, rest)) if rest
+
+      CharacterEncoding.detect(sample, truncated: sample.bytesize == SAMPLE_SIZE)
     end
 
     # @api private
@@ -142,19 +164,26 @@ module CSVUtils
     private
 
     # an empty file reads as an empty sample, which makes the options invalid
-    def read_sample(io)
-      sample = (io.is_a?(String) ? File.open(io, 'rb') { |file| file.read(SAMPLE_SIZE) } : io.read(SAMPLE_SIZE)) || ''
-      sample.bytesize < SAMPLE_SIZE ? sample : drop_last_multibyte_char(sample)
+    def open_io(io, &)
+      io.is_a?(String) ? File.open(io, 'rb', &) : yield(io)
     end
 
-    # a full sample can end mid-character, so its last multibyte character, whole or not, is dropped
-    def drop_last_multibyte_char(sample)
-      sample.sub(/[\xC0-\xFF][\x80-\xBF]*\z/n, '')
+    def chunks(sample, rest)
+      Enumerator.new do |chunks|
+        chunks << sample
+        while (chunk = rest.read(SAMPLE_SIZE))
+          chunks << chunk
+        end
+      end
     end
 
+    # The header row, which a quoted line break doesn't end. A stray quote falls back to the first line.
     def first_line(sample)
-      line_end = sample.index("\n")
-      line_end ? sample.byteslice(0, line_end + 1) : sample
+      line = sample.b[HEADER_ROW]
+      return line if line.bytesize == sample.bytesize || line.end_with?("\n", "\r")
+
+      line_break = sample.b.match(LINE_BREAK)
+      line_break ? sample.b.byteslice(0, line_break.end(0)) : sample.b
     end
 
     def wide_encoding?