csv-utils 0.6.0 → 0.8.0

data/bin/csv-splitter

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'optparse'
 
@@ -37,47 +38,34 @@ def get_split_file_name(file, num_files, total_files)
   end
 end
 
-csv_options = CSVUtils::CSVOptions.new(file)
+csv_options = CSVUtils::CSVOptions.new(file).to_csv_options.merge(liberal_parsing: true)
 # a single column file has no separator to detect
-col_sep = csv_options.col_separator || ','
+csv_options[:col_sep] ||= ','
 
-csv = CSV.open(
-  file,
-  'rb',
-  col_sep: col_sep,
-  liberal_parsing: true
-)
-
-headers = options[:no_header] ? nil : csv.shift
-
-total_rows = 0
-total_rows += 1 while csv.shift
+# one pass: parts are written under temporary names, then renamed once the number of parts is known
+part_files = []
+out = nil
+num_rows = 0
 
-csv.rewind
-csv.shift if headers
+CSV.open(file, 'rb', **CSVUtils::EncodingOptions.read(csv_options)) do |csv|
+  headers = options[:no_header] ? nil : csv.shift
 
-total_files = total_rows / options[:rows]
-total_files += 1 if (total_rows % options[:rows]) > 0
+  while (row = csv.shift)
+    if out.nil? || num_rows >= options[:rows]
+      out&.close
+      num_rows = 0
+      part_files << "#{file}.part-#{part_files.size + 1}.tmp"
+      out = CSV.open(part_files.last, 'wb', **CSVUtils::EncodingOptions.write(csv_options))
+      out << headers if headers
+    end
 
-num_rows = 0
-num_files = 0
-out = nil
-
-append_row_proc = proc do |row|
-  if out.nil? || num_rows >= options[:rows]
-    out.close if out
-    num_rows = 0
-    num_files += 1
-    out = CSV.open(get_split_file_name(file, num_files, total_files), 'wb', col_sep: col_sep)
-    out << headers if headers
+    out << row
+    num_rows += 1
   end
-
-  out << row
-  num_rows += 1
+ensure
+  out&.close
 end
 
-while (row = csv.shift)
-  append_row_proc.call(row)
+part_files.each_with_index do |part_file, idx|
+  File.rename(part_file, get_split_file_name(file, idx + 1, part_files.size))
 end
-
-out&.close

data/bin/csv-validator

@@ -1,26 +1,26 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'csv'
 begin
   require 'rchardet'
 rescue LoadError
-  $stderr.puts 'gem install rchardet'
+  warn 'gem install rchardet'
   exit 1
 end
 
+# checks copies, so the row's values keep their bytes for the messages and the correction file
 def utf8?(str)
-  str
-    .force_encoding('utf-8')
-    .valid_encoding?
+  str.dup.force_encoding('utf-8').valid_encoding?
 end
 
 def convert_to_utf8(str, current_encoding)
   return nil unless current_encoding
 
-  str.force_encoding(current_encoding)
-  return nil unless str.valid_encoding?
+  converted = str.dup.force_encoding(current_encoding)
+  return nil unless converted.valid_encoding?
 
-  str.encode('utf-8')
+  converted.encode('utf-8')
 end
 
 def detect_encoding(col)
@@ -28,7 +28,7 @@ def detect_encoding(col)
 end
 
 def strip_bom!(col)
-  col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+  col.sub!("\xEF\xBB\xBF".b, '')
 end
 
 id_column_name = ARGV[1]
@@ -42,7 +42,7 @@ begin
 
   id_column_name ||= headers[0]
   unless headers.include?(id_column_name)
-    $stderr.puts("header #{id_column_name} not found in current set of headers")
+    warn("header #{id_column_name} not found in current set of headers")
     exit 1
   end
 
@@ -64,20 +64,18 @@ begin
   while (row = csv.shift)
     csv_lineno += 1
 
-    unless row.size == headers.size
-      $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
-    end
+    warn "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}" unless row.size == headers.size
 
     row.each_with_index do |col, idx|
       next if col.nil? || utf8?(col)
 
-      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: non UTF-8 characters found in #{col.inspect}"
+      warn "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: non UTF-8 characters found in #{col.inspect}"
       encoding = detect_encoding(col)
       if (col_utf8_encoded = convert_to_utf8(col, encoding))
         puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{encoding} \"#{col_utf8_encoded}\""
         out_proc.call [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
       else
-        $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
+        warn "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
       end
     end
   end

data/lib/csv-utils.rb

@@ -6,6 +6,13 @@ require 'csv'
 # into memory. Each class is autoloaded on first use.
 module CSVUtils
   autoload :ByteOrderMark, 'csv_utils/byte_order_mark'
+  autoload :CharacterEncoding, 'csv_utils/character_encoding'
+  autoload :EncodingOptions, 'csv_utils/encoding_options'
+  autoload :Error, 'csv_utils/error'
+  autoload :HeaderNotFoundError, 'csv_utils/header_not_found_error'
+  autoload :MalformedRowError, 'csv_utils/malformed_row_error'
+  autoload :RowReader, 'csv_utils/row_reader'
+  autoload :UnsortedFileError, 'csv_utils/unsorted_file_error'
   autoload :CSVCompare, 'csv_utils/csv_compare'
   autoload :CSVExtender, 'csv_utils/csv_extender'
   autoload :CSVIterator, 'csv_utils/csv_iterator'

data/lib/csv_utils/character_encoding.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Guesses the character encoding of a file without a byte order mark from its bytes.
+  module CharacterEncoding
+    # Bytes Windows-1252 leaves undefined; converting them to UTF-8 raises.
+    # @return [Regexp]
+    WINDOWS_1252_UNDEFINED = /[\x81\x8D\x8F\x90\x9D]/n
+
+    # The last character of a chunk when it's multibyte, whole or cut off: a lead byte and continuation bytes.
+    TRAILING_MULTIBYTE_CHAR = /[\xC0-\xFF][\x80-\xBF]*\z/n
+    private_constant :TRAILING_MULTIBYTE_CHAR
+
+    # 'UTF-8' when the bytes are valid UTF-8. Otherwise 'Windows-1252', the usual encoding of Excel exports,
+    # or 'ISO-8859-1' when they have a byte Windows-1252 leaves undefined, since every ISO-8859-1 byte
+    # converts to UTF-8.
+    # @param sample [String] bytes from the start of the file
+    # @param truncated [Boolean] whether the sample was cut off, so its last character may be incomplete
+    # @return [String]
+    def self.detect(sample, truncated: false)
+      scan([sample], truncated)
+    end
+
+    # Like {.detect}, for a file read in chunks, so it can check a whole file without holding it in memory.
+    # A character split between chunks is checked whole.
+    # @param chunks [Enumerable<String>] the file's bytes in order
+    # @return [String]
+    def self.detect_stream(chunks)
+      scan(chunks, false)
+    end
+
+    # @api private
+    # @param chunks [Enumerable<String>]
+    # @param truncated [Boolean] whether to ignore an incomplete character at the end
+    # @return [String]
+    def self.scan(chunks, truncated)
+      utf8 = true
+      undefined = false
+      carry = ''.b
+      chunks.each do |chunk|
+        bytes = carry + chunk.b
+        whole, carry = split_trailing_char(bytes)
+        utf8 &&= whole.force_encoding(Encoding::UTF_8).valid_encoding?
+        undefined ||= bytes.match?(WINDOWS_1252_UNDEFINED)
+        break unless utf8 || !undefined
+      end
+      utf8 &&= truncated || carry.force_encoding(Encoding::UTF_8).valid_encoding?
+      encoding_name(utf8, undefined)
+    end
+
+    # @api private
+    # @param bytes [String] binary
+    # @return [Array(String, String)] the bytes before their last multibyte character, and that character
+    def self.split_trailing_char(bytes)
+      tail_start = [bytes.bytesize - 4, 0].max
+      idx = bytes.byteslice(tail_start..) =~ TRAILING_MULTIBYTE_CHAR
+      return [bytes.dup, ''.b] unless idx
+
+      [bytes.byteslice(0, tail_start + idx), bytes.byteslice((tail_start + idx)..)]
+    end
+
+    # @api private
+    # @param utf8 [Boolean]
+    # @param undefined [Boolean]
+    # @return [String]
+    def self.encoding_name(utf8, undefined)
+      return 'UTF-8' if utf8
+      return 'ISO-8859-1' if undefined
+
+      'Windows-1252'
+    end
+  end
+end

data/lib/csv_utils/csv_compare.rb

@@ -24,7 +24,7 @@ module CSVUtils
     # @param update_comparison_columns [Array<String>, nil] columns compared on matching records;
     #   without them no updates are yielded
     # @param csv_options [Hash] options passed to CSV.open for both files, ex: col_sep: "\t",
-    #   or encoding: 'BINARY' to read bytes that aren't valid UTF-8
+    #   or encoding: 'BINARY' to read bytes that aren't valid UTF-8; the :encoding defaults to 'bom|utf-8'
     # @yieldparam src [Hash{String => String}] record from the primary file
     # @yieldparam dest [Hash{String => String}] record from the secondary file
     # @yieldreturn [Integer] negative, 0 or positive like <=>, using the same order both files are sorted by
@@ -37,15 +37,21 @@ module CSVUtils
 
     # Walks both files and yields each difference.
     # @param secondary_data_file [String] path of the file to bring in line with the primary file
+    # @param check_order [Boolean] raise when a file isn't sorted the way the compare block expects, by comparing
+    #   each record with the one before it in the same file. A file is only checked when the block compares its
+    #   first record equal to itself, so a block written for different headers in each file skips the check.
     # @yieldparam action [Symbol] :create (only in the primary file), :update (in both, with changed
     #   update_comparison_columns) or :delete (only in the secondary file)
-    # @yieldparam record [Hash{String => String}] the primary record for :create and :update,
-    #   the secondary record for :delete
+    # @yieldparam record [CSVIterator::RowWrapper] the primary record for :create and :update, the secondary
+    #   record for :delete; a Hash of header to value that knows the line it starts on
     # @return [void]
-    def compare(secondary_data_file, &)
-      open_csv(primary_data_file) do |src, src_headers|
-        open_csv(secondary_data_file) do |dest, dest_headers|
-          compare_records(src, src_headers, dest, dest_headers, &)
+    # @raise [HeaderNotFoundError] when an update_comparison_column isn't in one of the files
+    # @raise [UnsortedFileError] when check_order finds a record out of order
+    def compare(secondary_data_file, check_order: true, &)
+      @check_order = check_order
+      open_csv(primary_data_file) do |src|
+        open_csv(secondary_data_file) do |dest|
+          compare_records(src, dest, &)
         end
       end
     end
@@ -53,22 +59,22 @@ module CSVUtils
     private
 
     # walks both sorted files at once, advancing whichever side is behind
-    def compare_records(src, src_headers, dest, dest_headers)
-      src_record = next_record(src_headers, src)
-      dest_record = next_record(dest_headers, dest)
+    def compare_records(src, dest)
+      src_record = next_record(src)
+      dest_record = next_record(dest)
 
       while src_record || dest_record
         case record_action(src_record, dest_record)
         when :create
           yield :create, src_record
-          src_record = next_record(src_headers, src)
+          src_record = next_record(src, src_record)
         when :delete
           yield :delete, dest_record
-          dest_record = next_record(dest_headers, dest)
+          dest_record = next_record(dest, dest_record)
         else
           yield :update, src_record if update_row?(src_record, dest_record)
-          src_record = next_record(src_headers, src)
-          dest_record = next_record(dest_headers, dest)
+          src_record = next_record(src, src_record)
+          dest_record = next_record(dest, dest_record)
         end
       end
     end
@@ -87,23 +93,33 @@ module CSVUtils
       end
     end
 
+    # yields the file's path, reader, headers and whether its order is checked, decided at its first record
     def open_csv(file)
-      csv = CSV.open(file, 'rb', **csv_options)
-      yield csv, read_headers(csv)
+      csv = CSV.open(file, 'rb', **EncodingOptions.read(csv_options))
+      reader = RowReader.new(csv)
+      headers = reader.shift || []
+      check_update_comparison_columns(headers)
+      yield({ file: file, reader: reader, headers: headers, check_order: nil })
     ensure
       csv&.close
     end
 
-    def read_headers(csv)
-      headers = csv.shift || []
-      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
-      headers
+    def check_update_comparison_columns(headers)
+      missing = (update_comparison_columns || []) - headers
+      raise HeaderNotFoundError.new(missing.first, headers) unless missing.empty?
     end
 
-    def next_record(headers, csv)
-      return unless (row = csv.shift)
+    def next_record(source, previous_record = nil)
+      return unless (row = source[:reader].shift)
 
-      headers.zip(row).to_h
+      record = CSVIterator::RowWrapper.create(source[:headers], row, source[:reader].lineno)
+      source[:check_order] = @check_order && compare_proc.call(record, record).zero? if source[:check_order].nil?
+      check_order(source[:file], previous_record, record) if previous_record && source[:check_order]
+      record
+    end
+
+    def check_order(file, previous_record, record)
+      raise UnsortedFileError.new(file, record.lineno) if compare_proc.call(previous_record, record).positive?
     end
 
     def update_row?(src_record, dest_record)

data/lib/csv_utils/csv_extender.rb

@@ -9,10 +9,11 @@ module CSVUtils
   class CSVExtender
     # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the extender
     # @param dest_csv [String, CSV, #<<] path or CSV to write
-    # @param csv_options [Hash] options passed to CSV.open for paths
+    # @param csv_options [Hash] options passed to CSV.open for paths; the source is read with its :encoding,
+    #   'bom|utf-8' by default, and the output written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(src_csv, dest_csv, csv_options = {})
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
-      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', EncodingOptions.read(csv_options))
+      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', EncodingOptions.write(csv_options))
     end
 
     # Appends columns one row at a time, then closes the files it opened.

data/lib/csv_utils/csv_iterator.rb

@@ -13,12 +13,10 @@ module CSVUtils
     # @return [Array<String>] the byte order marks stripped from the first header, as binary strings
     BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS.keys.freeze
 
-    # @return [Array<String>, nil] the raw row read before the current one, for debugging malformed rows
-    attr_reader :prev_row
-
     # A row as a hash of header to value that knows its line number in the file.
     class RowWrapper < Hash
-      # @return [Integer] line number of the row in the file, counting the header row as line 1
+      # @return [Integer] line the row starts on in the file, counting the header row as line 1. A quoted value
+      #   with a line break spans several lines, so this can differ from the row's position in the file.
       attr_accessor :lineno
 
       # @param headers [Array<String>]
@@ -26,8 +24,15 @@ module CSVUtils
       # @param lineno [Integer]
       # @return [RowWrapper]
       def self.create(headers, row, lineno)
-        row_wrapper = RowWrapper[headers.zip(row)]
+        row_wrapper = new
         row_wrapper.lineno = lineno
+        # an index loop instead of headers.zip(row), which allocates an array per column for every row
+        idx = 0
+        size = headers.size
+        while idx < size
+          row_wrapper[headers[idx]] = row[idx]
+          idx += 1
+        end
         row_wrapper
       end
 
@@ -41,41 +46,50 @@ module CSVUtils
       end
     end
 
-    # @param src_csv [String, CSV] path or CSV to read; it must support rewind
-    # @param csv_options [Hash] options passed to CSV.open for a path
-    # @param mode [String] file mode for a path; 'rb:BINARY' reads bytes as is
-    def initialize(src_csv, csv_options = {}, mode = 'rb')
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, mode, csv_options)
+    # An iterator for a path, with the separators and encoding detected by {CSVOptions}.
+    # @example
+    #   CSVUtils::CSVIterator.auto_detect('export.tsv').each { |row| puts row['name'] }
+    # @param path [String]
+    # @param full_scan [Boolean] check the encoding of the whole file, not only its first megabyte
+    # @return [CSVIterator]
+    def self.auto_detect(path, full_scan: false)
+      options = CSVOptions.new(path, full_scan: full_scan)
+      new(path, options.to_csv_options)
+    end
+
+    # A path is opened by each method call and closed when the call returns,
+    # so an idle iterator holds no file handle.
+    # @param src_csv [String, CSV] path or CSV to read; a CSV must support rewind and is never closed
+    # @param csv_options [Hash] options passed to CSV.open for a path. The :encoding defaults to 'bom|utf-8'
+    #   (see {EncodingOptions}); 'BINARY' reads bytes as is and 'Windows-1252:UTF-8' converts.
+    def initialize(src_csv, csv_options = {})
+      @src_csv = src_csv
+      @csv_options = EncodingOptions.read(csv_options)
+    end
+
+    # The raw row read before the current one (the header row for the first row), for debugging;
+    # after a full pass it's the last row.
+    # @return [Array<String>, nil]
+    def prev_row
+      @reader&.prev_row
     end
 
     # Yields each row from the start of the file.
     # @param headers [Array<String>, nil] headers to use; by default the first row is read as the headers
     # @yieldparam row [RowWrapper]
     # @return [Enumerator, nil] an enumerator without a block
-    def each(headers = nil)
+    # @raise [MalformedRowError] for a row CSV can't parse, with the line it starts on and {#prev_row}
+    # @raise [CSV::InvalidEncodingError] for bytes that don't match the encoding, with their line
+    def each(headers = nil, &)
       return enum_for(:each, headers) unless block_given?
 
-      @src_csv.rewind
-
-      lineno = 0
-      unless headers
-        headers = read_headers
-        lineno += 1
-      end
-
-      @prev_row = nil
-      while (row = @src_csv.shift)
-        lineno += 1
-        yield RowWrapper.create(headers, row, lineno)
-        @prev_row = row
-      end
+      open_csv { |csv| each_row(csv, headers, &) }
     end
 
     # The first row, without the byte order mark.
     # @return [Array<String>] empty for an empty file
     def headers
-      @src_csv.rewind
-      read_headers
+      open_csv { |csv| read_headers(csv) }
     end
 
     # Builds a lookup hash from every row.
@@ -83,10 +97,11 @@ module CSVUtils
     # @param value [String, nil] header whose value is the hash value
     # @yieldparam row [RowWrapper] used instead of value to compute the hash value
     # @return [Hash]
-    # @raise [RuntimeError] when key or value isn't a header
+    # @raise [HeaderNotFoundError] when key or value isn't a header
     def to_hash(key, value = nil, &)
-      raise("header #{key} not found in #{headers}") unless headers.include?(key)
-      raise("headers #{value} not found in #{headers}") if value && !headers.include?(value)
+      file_headers = headers
+      raise HeaderNotFoundError.new(key, file_headers) unless file_headers.include?(key)
+      raise HeaderNotFoundError.new(value, file_headers) if value && !file_headers.include?(value)
 
       value_proc =
         if value
@@ -103,11 +118,12 @@ module CSVUtils
     # Number of rows after the header row.
     # @return [Integer]
     def size
-      @src_csv.rewind
-      @src_csv.shift
-      cnt = 0
-      cnt += 1 while @src_csv.shift
-      cnt
+      open_csv do |csv|
+        csv.shift
+        cnt = 0
+        cnt += 1 while csv.shift
+        cnt
+      end
     end
 
     # Yields the rows in batches.
@@ -134,11 +150,26 @@ module CSVUtils
 
     private
 
+    # Opens a path for the duration of the block. Only a CSV that was passed in is rewound:
+    # rewinding goes back to byte 0, in front of a byte order mark the 'bom|' encoding skipped on open.
+    def open_csv
+      CSVWrapper.open(@src_csv, 'rb', @csv_options) do |csv|
+        csv.rewind unless @src_csv.is_a?(String)
+        yield csv
+      end
+    end
+
+    def each_row(csv, headers)
+      @reader = RowReader.new(csv)
+      headers ||= @reader.shift || []
+      while (row = @reader.shift)
+        yield RowWrapper.create(headers, row, @reader.lineno)
+      end
+    end
+
     # an empty file has no headers, and an empty first header cell is nil
-    def read_headers
-      headers = @src_csv.shift || []
-      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
-      headers
+    def read_headers(csv)
+      RowReader.new(csv).shift || []
     end
   end
 end