csv-utils 0.3.25 → 0.6.0

data/bin/csv-find-error

@@ -1,19 +1,21 @@
 #!/usr/bin/env ruby
 
 require 'csv'
-require 'shellwords'
+require 'rbconfig'
+
+file = ARGV[0] || abort('Usage: csv-find-error <csv file>')
 
 prev_row = nil
 begin
-  CSV.open(ARGV[0], 'rb').each { |row| prev_row = row }
+  CSV.open(file, 'rb') { |csv| csv.each { |row| prev_row = row } }
 rescue CSV::MalformedCSVError => e
   puts e.class.to_s + ': ' + e.message
   puts "previous row was #{prev_row}"
   if e.message =~ /line (\d+)/
     lineno = $1.to_i
-    cmd = "csv-readline #{Shellwords.escape(ARGV[0])} #{lineno}"
-    puts "running #{cmd}"
-    system(cmd)
+    # run the csv-readline next to this script, with this gem's lib, rather than whichever is on the PATH
+    puts "running csv-readline #{file} #{lineno}"
+    system(RbConfig.ruby, '-I', File.expand_path('../lib', __dir__), File.join(__dir__, 'csv-readline'), file, lineno.to_s)
   end
   exit 1
 end

data/bin/csv-grep

@@ -34,7 +34,7 @@ OptionParser.new do |opts|
     options[:limit] = v
   end
 
-  opts.on('-i', '--ignore-case', 'Ignore case') do |v|
+  opts.on('-i', '--ignore-case', 'Ignore case') do
     options[:search_regex_options] = Regexp::IGNORECASE
   end
 end.parse!
@@ -55,7 +55,7 @@ search_regex =
 headers =
   case options[:headers]
   when :first
-    [csv .headers.first]
+    [csv.headers.first]
   when :all
     csv.headers
   else
@@ -63,7 +63,7 @@ headers =
   end
 
 missing_headers = headers - csv.headers
-raise("unknown headers #{headers.join(', ')}") unless missing_headers.empty?
+raise("unknown headers #{missing_headers.join(', ')}") unless missing_headers.empty?
 
 matching_row_proc = proc do |row|
   result = false

data/bin/csv-readline

@@ -2,21 +2,10 @@
 
 require 'optparse'
 
-BYTE_ORDER_MARKS = {
-  "\xEF\xBB\xBF".force_encoding('ASCII-8BIT') => 'UTF-8',
-  "\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-16',
-  "\xFF\xFE".force_encoding('ASCII-8BIT') => 'UTF-16',
-  "\x00\x00\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-32',
-  "\xFF\xFE\x00\x00".force_encoding('ASCII-8BIT') => 'UTF-32'
-}
+require 'csv-utils'
 
 def strip_byte_order_mark(line)
-  BYTE_ORDER_MARKS.keys.detect do |bom|
-    if line =~ /\A#{bom}/
-      return line.sub(/\A#{bom}/, '')
-    end
-  end
-  line
+  CSVUtils::ByteOrderMark.strip(line)
 end
 
 def csv_parse_line(line)
@@ -87,7 +76,7 @@ end
 
 def parse_csv_row(file, lineno, number_of_lines)
   cnt = 1
-  str = ''
+  str = +''
   file.each do |line|
     cnt += 1
     next if cnt < lineno
@@ -114,16 +103,15 @@ OptionParser.new do |opts|
   end
 end.parse!
 
-file = File.open(ARGV[0], 'rb')
 lineno = ARGV[1].to_i
 number_of_lines = (ARGV[2] || 1).to_i
 
 raise "no lineno specified" unless lineno > 0
 
-headers = strip_byte_order_mark(file.readline.strip).split(',')
-
-data = headers.zip(parse_csv_row(file, lineno, number_of_lines))
-file.close
+data = File.open(ARGV[0], 'rb') do |file|
+  headers = strip_byte_order_mark(file.readline.strip).split(',')
+  headers.zip(parse_csv_row(file, lineno, number_of_lines))
+end
 
 cnt = 0
 data.each do |k, (v, status)|

data/bin/csv-splitter

@@ -15,6 +15,8 @@ OptionParser.new do |opts|
   end
 
   opts.on('-r', '--rows ROWS', Integer, 'Max number of rows per a CSV') do |v|
+    raise OptionParser::InvalidArgument, "--rows must be positive (#{v})" unless v.positive?
+
     options[:rows] = v
   end
 
@@ -36,20 +38,20 @@ def get_split_file_name(file, num_files, total_files)
 end
 
 csv_options = CSVUtils::CSVOptions.new(file)
+# a single column file has no separator to detect
+col_sep = csv_options.col_separator || ','
 
 csv = CSV.open(
   file,
   'rb',
-  col_sep: csv_options.col_separator,
+  col_sep: col_sep,
   liberal_parsing: true
 )
 
 headers = options[:no_header] ? nil : csv.shift
 
 total_rows = 0
-while csv.shift
-  total_rows +=1
-end
+total_rows += 1 while csv.shift
 
 csv.rewind
 csv.shift if headers
@@ -66,7 +68,7 @@ append_row_proc = proc do |row|
     out.close if out
     num_rows = 0
     num_files += 1
-    out = CSV.open(get_split_file_name(file, num_files, total_files), 'wb', col_sep: csv_options.col_separator)
+    out = CSV.open(get_split_file_name(file, num_files, total_files), 'wb', col_sep: col_sep)
     out << headers if headers
   end
 
@@ -78,4 +80,4 @@ while (row = csv.shift)
   append_row_proc.call(row)
 end
 
-out.close
+out&.close

data/bin/csv-validator

@@ -15,6 +15,8 @@ def utf8?(str)
 end
 
 def convert_to_utf8(str, current_encoding)
+  return nil unless current_encoding
+
   str.force_encoding(current_encoding)
   return nil unless str.valid_encoding?
 
@@ -29,53 +31,57 @@ def strip_bom!(col)
   col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
 end
 
-csv = CSV.open(ARGV[0], 'rb')
 id_column_name = ARGV[1]
+# read bytes as is; reading as UTF-8 would raise on the values this script looks for
+csv = CSV.open(ARGV[0] || abort('Usage: csv-validator <csv file> [<id header>]'), 'rb:BINARY')
+out = nil
 
-headers = csv.shift
-strip_bom!(headers[0])
-
-id_column_name ||= headers[0]
-unless headers.include?(id_column_name)
-  $stderr.puts("header #{id_column_name} not found in current set of headers")
-  exit 1
-end
+begin
+  headers = csv.shift || []
+  strip_bom!(headers[0]) if headers[0]
 
-id_column_num = headers.index(id_column_name)
+  id_column_name ||= headers[0]
+  unless headers.include?(id_column_name)
+    $stderr.puts("header #{id_column_name} not found in current set of headers")
+    exit 1
+  end
 
-out = nil
-out_proc = Proc.new do |row|
-  out ||=
-    begin
-      out = CSV.open('utf8-correctsion.csv', 'wb')
-      out << [id_column_name, 'Row', 'Col', 'Header', 'Value']
-      out
-    end
+  id_column_num = headers.index(id_column_name)
 
-  out << row
-end
+  out_proc = proc do |row|
+    out ||=
+      begin
+        out = CSV.open('utf8-correction.csv', 'wb')
+        out << [id_column_name, 'Row', 'Col', 'Header', 'Value']
+        out
+      end
 
-csv_lineno = 1
+    out << row
+  end
 
-while (row = csv.shift)
-  csv_lineno += 1
+  csv_lineno = 1
 
-  unless row.size == headers.size
-    $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
-  end
+  while (row = csv.shift)
+    csv_lineno += 1
 
-  row.each_with_index do |col, idx|
-    next if col.nil? || utf8?(col)
+    unless row.size == headers.size
+      $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
+    end
 
-    $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: none UTF-8 characters found in \"#{col}\""
-    if (col_utf8_encoded = convert_to_utf8(col, detect_encoding(col)))
-      puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{detect_encoding(col)} \"#{col_utf8_encoded}\""
-      out_proc.call [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
-    else
-      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
+    row.each_with_index do |col, idx|
+      next if col.nil? || utf8?(col)
+
+      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: non UTF-8 characters found in #{col.inspect}"
+      encoding = detect_encoding(col)
+      if (col_utf8_encoded = convert_to_utf8(col, encoding))
+        puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{encoding} \"#{col_utf8_encoded}\""
+        out_proc.call [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
+      else
+        $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
+      end
     end
   end
+ensure
+  csv.close
+  out&.close
 end
-
-csv.close
-out.close if out

data/lib/csv-utils.rb

@@ -1,13 +1,18 @@
+# frozen_string_literal: true
+
 require 'csv'
 
-# Collection of tools for working with CSV files.
+# Tools for comparing, sorting, transforming and debugging CSV files, including ones too large to load
+# into memory. Each class is autoloaded on first use.
 module CSVUtils
+  autoload :ByteOrderMark, 'csv_utils/byte_order_mark'
   autoload :CSVCompare, 'csv_utils/csv_compare'
   autoload :CSVExtender, 'csv_utils/csv_extender'
   autoload :CSVIterator, 'csv_utils/csv_iterator'
   autoload :CSVOptions, 'csv_utils/csv_options'
   autoload :CSVReport, 'csv_utils/csv_report'
   autoload :CSVRow, 'csv_utils/csv_row'
+  autoload :CSVRowMatcher, 'csv_utils/csv_row_matcher'
   autoload :CSVSort, 'csv_utils/csv_sort'
   autoload :CSVTransformer, 'csv_utils/csv_transformer'
   autoload :CSVWrapper, 'csv_utils/csv_wrapper'

data/lib/csv_utils/byte_order_mark.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Detects and strips the byte order mark at the start of a CSV file.
+  module ByteOrderMark
+    # Byte order marks, as binary strings, to the encoding they indicate.
+    # This list is from https://en.wikipedia.org/wiki/Byte_order_mark.
+    # UTF-32 LE starts with the UTF-16 LE mark, so longer marks come first.
+    # @return [Hash{String => String}]
+    ENCODINGS = {
+      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'UTF-32',
+      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT').freeze => 'UTF-32',
+      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT').freeze => 'UTF-8',
+      (+"\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'UTF-16',
+      (+"\xFF\xFE").force_encoding('ASCII-8BIT').freeze => 'UTF-16'
+    }.freeze
+
+    # The byte order mark the string starts with. Compares bytes, so it works whatever the string's encoding is.
+    # @param str [String]
+    # @return [String, nil] the mark as a binary string
+    def self.detect(str)
+      prefix = str.byteslice(0, 4).b
+      ENCODINGS.keys.detect { |bom| prefix.start_with?(bom) }
+    end
+
+    # The string without its leading byte order mark, in its original encoding.
+    # @param str [String]
+    # @return [String]
+    def self.strip(str)
+      bom = detect(str)
+      bom ? str.byteslice(bom.bytesize..) : str
+    end
+  end
+end

data/lib/csv_utils/csv_compare.rb

@@ -1,87 +1,117 @@
-# CSVUtils::CSVCompare purpose is to determine which rows in the secondary_data_file need to be created, deleted or updated
-# **requires both CSV files to be sorted on the same columns, CSVUtils::CSVSort can accomplish this
-# In order to receive updates, update_comparison_columns must configured or use inheritance and change the update_row? method
-class CSVUtils::CSVCompare
-  # primary_data_file is the source of truth
-  # compare_proc used to compare the id column(s)
-  # update_comparison_columns column(s) to compare for equality, ex: updated_at, timestamp, hash
-  #  caveat: update_comparison_columns need to be in both csv files
-  attr_reader :primary_data_file,
-              :update_comparison_columns,
-              :compare_proc
-
-  def initialize(primary_data_file, update_comparison_columns=nil, &block)
-    @primary_data_file = primary_data_file
-    @update_comparison_columns = update_comparison_columns
-    @compare_proc = block
-  end
-
-  def compare(secondary_data_file)
-    src = CSV.open(primary_data_file, 'rb')
-    src_headers = src.shift
-    strip_bom!(src_headers[0])
-    dest = CSV.open(secondary_data_file, 'rb')
-    dest_headers = dest.shift
-    strip_bom!(dest_headers[0])
-
-    read_next_src = true
-    read_next_dest = true
-
-    while(!src.eof? || !dest.eof?)
-      src_record = next_record_from_file(src_headers, src) if read_next_src
-      dest_record = next_record_from_file(dest_headers, dest) if read_next_dest
-
-      if ! src_record
-        read_next_src = false
-        read_next_dest = true
+# frozen_string_literal: true
+
+module CSVUtils
+  # Compares two CSV files sorted on the same key and yields what it takes to make the secondary file
+  # match the primary one: records to create, update or delete. Sort both files first, {CSVSort} can do it.
+  #
+  # Updates are only reported for the update_comparison_columns; subclass and override update_row? for other rules.
+  #
+  # @example
+  #   comparer = CSVUtils::CSVCompare.new('primary.csv', ['updated_at']) { |src, dest| src['id'].to_i <=> dest['id'].to_i }
+  #   comparer.compare('secondary.csv') { |action, record| puts "#{action} #{record['id']}" }
+  class CSVCompare
+    # @return [String] path of the source of truth
+    attr_reader :primary_data_file
+    # @return [Array<String>, nil] columns, present in both files, that trigger an update when they differ
+    #   (ex: updated_at, a timestamp or a hash of the row)
+    attr_reader :update_comparison_columns
+    # @return [Proc] compares the key columns of a primary and a secondary record, returning -1, 0 or 1
+    attr_reader :compare_proc
+    # @return [Hash] options passed to CSV.open for both files
+    attr_reader :csv_options
+
+    # @param primary_data_file [String] path of the source of truth
+    # @param update_comparison_columns [Array<String>, nil] columns compared on matching records;
+    #   without them no updates are yielded
+    # @param csv_options [Hash] options passed to CSV.open for both files, ex: col_sep: "\t",
+    #   or encoding: 'BINARY' to read bytes that aren't valid UTF-8
+    # @yieldparam src [Hash{String => String}] record from the primary file
+    # @yieldparam dest [Hash{String => String}] record from the secondary file
+    # @yieldreturn [Integer] negative, 0 or positive like <=>, using the same order both files are sorted by
+    def initialize(primary_data_file, update_comparison_columns = nil, csv_options = {}, &block)
+      @primary_data_file = primary_data_file
+      @update_comparison_columns = update_comparison_columns
+      @csv_options = csv_options
+      @compare_proc = block
+    end
 
-        yield :delete, dest_record
-      elsif ! dest_record
-        read_next_src = true
-        read_next_dest = false
+    # Walks both files and yields each difference.
+    # @param secondary_data_file [String] path of the file to bring in line with the primary file
+    # @yieldparam action [Symbol] :create (only in the primary file), :update (in both, with changed
+    #   update_comparison_columns) or :delete (only in the secondary file)
+    # @yieldparam record [Hash{String => String}] the primary record for :create and :update,
+    #   the secondary record for :delete
+    # @return [void]
+    def compare(secondary_data_file, &)
+      open_csv(primary_data_file) do |src, src_headers|
+        open_csv(secondary_data_file) do |dest, dest_headers|
+          compare_records(src, src_headers, dest, dest_headers, &)
+        end
+      end
+    end
 
-        yield :create, src_record
-      elsif compare_proc.call(src_record, dest_record) == 0
-        read_next_src = true
-        read_next_dest = true
+    private
+
+    # walks both sorted files at once, advancing whichever side is behind
+    def compare_records(src, src_headers, dest, dest_headers)
+      src_record = next_record(src_headers, src)
+      dest_record = next_record(dest_headers, dest)
+
+      while src_record || dest_record
+        case record_action(src_record, dest_record)
+        when :create
+          yield :create, src_record
+          src_record = next_record(src_headers, src)
+        when :delete
+          yield :delete, dest_record
+          dest_record = next_record(dest_headers, dest)
+        else
+          yield :update, src_record if update_row?(src_record, dest_record)
+          src_record = next_record(src_headers, src)
+          dest_record = next_record(dest_headers, dest)
+        end
+      end
+    end
 
-        yield(:update, src_record) if update_row?(src_record, dest_record)
-      elsif compare_proc.call(src_record, dest_record) > 0
-        read_next_src = false
-        read_next_dest = true
+    def record_action(src_record, dest_record)
+      return :delete unless src_record
+      return :create unless dest_record
 
-        yield :delete, dest_record
+      result = compare_proc.call(src_record, dest_record)
+      if result.zero?
+        :match
+      elsif result.positive?
+        :delete
       else
-        read_next_src = true
-        read_next_dest = false
-
-        yield :create, src_record
+        :create
       end
     end
 
-    src.close
-    dest.close
-  end
-
-  private
-
-  def next_record_from_file(headers, file)
-    return nil if file.eof?
+    def open_csv(file)
+      csv = CSV.open(file, 'rb', **csv_options)
+      yield csv, read_headers(csv)
+    ensure
+      csv&.close
+    end
 
-    Hash[headers.zip(file.shift)]
-  end
+    def read_headers(csv)
+      headers = csv.shift || []
+      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
+      headers
+    end
 
-  def update_row?(src_record, dest_record)
-    return false unless update_comparison_columns
+    def next_record(headers, csv)
+      return unless (row = csv.shift)
 
-    update_comparison_columns.each do |column_name|
-      return true unless src_record[column_name] == dest_record[column_name]
+      headers.zip(row).to_h
     end
 
-    false
-  end
+    def update_row?(src_record, dest_record)
+      return false unless update_comparison_columns
 
-  def strip_bom!(col)
-    col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+      update_comparison_columns.any? do |column_name|
+        src_record[column_name] != dest_record[column_name]
+      end
+    end
   end
 end

data/lib/csv_utils/csv_extender.rb

@@ -1,63 +1,88 @@
-# Utility class for appending data to a csv file.
-class CSVUtils::CSVExtender
-  def initialize(src_csv, dest_csv, csv_options = {})
-    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
-    @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
-  end
+# frozen_string_literal: true
+
+module CSVUtils
+  # Copies a CSV file to a new one with extra columns added to every row.
+  #
+  # @example
+  #   extender = CSVUtils::CSVExtender.new('input.csv', 'output.csv')
+  #   extender.append(['name_length']) { |row, headers| [row[headers.index('name')].size] }
+  class CSVExtender
+    # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the extender
+    # @param dest_csv [String, CSV, #<<] path or CSV to write
+    # @param csv_options [Hash] options passed to CSV.open for paths
+    def initialize(src_csv, dest_csv, csv_options = {})
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+    end
 
-  def append(additional_headers)
-    process(additional_headers) do |current_headers|
-      while (row = @src_csv.shift)
-        additional_columns = yield row, current_headers
-        @dest_csv << (row + additional_columns)
+    # Appends columns one row at a time, then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @yieldparam row [Array<String>] the source row
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array] values to append to the row
+    # @return [void]
+    def append(additional_headers)
+      process(additional_headers) do |current_headers|
+        while (row = @src_csv.shift)
+          additional_columns = yield row, current_headers
+          @dest_csv << (row + additional_columns)
+        end
       end
     end
-  end
 
-  def append_in_batches(additional_headers, batch_size = 1_000)
-    process(additional_headers) do |current_headers|
-      batch = []
+    # Appends columns a batch of rows at a time, for lookups that are cheaper in bulk
+    # (ex: one database query per batch), then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @param batch_size [Integer] rows per batch
+    # @yieldparam batch [Array<Array<String>>] the source rows
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array<Array>] values to append, one array per row in the batch
+    # @return [void]
+    def append_in_batches(additional_headers, batch_size = 1_000)
+      process(additional_headers) do |current_headers|
+        batch = []
+
+        process_batch_proc = proc do
+          additional_rows = yield batch, current_headers
 
-      process_batch_proc = Proc.new do
-        additional_rows = yield batch, current_headers
+          batch.each_with_index do |row, idx|
+            @dest_csv << (row + additional_rows[idx])
+          end
 
-        batch.each_with_index do |row, idx|
-          @dest_csv << (row + additional_rows[idx])
+          batch = []
         end
 
-        batch = []
-      end
+        while (row = @src_csv.shift)
+          batch << row
 
-      while (row = @src_csv.shift)
-        batch << row
+          process_batch_proc.call if batch.size >= batch_size
+        end
 
-        process_batch_proc.call if batch.size >= batch_size
+        process_batch_proc.call if batch.size.positive?
       end
-
-      process_batch_proc.call if batch.size > 0
     end
-  end
 
-  private
+    private
 
-  def process(additional_headers)
-    current_headers = append_headers(additional_headers)
+    def process(additional_headers)
+      current_headers = append_headers(additional_headers)
 
-    yield current_headers
-
-    close
-  end
+      yield current_headers
+    ensure
+      close
+    end
 
-  def close
-    @src_csv.close
-    @dest_csv.close
-  end
+    def close
+      @src_csv.close
+      @dest_csv.close
+    end
 
-  def append_headers(additional_headers)
-    return nil unless additional_headers
+    def append_headers(additional_headers)
+      return nil unless additional_headers
 
-    current_headers = @src_csv.shift
-    @dest_csv << (current_headers + additional_headers)
-    current_headers
+      current_headers = @src_csv.shift
+      @dest_csv << (current_headers + additional_headers)
+      current_headers
+    end
   end
 end