csv-utils 0.5.0 → 0.7.0

data/lib/csv_utils/byte_order_mark.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Detects and strips the byte order mark at the start of a CSV file.
+  module ByteOrderMark
+    # Byte order marks, as binary strings, to the encoding they indicate.
+    # This list is from https://en.wikipedia.org/wiki/Byte_order_mark.
+    # UTF-32 LE starts with the UTF-16 LE mark, so longer marks come first.
+    # @return [Hash{String => String}]
+    ENCODINGS = {
+      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'UTF-32',
+      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT').freeze => 'UTF-32',
+      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT').freeze => 'UTF-8',
+      (+"\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'UTF-16',
+      (+"\xFF\xFE").force_encoding('ASCII-8BIT').freeze => 'UTF-16'
+    }.freeze
+
+    # The byte order mark the string starts with. Compares bytes, so it works whatever the string's encoding is.
+    # @param str [String]
+    # @return [String, nil] the mark as a binary string
+    def self.detect(str)
+      prefix = str.byteslice(0, 4).b
+      ENCODINGS.keys.detect { |bom| prefix.start_with?(bom) }
+    end
+
+    # The string without its leading byte order mark, in its original encoding.
+    # @param str [String]
+    # @return [String]
+    def self.strip(str)
+      bom = detect(str)
+      bom ? str.byteslice(bom.bytesize..) : str
+    end
+  end
+end

data/lib/csv_utils/character_encoding.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Guesses the character encoding of a file without a byte order mark from a sample of its bytes.
+  module CharacterEncoding
+    # Bytes Windows-1252 leaves undefined; converting them to UTF-8 raises.
+    # @return [Regexp]
+    WINDOWS_1252_UNDEFINED = /[\x81\x8D\x8F\x90\x9D]/n
+
+    # 'UTF-8' when the sample is valid UTF-8. Otherwise 'Windows-1252', the usual encoding of Excel exports,
+    # or 'ISO-8859-1' when the sample has a byte Windows-1252 leaves undefined, since every ISO-8859-1 byte
+    # converts to UTF-8.
+    # @param sample [String] bytes from the start of the file, ending on a whole character
+    # @return [String]
+    def self.detect(sample)
+      bytes = sample.b
+      return 'UTF-8' if bytes.dup.force_encoding(Encoding::UTF_8).valid_encoding?
+      return 'ISO-8859-1' if bytes.match?(WINDOWS_1252_UNDEFINED)
+
+      'Windows-1252'
+    end
+  end
+end

data/lib/csv_utils/csv_compare.rb

@@ -1,93 +1,117 @@
 # frozen_string_literal: true
 
-# CSVUtils::CSVCompare purpose is to determine which rows in the secondary_data_file need to be created, deleted or updated
-# **requires both CSV files to be sorted on the same columns, CSVUtils::CSVSort can accomplish this
-# In order to receive updates, update_comparison_columns must configured or use inheritance and change the update_row? method
 module CSVUtils
+  # Compares two CSV files sorted on the same key and yields what it takes to make the secondary file
+  # match the primary one: records to create, update or delete. Sort both files first, {CSVSort} can do it.
+  #
+  # Updates are only reported for the update_comparison_columns; subclass and override update_row? for other rules.
+  #
+  # @example
+  #   comparer = CSVUtils::CSVCompare.new('primary.csv', ['updated_at']) { |src, dest| src['id'].to_i <=> dest['id'].to_i }
+  #   comparer.compare('secondary.csv') { |action, record| puts "#{action} #{record['id']}" }
   class CSVCompare
-    # primary_data_file is the source of truth
-    # compare_proc used to compare the id column(s)
-    # update_comparison_columns column(s) to compare for equality, ex: updated_at, timestamp, hash
-    #  caveat: update_comparison_columns need to be in both csv files
-    attr_reader :primary_data_file,
-                :update_comparison_columns,
-                :compare_proc
+    # @return [String] path of the source of truth
+    attr_reader :primary_data_file
+    # @return [Array<String>, nil] columns, present in both files, that trigger an update when they differ
+    #   (ex: updated_at, a timestamp or a hash of the row)
+    attr_reader :update_comparison_columns
+    # @return [Proc] compares the key columns of a primary and a secondary record, returning -1, 0 or 1
+    attr_reader :compare_proc
+    # @return [Hash] options passed to CSV.open for both files
+    attr_reader :csv_options
 
-    def initialize(primary_data_file, update_comparison_columns = nil, &block)
+    # @param primary_data_file [String] path of the source of truth
+    # @param update_comparison_columns [Array<String>, nil] columns compared on matching records;
+    #   without them no updates are yielded
+    # @param csv_options [Hash] options passed to CSV.open for both files, ex: col_sep: "\t",
+    #   or encoding: 'BINARY' to read bytes that aren't valid UTF-8; the :encoding defaults to 'bom|utf-8'
+    # @yieldparam src [Hash{String => String}] record from the primary file
+    # @yieldparam dest [Hash{String => String}] record from the secondary file
+    # @yieldreturn [Integer] negative, 0 or positive like <=>, using the same order both files are sorted by
+    def initialize(primary_data_file, update_comparison_columns = nil, csv_options = {}, &block)
       @primary_data_file = primary_data_file
       @update_comparison_columns = update_comparison_columns
+      @csv_options = csv_options
       @compare_proc = block
     end
 
-    # rubocop:disable Metrics/MethodLength
-    def compare(secondary_data_file)
-      src = CSV.open(primary_data_file, 'rb')
-      begin
-        src_headers = src.shift
-        strip_bom!(src_headers[0])
-        dest = CSV.open(secondary_data_file, 'rb')
-        begin
-          dest_headers = dest.shift
-          strip_bom!(dest_headers[0])
+    # Walks both files and yields each difference.
+    # @param secondary_data_file [String] path of the file to bring in line with the primary file
+    # @yieldparam action [Symbol] :create (only in the primary file), :update (in both, with changed
+    #   update_comparison_columns) or :delete (only in the secondary file)
+    # @yieldparam record [Hash{String => String}] the primary record for :create and :update,
+    #   the secondary record for :delete
+    # @return [void]
+    def compare(secondary_data_file, &)
+      open_csv(primary_data_file) do |src, src_headers|
+        open_csv(secondary_data_file) do |dest, dest_headers|
+          compare_records(src, src_headers, dest, dest_headers, &)
+        end
+      end
+    end
 
-          read_next_src = true
-          read_next_dest = true
+    private
 
-          while !src.eof? || !dest.eof?
-            src_record = next_record_from_file(src_headers, src) if read_next_src
-            dest_record = next_record_from_file(dest_headers, dest) if read_next_dest
+    # walks both sorted files at once, advancing whichever side is behind
+    def compare_records(src, src_headers, dest, dest_headers)
+      src_record = next_record(src_headers, src)
+      dest_record = next_record(dest_headers, dest)
 
-            if !src_record
-              read_next_src = false
-              read_next_dest = true
-              yield :delete, dest_record
-            elsif !dest_record
-              read_next_src = true
-              read_next_dest = false
-              yield :create, src_record
-            elsif compare_proc.call(src_record, dest_record).zero?
-              read_next_src = true
-              read_next_dest = true
-              yield(:update, src_record) if update_row?(src_record, dest_record)
-            elsif compare_proc.call(src_record, dest_record).positive?
-              read_next_src = false
-              read_next_dest = true
-              yield :delete, dest_record
-            else
-              read_next_src = true
-              read_next_dest = false
-              yield :create, src_record
-            end
-          end
-        ensure
-          dest.close
+      while src_record || dest_record
+        case record_action(src_record, dest_record)
+        when :create
+          yield :create, src_record
+          src_record = next_record(src_headers, src)
+        when :delete
+          yield :delete, dest_record
+          dest_record = next_record(dest_headers, dest)
+        else
+          yield :update, src_record if update_row?(src_record, dest_record)
+          src_record = next_record(src_headers, src)
+          dest_record = next_record(dest_headers, dest)
         end
-      ensure
-        src.close
       end
     end
-    # rubocop:enable Metrics/MethodLength
 
-    private
+    def record_action(src_record, dest_record)
+      return :delete unless src_record
+      return :create unless dest_record
 
-    def next_record_from_file(headers, file)
-      return nil if file.eof?
+      result = compare_proc.call(src_record, dest_record)
+      if result.zero?
+        :match
+      elsif result.positive?
+        :delete
+      else
+        :create
+      end
+    end
 
-      headers.zip(file.shift).to_h
+    def open_csv(file)
+      csv = CSV.open(file, 'rb', **EncodingOptions.read(csv_options))
+      yield csv, read_headers(csv)
+    ensure
+      csv&.close
     end
 
-    def update_row?(src_record, dest_record)
-      return false unless update_comparison_columns
+    def read_headers(csv)
+      headers = csv.shift || []
+      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
+      headers
+    end
 
-      update_comparison_columns.each do |column_name|
-        return true unless src_record[column_name] == dest_record[column_name]
-      end
+    def next_record(headers, csv)
+      return unless (row = csv.shift)
 
-      false
+      headers.zip(row).to_h
     end
 
-    def strip_bom!(col)
-      col.sub!((+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT'), '')
+    def update_row?(src_record, dest_record)
+      return false unless update_comparison_columns
+
+      update_comparison_columns.any? do |column_name|
+        src_record[column_name] != dest_record[column_name]
+      end
     end
   end
 end

data/lib/csv_utils/csv_extender.rb

@@ -1,13 +1,27 @@
 # frozen_string_literal: true
 
-# Utility class for appending data to a csv file.
 module CSVUtils
+  # Copies a CSV file to a new one with extra columns added to every row.
+  #
+  # @example
+  #   extender = CSVUtils::CSVExtender.new('input.csv', 'output.csv')
+  #   extender.append(['name_length']) { |row, headers| [row[headers.index('name')].size] }
   class CSVExtender
+    # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the extender
+    # @param dest_csv [String, CSV, #<<] path or CSV to write
+    # @param csv_options [Hash] options passed to CSV.open for paths; the source is read with its :encoding,
+    #   'bom|utf-8' by default, and the output written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(src_csv, dest_csv, csv_options = {})
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
-      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', EncodingOptions.read(csv_options))
+      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', EncodingOptions.write(csv_options))
     end
 
+    # Appends columns one row at a time, then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @yieldparam row [Array<String>] the source row
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array] values to append to the row
+    # @return [void]
     def append(additional_headers)
       process(additional_headers) do |current_headers|
         while (row = @src_csv.shift)
@@ -17,6 +31,14 @@ module CSVUtils
       end
     end
 
+    # Appends columns a batch of rows at a time, for lookups that are cheaper in bulk
+    # (ex: one database query per batch), then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @param batch_size [Integer] rows per batch
+    # @yieldparam batch [Array<Array<String>>] the source rows
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array<Array>] values to append, one array per row in the batch
+    # @return [void]
     def append_in_batches(additional_headers, batch_size = 1_000)
       process(additional_headers) do |current_headers|
         batch = []
@@ -47,7 +69,7 @@ module CSVUtils
       current_headers = append_headers(additional_headers)
 
       yield current_headers
-
+    ensure
       close
     end
 

data/lib/csv_utils/csv_iterator.rb

@@ -1,29 +1,45 @@
 # frozen_string_literal: true
 
-# Search a CSV given a series of steps
 module CSVUtils
+  # Enumerates the rows of a CSV file as hashes keyed by header, with the line number of each row.
+  #
+  # @example
+  #   iterator = CSVUtils::CSVIterator.new('data.csv')
+  #   iterator.each { |row| puts "#{row.lineno}: #{row['name']}" }
+  #   iterator.select { |row| row['status'] == 'active' }
   class CSVIterator
     include Enumerable
 
-    BYTE_ORDER_MARKS = [
-      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT'),       # UTF-8
-      (+"\xFE\xFF").force_encoding('ASCII-8BIT'),           # UTF-16 BE
-      (+"\xFF\xFE").force_encoding('ASCII-8BIT'),           # UTF-16 LE
-      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT'),   # UTF-32 BE
-      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT')    # UTF-32 LE
-    ].freeze
+    # @return [Array<String>] the byte order marks stripped from the first header, as binary strings
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS.keys.freeze
 
+    # @return [Array<String>, nil] the raw row read before the current one, for debugging malformed rows
     attr_reader :prev_row
 
+    # A row as a hash of header to value that knows its line number in the file.
     class RowWrapper < Hash
+      # @return [Integer] line number of the row in the file, counting the header row as line 1
       attr_accessor :lineno
 
+      # @param headers [Array<String>]
+      # @param row [Array<String>]
+      # @param lineno [Integer]
+      # @return [RowWrapper]
       def self.create(headers, row, lineno)
-        row_wrapper = RowWrapper[headers.zip(row)]
+        row_wrapper = new
         row_wrapper.lineno = lineno
+        # an index loop instead of headers.zip(row), which allocates an array per column for every row
+        idx = 0
+        size = headers.size
+        while idx < size
+          row_wrapper[headers[idx]] = row[idx]
+          idx += 1
+        end
         row_wrapper
       end
 
+      # Numbered "header: value" lines for the non-blank values, as csv-grep prints them.
+      # @return [String]
       def to_pretty_s
         reject { |_, v| v.nil? || v.strip.empty? }
           .each_with_index
@@ -32,38 +48,52 @@ module CSVUtils
       end
     end
 
-    def initialize(src_csv, csv_options = {}, mode = 'rb')
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, mode, csv_options)
+    # An iterator for a path, with the separators and encoding detected by {CSVOptions}.
+    # @example
+    #   CSVUtils::CSVIterator.auto_detect('export.tsv').each { |row| puts row['name'] }
+    # @param path [String]
+    # @return [CSVIterator]
+    def self.auto_detect(path)
+      options = CSVOptions.new(path)
+      new(path, options.to_csv_options)
     end
 
-    def each(headers = nil)
-      @src_csv.rewind
+    # A path is opened by each method call and closed when the call returns,
+    # so an idle iterator holds no file handle.
+    # @param src_csv [String, CSV] path or CSV to read; a CSV must support rewind and is never closed
+    # @param csv_options [Hash] options passed to CSV.open for a path. The :encoding defaults to 'bom|utf-8'
+    #   (see {EncodingOptions}); 'BINARY' reads bytes as is and 'Windows-1252:UTF-8' converts.
+    def initialize(src_csv, csv_options = {})
+      @src_csv = src_csv
+      @csv_options = EncodingOptions.read(csv_options)
+    end
 
-      lineno = 0
-      unless headers
-        headers = @src_csv.shift
-        strip_bom!(headers[0])
-        lineno += 1
-      end
+    # Yields each row from the start of the file.
+    # @param headers [Array<String>, nil] headers to use; by default the first row is read as the headers
+    # @yieldparam row [RowWrapper]
+    # @return [Enumerator, nil] an enumerator without a block
+    def each(headers = nil, &)
+      return enum_for(:each, headers) unless block_given?
 
-      @prev_row = nil
-      while (row = @src_csv.shift)
-        lineno += 1
-        yield RowWrapper.create(headers, row, lineno)
-        @prev_row = row
-      end
+      open_csv { |csv| each_row(csv, headers, &) }
     end
 
+    # The first row, without the byte order mark.
+    # @return [Array<String>] empty for an empty file
     def headers
-      @src_csv.rewind
-      headers = @src_csv.shift
-      strip_bom!(headers[0])
-      headers
+      open_csv { |csv| read_headers(csv) }
     end
 
+    # Builds a lookup hash from every row.
+    # @param key [String] header whose value is the hash key
+    # @param value [String, nil] header whose value is the hash value
+    # @yieldparam row [RowWrapper] used instead of value to compute the hash value
+    # @return [Hash]
+    # @raise [RuntimeError] when key or value isn't a header
     def to_hash(key, value = nil, &)
-      raise("header #{key} not found in #{headers}") unless headers.include?(key)
-      raise("headers #{value} not found in #{headers}") if value && !headers.include?(value)
+      file_headers = headers
+      raise("header #{key} not found in #{file_headers}") unless file_headers.include?(key)
+      raise("headers #{value} not found in #{file_headers}") if value && !file_headers.include?(value)
 
       value_proc =
         if value
@@ -72,19 +102,26 @@ module CSVUtils
           proc(&)
         end
 
-      each_with_object({}) do |row, hsh|
-        hsh[row[key]] = value_proc.call(row)
+      to_h do |row|
+        [row[key], value_proc.call(row)]
       end
     end
 
+    # Number of rows after the header row.
+    # @return [Integer]
     def size
-      @src_csv.rewind
-      @src_csv.shift
-      cnt = 0
-      cnt += 1 while @src_csv.shift
-      cnt
+      open_csv do |csv|
+        csv.shift
+        cnt = 0
+        cnt += 1 while csv.shift
+        cnt
+      end
     end
 
+    # Yields the rows in batches.
+    # @param batch_size [Integer] rows per batch; the last batch may be smaller
+    # @yieldparam batch [Array<RowWrapper>]
+    # @return [nil]
     def each_batch(batch_size = 1_000)
       batch = []
 
@@ -105,13 +142,43 @@ module CSVUtils
 
     private
 
-    def strip_bom!(col)
-      BYTE_ORDER_MARKS.each do |bom|
-        if col.start_with?(bom)
-          col.sub!(bom, '')
-          break
-        end
+    # Opens a path for the duration of the block. Only a CSV that was passed in is rewound:
+    # rewinding goes back to byte 0, in front of a byte order mark the 'bom|' encoding skipped on open.
+    def open_csv
+      CSVWrapper.open(@src_csv, 'rb', @csv_options) do |csv|
+        csv.rewind unless @src_csv.is_a?(String)
+        yield csv
       end
     end
+
+    def each_row(csv, headers)
+      @prev_row = nil
+      row = shift_first_row(csv)
+      lineno = 1
+      unless headers
+        headers = row || []
+        row = csv.shift
+        lineno += 1
+      end
+
+      while row
+        yield RowWrapper.create(headers, row, lineno)
+        @prev_row = row
+        row = csv.shift
+        lineno += 1
+      end
+    end
+
+    # an empty file has no headers, and an empty first header cell is nil
+    def read_headers(csv)
+      shift_first_row(csv) || []
+    end
+
+    # the first row without the byte order mark, whether it's read as headers or as data
+    def shift_first_row(csv)
+      row = csv.shift
+      row[0] = ByteOrderMark.strip(row[0]) if row && row[0]
+      row
+    end
   end
 end