csv-utils 0.5.0 → 0.6.0

data/lib/csv_utils/csv_compare.rb

@@ -1,93 +1,117 @@
 # frozen_string_literal: true
 
-# CSVUtils::CSVCompare purpose is to determine which rows in the secondary_data_file need to be created, deleted or updated
-# **requires both CSV files to be sorted on the same columns, CSVUtils::CSVSort can accomplish this
-# In order to receive updates, update_comparison_columns must configured or use inheritance and change the update_row? method
 module CSVUtils
+  # Compares two CSV files sorted on the same key and yields what it takes to make the secondary file
+  # match the primary one: records to create, update or delete. Sort both files first, {CSVSort} can do it.
+  #
+  # Updates are only reported for the update_comparison_columns; subclass and override update_row? for other rules.
+  #
+  # @example
+  #   comparer = CSVUtils::CSVCompare.new('primary.csv', ['updated_at']) { |src, dest| src['id'].to_i <=> dest['id'].to_i }
+  #   comparer.compare('secondary.csv') { |action, record| puts "#{action} #{record['id']}" }
   class CSVCompare
-    # primary_data_file is the source of truth
-    # compare_proc used to compare the id column(s)
-    # update_comparison_columns column(s) to compare for equality, ex: updated_at, timestamp, hash
-    #  caveat: update_comparison_columns need to be in both csv files
-    attr_reader :primary_data_file,
-                :update_comparison_columns,
-                :compare_proc
+    # @return [String] path of the source of truth
+    attr_reader :primary_data_file
+    # @return [Array<String>, nil] columns, present in both files, that trigger an update when they differ
+    #   (ex: updated_at, a timestamp or a hash of the row)
+    attr_reader :update_comparison_columns
+    # @return [Proc] compares the key columns of a primary and a secondary record, returning -1, 0 or 1
+    attr_reader :compare_proc
+    # @return [Hash] options passed to CSV.open for both files
+    attr_reader :csv_options
 
-    def initialize(primary_data_file, update_comparison_columns = nil, &block)
+    # @param primary_data_file [String] path of the source of truth
+    # @param update_comparison_columns [Array<String>, nil] columns compared on matching records;
+    #   without them no updates are yielded
+    # @param csv_options [Hash] options passed to CSV.open for both files, ex: col_sep: "\t",
+    #   or encoding: 'BINARY' to read bytes that aren't valid UTF-8
+    # @yieldparam src [Hash{String => String}] record from the primary file
+    # @yieldparam dest [Hash{String => String}] record from the secondary file
+    # @yieldreturn [Integer] negative, 0 or positive like <=>, using the same order both files are sorted by
+    def initialize(primary_data_file, update_comparison_columns = nil, csv_options = {}, &block)
       @primary_data_file = primary_data_file
       @update_comparison_columns = update_comparison_columns
+      @csv_options = csv_options
       @compare_proc = block
     end
 
-    # rubocop:disable Metrics/MethodLength
-    def compare(secondary_data_file)
-      src = CSV.open(primary_data_file, 'rb')
-      begin
-        src_headers = src.shift
-        strip_bom!(src_headers[0])
-        dest = CSV.open(secondary_data_file, 'rb')
-        begin
-          dest_headers = dest.shift
-          strip_bom!(dest_headers[0])
+    # Walks both files and yields each difference.
+    # @param secondary_data_file [String] path of the file to bring in line with the primary file
+    # @yieldparam action [Symbol] :create (only in the primary file), :update (in both, with changed
+    #   update_comparison_columns) or :delete (only in the secondary file)
+    # @yieldparam record [Hash{String => String}] the primary record for :create and :update,
+    #   the secondary record for :delete
+    # @return [void]
+    def compare(secondary_data_file, &)
+      open_csv(primary_data_file) do |src, src_headers|
+        open_csv(secondary_data_file) do |dest, dest_headers|
+          compare_records(src, src_headers, dest, dest_headers, &)
+        end
+      end
+    end
 
-          read_next_src = true
-          read_next_dest = true
+    private
 
-          while !src.eof? || !dest.eof?
-            src_record = next_record_from_file(src_headers, src) if read_next_src
-            dest_record = next_record_from_file(dest_headers, dest) if read_next_dest
+    # walks both sorted files at once, advancing whichever side is behind
+    def compare_records(src, src_headers, dest, dest_headers)
+      src_record = next_record(src_headers, src)
+      dest_record = next_record(dest_headers, dest)
 
-            if !src_record
-              read_next_src = false
-              read_next_dest = true
-              yield :delete, dest_record
-            elsif !dest_record
-              read_next_src = true
-              read_next_dest = false
-              yield :create, src_record
-            elsif compare_proc.call(src_record, dest_record).zero?
-              read_next_src = true
-              read_next_dest = true
-              yield(:update, src_record) if update_row?(src_record, dest_record)
-            elsif compare_proc.call(src_record, dest_record).positive?
-              read_next_src = false
-              read_next_dest = true
-              yield :delete, dest_record
-            else
-              read_next_src = true
-              read_next_dest = false
-              yield :create, src_record
-            end
-          end
-        ensure
-          dest.close
+      while src_record || dest_record
+        case record_action(src_record, dest_record)
+        when :create
+          yield :create, src_record
+          src_record = next_record(src_headers, src)
+        when :delete
+          yield :delete, dest_record
+          dest_record = next_record(dest_headers, dest)
+        else
+          yield :update, src_record if update_row?(src_record, dest_record)
+          src_record = next_record(src_headers, src)
+          dest_record = next_record(dest_headers, dest)
         end
-      ensure
-        src.close
       end
     end
-    # rubocop:enable Metrics/MethodLength
 
-    private
+    def record_action(src_record, dest_record)
+      return :delete unless src_record
+      return :create unless dest_record
 
-    def next_record_from_file(headers, file)
-      return nil if file.eof?
+      result = compare_proc.call(src_record, dest_record)
+      if result.zero?
+        :match
+      elsif result.positive?
+        :delete
+      else
+        :create
+      end
+    end
 
-      headers.zip(file.shift).to_h
+    def open_csv(file)
+      csv = CSV.open(file, 'rb', **csv_options)
+      yield csv, read_headers(csv)
+    ensure
+      csv&.close
     end
 
-    def update_row?(src_record, dest_record)
-      return false unless update_comparison_columns
+    def read_headers(csv)
+      headers = csv.shift || []
+      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
+      headers
+    end
 
-      update_comparison_columns.each do |column_name|
-        return true unless src_record[column_name] == dest_record[column_name]
-      end
+    def next_record(headers, csv)
+      return unless (row = csv.shift)
 
-      false
+      headers.zip(row).to_h
     end
 
-    def strip_bom!(col)
-      col.sub!((+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT'), '')
+    def update_row?(src_record, dest_record)
+      return false unless update_comparison_columns
+
+      update_comparison_columns.any? do |column_name|
+        src_record[column_name] != dest_record[column_name]
+      end
     end
   end
 end

data/lib/csv_utils/csv_extender.rb

@@ -1,13 +1,26 @@
 # frozen_string_literal: true
 
-# Utility class for appending data to a csv file.
 module CSVUtils
+  # Copies a CSV file to a new one with extra columns added to every row.
+  #
+  # @example
+  #   extender = CSVUtils::CSVExtender.new('input.csv', 'output.csv')
+  #   extender.append(['name_length']) { |row, headers| [row[headers.index('name')].size] }
   class CSVExtender
+    # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the extender
+    # @param dest_csv [String, CSV, #<<] path or CSV to write
+    # @param csv_options [Hash] options passed to CSV.open for paths
     def initialize(src_csv, dest_csv, csv_options = {})
       @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
       @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
     end
 
+    # Appends columns one row at a time, then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @yieldparam row [Array<String>] the source row
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array] values to append to the row
+    # @return [void]
     def append(additional_headers)
       process(additional_headers) do |current_headers|
         while (row = @src_csv.shift)
@@ -17,6 +30,14 @@ module CSVUtils
       end
     end
 
+    # Appends columns a batch of rows at a time, for lookups that are cheaper in bulk
+    # (ex: one database query per batch), then closes the files it opened.
+    # @param additional_headers [Array<String>, nil] headers to add; nil when the source has no header row
+    # @param batch_size [Integer] rows per batch
+    # @yieldparam batch [Array<Array<String>>] the source rows
+    # @yieldparam headers [Array<String>, nil] the source headers
+    # @yieldreturn [Array<Array>] values to append, one array per row in the batch
+    # @return [void]
     def append_in_batches(additional_headers, batch_size = 1_000)
       process(additional_headers) do |current_headers|
         batch = []
@@ -47,7 +68,7 @@ module CSVUtils
       current_headers = append_headers(additional_headers)
 
       yield current_headers
-
+    ensure
       close
     end
 

data/lib/csv_utils/csv_iterator.rb

@@ -1,29 +1,38 @@
 # frozen_string_literal: true
 
-# Search a CSV given a series of steps
 module CSVUtils
+  # Enumerates the rows of a CSV file as hashes keyed by header, with the line number of each row.
+  #
+  # @example
+  #   iterator = CSVUtils::CSVIterator.new('data.csv')
+  #   iterator.each { |row| puts "#{row.lineno}: #{row['name']}" }
+  #   iterator.select { |row| row['status'] == 'active' }
   class CSVIterator
     include Enumerable
 
-    BYTE_ORDER_MARKS = [
-      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT'),       # UTF-8
-      (+"\xFE\xFF").force_encoding('ASCII-8BIT'),           # UTF-16 BE
-      (+"\xFF\xFE").force_encoding('ASCII-8BIT'),           # UTF-16 LE
-      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT'),   # UTF-32 BE
-      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT')    # UTF-32 LE
-    ].freeze
+    # @return [Array<String>] the byte order marks stripped from the first header, as binary strings
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS.keys.freeze
 
+    # @return [Array<String>, nil] the raw row read before the current one, for debugging malformed rows
     attr_reader :prev_row
 
+    # A row as a hash of header to value that knows its line number in the file.
     class RowWrapper < Hash
+      # @return [Integer] line number of the row in the file, counting the header row as line 1
       attr_accessor :lineno
 
+      # @param headers [Array<String>]
+      # @param row [Array<String>]
+      # @param lineno [Integer]
+      # @return [RowWrapper]
       def self.create(headers, row, lineno)
         row_wrapper = RowWrapper[headers.zip(row)]
         row_wrapper.lineno = lineno
         row_wrapper
       end
 
+      # Numbered "header: value" lines for the non-blank values, as csv-grep prints them.
+      # @return [String]
       def to_pretty_s
         reject { |_, v| v.nil? || v.strip.empty? }
           .each_with_index
@@ -32,17 +41,25 @@ module CSVUtils
       end
     end
 
+    # @param src_csv [String, CSV] path or CSV to read; it must support rewind
+    # @param csv_options [Hash] options passed to CSV.open for a path
+    # @param mode [String] file mode for a path; 'rb:BINARY' reads bytes as is
     def initialize(src_csv, csv_options = {}, mode = 'rb')
       @src_csv = CSVUtils::CSVWrapper.new(src_csv, mode, csv_options)
     end
 
+    # Yields each row from the start of the file.
+    # @param headers [Array<String>, nil] headers to use; by default the first row is read as the headers
+    # @yieldparam row [RowWrapper]
+    # @return [Enumerator, nil] an enumerator without a block
     def each(headers = nil)
+      return enum_for(:each, headers) unless block_given?
+
       @src_csv.rewind
 
       lineno = 0
       unless headers
-        headers = @src_csv.shift
-        strip_bom!(headers[0])
+        headers = read_headers
         lineno += 1
       end
 
@@ -54,13 +71,19 @@ module CSVUtils
       end
     end
 
+    # The first row, without the byte order mark.
+    # @return [Array<String>] empty for an empty file
     def headers
       @src_csv.rewind
-      headers = @src_csv.shift
-      strip_bom!(headers[0])
-      headers
+      read_headers
     end
 
+    # Builds a lookup hash from every row.
+    # @param key [String] header whose value is the hash key
+    # @param value [String, nil] header whose value is the hash value
+    # @yieldparam row [RowWrapper] used instead of value to compute the hash value
+    # @return [Hash]
+    # @raise [RuntimeError] when key or value isn't a header
     def to_hash(key, value = nil, &)
       raise("header #{key} not found in #{headers}") unless headers.include?(key)
       raise("headers #{value} not found in #{headers}") if value && !headers.include?(value)
@@ -72,11 +95,13 @@ module CSVUtils
           proc(&)
         end
 
-      each_with_object({}) do |row, hsh|
-        hsh[row[key]] = value_proc.call(row)
+      to_h do |row|
+        [row[key], value_proc.call(row)]
       end
     end
 
+    # Number of rows after the header row.
+    # @return [Integer]
     def size
       @src_csv.rewind
       @src_csv.shift
@@ -85,6 +110,10 @@ module CSVUtils
       cnt
     end
 
+    # Yields the rows in batches.
+    # @param batch_size [Integer] rows per batch; the last batch may be smaller
+    # @yieldparam batch [Array<RowWrapper>]
+    # @return [nil]
     def each_batch(batch_size = 1_000)
       batch = []
 
@@ -105,13 +134,11 @@ module CSVUtils
 
     private
 
-    def strip_bom!(col)
-      BYTE_ORDER_MARKS.each do |bom|
-        if col.start_with?(bom)
-          col.sub!(bom, '')
-          break
-        end
-      end
+    # an empty file has no headers, and an empty first header cell is nil
+    def read_headers
+      headers = @src_csv.shift || []
+      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
+      headers
     end
   end
 end

data/lib/csv_utils/csv_options.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
-# Auto detect a csv files options
 module CSVUtils
+  # Detects the separators, byte order mark, encoding and column count of a CSV file from its first line.
+  #
+  # @example
+  #   options = CSVUtils::CSVOptions.new('data.csv')
+  #   CSV.open('data.csv', 'rb', col_sep: options.col_separator) if options.valid?
   class CSVOptions
-    # this list is from https://en.wikipedia.org/wiki/Byte_order_mark
-    BYTE_ORDER_MARKS = {
-      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT') => 'UTF-8',
-      (+"\xFE\xFF").force_encoding('ASCII-8BIT') => 'UTF-16',
-      (+"\xFF\xFE").force_encoding('ASCII-8BIT') => 'UTF-16',
-      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT') => 'UTF-32',
-      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT') => 'UTF-32'
-    }.freeze
+    # @return [Hash{String => String}] byte order marks, as binary strings, to the encoding they indicate
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS
 
+    # @return [Array<String>] column separators to look for, the first one found in the line wins
     COL_SEPARATORS = [
       "\x02",
       "\t",
@@ -19,25 +18,27 @@ module CSVUtils
       ','
     ].freeze
 
+    # @return [Array<String>] row separators to look for, the first one found in the line wins
     ROW_SEPARATORS = [
       "\r\n",
       "\n",
       "\r"
     ].freeze
 
-    attr_reader :columns,
-                :byte_order_mark,
-                :encoding,
-                :col_separator,
-                :row_separator
+    # @return [Integer, nil] number of headers, nil without a column separator
+    attr_reader :columns
+    # @return [String, nil] the byte order mark the file starts with, as a binary string
+    attr_reader :byte_order_mark
+    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32', from the byte order mark; 'UTF-8' without one
+    attr_reader :encoding
+    # @return [String, nil] one of {COL_SEPARATORS}
+    attr_reader :col_separator
+    # @return [String, nil] one of {ROW_SEPARATORS}
+    attr_reader :row_separator
 
+    # @param io [String, IO] path of the file, or an IO positioned at its first line
     def initialize(io)
-      line =
-        if io.is_a?(String)
-          File.open(io, 'rb', &:readline)
-        else
-          io.readline
-        end
+      line = read_first_line(io)
 
       @col_separator = auto_detect_col_sep(line)
       @row_separator = auto_detect_row_sep(line)
@@ -46,42 +47,79 @@ module CSVUtils
       @columns = get_number_of_columns(line) if @col_separator
     end
 
+    # Whether both separators were found. An empty file, or a single line without a newline, isn't valid.
+    # @return [Boolean]
     def valid?
       return false if @col_separator.nil? || @row_separator.nil?
 
       true
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_col_sep(line)
       COL_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_row_sep(line)
       ROW_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # Parses the header line like a CSV row, so quoted headers can contain the separator,
+    # and falls back to a plain split when the line is malformed.
+    # @param line [String] the first line of the file
+    # @return [Array<String, nil>] headers without the byte order mark or row separator
     def get_headers(line)
-      headers = line.split(col_separator)
-      headers[0] = strip_byte_order_marks(headers[0])
-      headers
+      line = strip_byte_order_marks(line)
+      CSV.parse_line(line, **header_parse_options) || []
+    rescue CSV::MalformedCSVError
+      line.chomp(row_separator.to_s).split(col_separator, -1)
     end
 
+    # @api private
+    # @param line [String]
+    # @return [Integer]
     def get_number_of_columns(line)
       get_headers(line).size
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def get_byte_order_mark(line)
-      BYTE_ORDER_MARKS.keys.detect do |bom|
-        line =~ /\A#{bom}/
-      end
+      ByteOrderMark.detect(line)
     end
 
+    # @api private
+    # @param bom [String, nil]
+    # @return [String]
     def get_character_encoding(bom)
       BYTE_ORDER_MARKS[bom] || 'UTF-8'
     end
 
+    # @api private
+    # @param header [String]
+    # @return [String]
     def strip_byte_order_marks(header)
-      @byte_order_mark ? header.sub(@byte_order_mark, '') : header
+      ByteOrderMark.strip(header)
+    end
+
+    private
+
+    # an empty file reads as an empty line, which makes the options invalid
+    def read_first_line(io)
+      line = io.is_a?(String) ? File.open(io, 'rb', &:gets) : io.gets
+      line || ''
+    end
+
+    def header_parse_options
+      options = { col_sep: col_separator }
+      options[:row_sep] = row_separator if row_separator
+      options
     end
   end
 end

data/lib/csv_utils/csv_report.rb

@@ -1,11 +1,22 @@
 # frozen_string_literal: true
 
-# Builds a csv file from csv rows
 module CSVUtils
+  # Writes a CSV file from arrays or {CSVRow} objects.
+  #
+  # @example
+  #   CSVUtils::CSVReport.new('users.csv', UserCSVRow) do |report|
+  #     users.each { |user| report << user }
+  #   end
   class CSVReport
-    attr_reader :csv,
-                :must_close
+    # @return [CSV, #<<] where rows are written
+    attr_reader :csv
+    # @return [Boolean] whether the report opened the file and closes it after {#generate}
+    attr_reader :must_close
 
+    # @param csv [String, CSV, #<<] path to write, opened in 'wb' mode by default, or a CSV to append to
+    # @param headers [Array<String>, Class, nil] header row, or a {CSVRow} class to take the headers from
+    # @param csv_options [Hash] options passed to CSV.open for a path; :mode overrides the file mode
+    # @yieldparam report [CSVReport] when given, the block is passed to {#generate}
     def initialize(csv, headers = nil, csv_options = {}, &block)
       @csv =
         if csv.is_a?(String)
@@ -23,11 +34,18 @@ module CSVUtils
       generate(&block) if block
     end
 
+    # Yields the report, then closes the file if the report opened it, even when the block raises.
+    # @yieldparam report [CSVReport]
+    # @return [void]
     def generate
       yield self
+    ensure
       close if @must_close
     end
 
+    # Writes a row.
+    # @param csv_row [Array, #to_a] an array of values or a {CSVRow} object
+    # @return [CSV, #<<]
     def append(csv_row)
       @csv <<
         if csv_row.is_a?(Array)
@@ -38,10 +56,15 @@ module CSVUtils
     end
     alias << append
 
+    # Writes the header row.
+    # @param csv_row [Array<String>, #csv_headers] headers, or a {CSVRow} class or object to take them from
+    # @return [CSV, #<<]
     def add_headers(csv_row)
       append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
     end
 
+    # Closes the underlying CSV.
+    # @return [void]
     def close
       @csv.close
     end

data/lib/csv_utils/csv_row.rb

@@ -3,18 +3,44 @@
 require 'inheritance-helper'
 
 module CSVUtils
+  # Mixin that declares how an object turns into a CSV row, for use with {CSVReport}.
+  # Columns are inherited by subclasses.
+  #
+  # @example
+  #   class UserCSVRow < User
+  #     include CSVUtils::CSVRow
+  #
+  #     csv_column :id, header: 'ID'
+  #     csv_column :name
+  #     csv_column(:email) { email.downcase }
+  #   end
   module CSVRow
+    # @api private
+    # @param base [Class]
+    # @return [void]
     def self.included(base)
       base.extend InheritanceHelper::Methods
       base.extend ClassMethods
     end
 
+    # Class level DSL for declaring columns.
     module ClassMethods
+      # Declared columns, keyed by name, in declaration order.
+      # @return [Hash{Symbol => Hash}]
       def csv_columns
         {}
       end
 
+      # Declares a column.
+      # @param header [Symbol, String] column name; also the default header and the method that provides the value
+      # @param options [Hash]
+      # @option options [String] :header header text
+      # @option options [Symbol] :method method that provides the value
+      # @option options [Proc] :proc evaluated on the object to provide the value
+      # @yield evaluated on the object to provide the value, like :proc
+      # @return [void]
       def csv_column(header, options = {}, &block)
+        options = options.dup
         options[:header] ||= header.to_s
 
         if block
@@ -26,6 +52,7 @@ module CSVUtils
         add_value_to_class_method(:csv_columns, header => options)
       end
 
+      # @return [Array<String>] the header of each column
       def csv_headers
         csv_columns.values.map { |column_options| csv_column_header(column_options) }
       end
@@ -37,11 +64,14 @@ module CSVUtils
       end
     end
 
+    # The value of each column.
+    # @return [Array]
     def csv_row
       self.class.csv_columns.values.map { |column_options| csv_column_value(column_options) }
     end
     alias to_a csv_row
 
+    # @return [Array<String>] the header of each column
     def csv_headers
       self.class.csv_headers
     end