csv-utils 0.5.0 → 0.7.0

data/lib/csv_utils/csv_options.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
-# Auto detect a csv files options
 module CSVUtils
+  # Detects the separators, byte order mark, encoding and column count of a CSV file from its first line.
+  #
+  # @example
+  #   options = CSVUtils::CSVOptions.new('data.csv')
+  #   CSV.open('data.csv', 'rb', col_sep: options.col_separator) if options.valid?
   class CSVOptions
-    # this list is from https://en.wikipedia.org/wiki/Byte_order_mark
-    BYTE_ORDER_MARKS = {
-      (+"\xEF\xBB\xBF").force_encoding('ASCII-8BIT') => 'UTF-8',
-      (+"\xFE\xFF").force_encoding('ASCII-8BIT') => 'UTF-16',
-      (+"\xFF\xFE").force_encoding('ASCII-8BIT') => 'UTF-16',
-      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT') => 'UTF-32',
-      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT') => 'UTF-32'
-    }.freeze
+    # @return [Hash{String => String}] byte order marks, as binary strings, to the encoding they indicate
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS
 
+    # @return [Array<String>] column separators to look for, the first one found in the line wins
     COL_SEPARATORS = [
       "\x02",
       "\t",
@@ -19,69 +18,157 @@ module CSVUtils
       ','
     ].freeze
 
+    # @return [Array<String>] row separators to look for, the first one found in the line wins
     ROW_SEPARATORS = [
       "\r\n",
       "\n",
       "\r"
     ].freeze
 
-    attr_reader :columns,
-                :byte_order_mark,
-                :encoding,
-                :col_separator,
-                :row_separator
+    # @return [Hash{String => String}] UTF-16 and UTF-32 byte order marks, as binary strings, to the :encoding
+    #   option that decodes them to UTF-8
+    WIDE_ENCODINGS = {
+      (+"\x00\x00\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-32BE:UTF-8',
+      (+"\xFF\xFE\x00\x00").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-32LE:UTF-8',
+      (+"\xFE\xFF").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-16BE:UTF-8',
+      (+"\xFF\xFE").force_encoding('ASCII-8BIT').freeze => 'BOM|UTF-16LE:UTF-8'
+    }.freeze
+
+    # @return [Integer] bytes read to detect the encoding of a file without a byte order mark
+    SAMPLE_SIZE = 1024 * 1024
 
+    # @return [Integer, nil] number of headers, nil without a column separator
+    attr_reader :columns
+    # @return [String, nil] the byte order mark the file starts with, as a binary string
+    attr_reader :byte_order_mark
+    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32' from the byte order mark; without one, from the bytes of
+    #   the first {SAMPLE_SIZE} bytes: 'UTF-8', 'Windows-1252' or 'ISO-8859-1' (see {CharacterEncoding.detect})
+    attr_reader :encoding
+    # @return [String, nil] one of {COL_SEPARATORS}
+    attr_reader :col_separator
+    # @return [String, nil] one of {ROW_SEPARATORS}
+    attr_reader :row_separator
+
+    # @param io [String, IO] path of the file, or an IO positioned at its first line; up to {SAMPLE_SIZE}
+    #   bytes are read from it
     def initialize(io)
-      line =
-        if io.is_a?(String)
-          File.open(io, 'rb', &:readline)
-        else
-          io.readline
-        end
+      sample = read_sample(io)
+      line = first_line(sample)
 
       @col_separator = auto_detect_col_sep(line)
       @row_separator = auto_detect_row_sep(line)
       @byte_order_mark = get_byte_order_mark(line)
-      @encoding = get_character_encoding(@byte_order_mark)
+      @encoding = get_character_encoding(@byte_order_mark, sample)
       @columns = get_number_of_columns(line) if @col_separator
     end
 
+    # Whether both separators were found. An empty file, or a single line without a newline, isn't valid.
+    # @return [Boolean]
     def valid?
       return false if @col_separator.nil? || @row_separator.nil?
 
       true
     end
 
+    # Options for CSV.open, or any class in this library, with the detected separators and an :encoding that
+    # reads values as UTF-8 strings: 'bom|utf-8', 'Windows-1252:UTF-8', 'BOM|UTF-16LE:UTF-8' and so on.
+    # A separator that wasn't detected is left to CSV, and so is the row separator of UTF-16 and UTF-32 files,
+    # since it's detected from raw bytes.
+    # @example
+    #   CSV.open(path, 'rb', **options.to_csv_options)
+    #   CSVUtils::CSVSort.new(path, 'sorted.csv', true, options.to_csv_options).sort
+    # @return [Hash{Symbol => String}]
+    def to_csv_options
+      options = { encoding: encoding_option }
+      options[:col_sep] = col_separator if col_separator
+      options[:row_sep] = row_separator if row_separator && !wide_encoding?
+      options
+    end
+
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_col_sep(line)
       COL_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_row_sep(line)
       ROW_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # Parses the header line like a CSV row, so quoted headers can contain the separator,
+    # and falls back to a plain split when the line is malformed.
+    # @param line [String] the first line of the file
+    # @return [Array<String, nil>] headers without the byte order mark or row separator
     def get_headers(line)
-      headers = line.split(col_separator)
-      headers[0] = strip_byte_order_marks(headers[0])
-      headers
+      line = strip_byte_order_marks(line)
+      CSV.parse_line(line, **header_parse_options) || []
+    rescue CSV::MalformedCSVError
+      line.chomp(row_separator.to_s).split(col_separator, -1)
     end
 
+    # @api private
+    # @param line [String]
+    # @return [Integer]
     def get_number_of_columns(line)
       get_headers(line).size
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def get_byte_order_mark(line)
-      BYTE_ORDER_MARKS.keys.detect do |bom|
-        line =~ /\A#{bom}/
-      end
+      ByteOrderMark.detect(line)
     end
 
-    def get_character_encoding(bom)
-      BYTE_ORDER_MARKS[bom] || 'UTF-8'
+    # @api private
+    # @param bom [String, nil]
+    # @param sample [String] bytes from the start of the file
+    # @return [String]
+    def get_character_encoding(bom, sample)
+      BYTE_ORDER_MARKS[bom] || CharacterEncoding.detect(sample)
     end
 
+    # @api private
+    # @param header [String]
+    # @return [String]
     def strip_byte_order_marks(header)
-      @byte_order_mark ? header.sub(@byte_order_mark, '') : header
+      ByteOrderMark.strip(header)
+    end
+
+    private
+
+    # an empty file reads as an empty sample, which makes the options invalid
+    def read_sample(io)
+      sample = (io.is_a?(String) ? File.open(io, 'rb') { |file| file.read(SAMPLE_SIZE) } : io.read(SAMPLE_SIZE)) || ''
+      sample.bytesize < SAMPLE_SIZE ? sample : drop_last_multibyte_char(sample)
+    end
+
+    # a full sample can end mid-character, so its last multibyte character, whole or not, is dropped
+    def drop_last_multibyte_char(sample)
+      sample.sub(/[\xC0-\xFF][\x80-\xBF]*\z/n, '')
+    end
+
+    def first_line(sample)
+      line_end = sample.index("\n")
+      line_end ? sample.byteslice(0, line_end + 1) : sample
+    end
+
+    def wide_encoding?
+      WIDE_ENCODINGS.key?(byte_order_mark)
+    end
+
+    def encoding_option
+      WIDE_ENCODINGS.fetch(byte_order_mark) { encoding == 'UTF-8' ? EncodingOptions::DEFAULT_ENCODING : "#{encoding}:UTF-8" }
+    end
+
+    def header_parse_options
+      options = { col_sep: col_separator }
+      options[:row_sep] = row_separator if row_separator
+      options
     end
   end
 end

data/lib/csv_utils/csv_report.rb

@@ -1,16 +1,28 @@
 # frozen_string_literal: true
 
-# Builds a csv file from csv rows
 module CSVUtils
+  # Writes a CSV file from arrays or {CSVRow} objects.
+  #
+  # @example
+  #   CSVUtils::CSVReport.new('users.csv', UserCSVRow) do |report|
+  #     users.each { |user| report << user }
+  #   end
   class CSVReport
-    attr_reader :csv,
-                :must_close
+    # @return [CSV, #<<] where rows are written
+    attr_reader :csv
+    # @return [Boolean] whether the report opened the file and closes it after {#generate}
+    attr_reader :must_close
 
+    # @param csv [String, CSV, #<<] path to write, opened in 'wb' mode by default, or a CSV to append to
+    # @param headers [Array<String>, Class, nil] header row, or a {CSVRow} class to take the headers from
+    # @param csv_options [Hash] options passed to CSV.open for a path; :mode overrides the file mode and
+    #   :encoding, UTF-8 by default, is the encoding of the file written
+    # @yieldparam report [CSVReport] when given, the block is passed to {#generate}
     def initialize(csv, headers = nil, csv_options = {}, &block)
       @csv =
         if csv.is_a?(String)
           @must_close = true
-          opts = csv_options.dup
+          opts = EncodingOptions.write(csv_options)
           mode = opts.delete(:mode) || 'wb'
           CSV.open(csv, mode, **opts)
         else
@@ -23,11 +35,18 @@ module CSVUtils
       generate(&block) if block
     end
 
+    # Yields the report, then closes the file if the report opened it, even when the block raises.
+    # @yieldparam report [CSVReport]
+    # @return [void]
     def generate
       yield self
+    ensure
       close if @must_close
     end
 
+    # Writes a row.
+    # @param csv_row [Array, #to_a] an array of values or a {CSVRow} object
+    # @return [CSV, #<<]
     def append(csv_row)
       @csv <<
         if csv_row.is_a?(Array)
@@ -38,10 +57,15 @@ module CSVUtils
     end
     alias << append
 
+    # Writes the header row.
+    # @param csv_row [Array<String>, #csv_headers] headers, or a {CSVRow} class or object to take them from
+    # @return [CSV, #<<]
     def add_headers(csv_row)
       append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
     end
 
+    # Closes the underlying CSV.
+    # @return [void]
     def close
       @csv.close
     end

data/lib/csv_utils/csv_row.rb

@@ -3,18 +3,44 @@
 require 'inheritance-helper'
 
 module CSVUtils
+  # Mixin that declares how an object turns into a CSV row, for use with {CSVReport}.
+  # Columns are inherited by subclasses.
+  #
+  # @example
+  #   class UserCSVRow < User
+  #     include CSVUtils::CSVRow
+  #
+  #     csv_column :id, header: 'ID'
+  #     csv_column :name
+  #     csv_column(:email) { email.downcase }
+  #   end
   module CSVRow
+    # @api private
+    # @param base [Class]
+    # @return [void]
     def self.included(base)
       base.extend InheritanceHelper::Methods
       base.extend ClassMethods
     end
 
+    # Class level DSL for declaring columns.
     module ClassMethods
+      # Declared columns, keyed by name, in declaration order.
+      # @return [Hash{Symbol => Hash}]
       def csv_columns
         {}
       end
 
+      # Declares a column.
+      # @param header [Symbol, String] column name; also the default header and the method that provides the value
+      # @param options [Hash]
+      # @option options [String] :header header text
+      # @option options [Symbol] :method method that provides the value
+      # @option options [Proc] :proc evaluated on the object to provide the value
+      # @yield evaluated on the object to provide the value, like :proc
+      # @return [void]
       def csv_column(header, options = {}, &block)
+        options = options.dup
         options[:header] ||= header.to_s
 
         if block
@@ -26,6 +52,7 @@ module CSVUtils
         add_value_to_class_method(:csv_columns, header => options)
       end
 
+      # @return [Array<String>] the header of each column
       def csv_headers
         csv_columns.values.map { |column_options| csv_column_header(column_options) }
       end
@@ -37,11 +64,14 @@ module CSVUtils
       end
     end
 
+    # The value of each column.
+    # @return [Array]
     def csv_row
       self.class.csv_columns.values.map { |column_options| csv_column_value(column_options) }
     end
     alias to_a csv_row
 
+    # @return [Array<String>] the header of each column
     def csv_headers
       self.class.csv_headers
     end

data/lib/csv_utils/csv_row_matcher.rb

@@ -1,15 +1,27 @@
 # frozen_string_literal: true
 
 module CSVUtils
+  # Matches rows (hashes of header to value) whose values match a regular expression.
+  #
+  # @example
+  #   matcher = CSVUtils::CSVRowMatcher.new(/error/i, ['status', 'message'])
+  #   CSVUtils::CSVIterator.new('logs.csv').select(&matcher)
   class CSVRowMatcher
-    attr_accessor :regex,
-                  :columns
+    # @return [Regexp]
+    attr_accessor :regex
+    # @return [Array<String>, :all] headers to search, or :all for every value
+    attr_accessor :columns
 
+    # @param regex [Regexp]
+    # @param columns [Array<String>, :all] headers to search
     def initialize(regex, columns = :all)
       self.regex = regex
       self.columns = columns
     end
 
+    # Whether any searched value matches. Missing and nil values never match.
+    # @param row [Hash{String => String}]
+    # @return [Boolean]
     def match?(row)
       if columns == :all
         row.each_value do |value|
@@ -25,6 +37,8 @@ module CSVUtils
       false
     end
 
+    # Lets the matcher be passed to Enumerable methods as a block, as in rows.select(&matcher).
+    # @return [Proc]
     def to_proc
       proc do |row|
         match?(row)

data/lib/csv_utils/csv_sort.rb

@@ -2,38 +2,62 @@
 
 require 'fileutils'
 
-# Utility class for sorting the rows for a csv file
 module CSVUtils
+  # Sorts a CSV file too large to load into memory with an external merge sort: batches of rows are
+  # sorted into temporary part files next to the new file, which are then merged in pairs.
+  #
+  # @example
+  #   CSVUtils::CSVSort.new('input.csv', 'sorted.csv').sort { |a, b| a[0].to_i <=> b[0].to_i }
   class CSVSort
-    attr_reader :csv_file,
-                :new_csv_file,
-                :has_headers,
-                :csv_options,
-                :headers
-
+    # @return [String] path of the file to sort
+    attr_reader :csv_file
+    # @return [String] path of the sorted file to write
+    attr_reader :new_csv_file
+    # @return [Boolean] whether the first row is a header row, kept at the top
+    attr_reader :has_headers
+    # @return [Hash] options passed to CSV.open
+    attr_reader :csv_options
+    # @return [Array<String>, nil] the header row, once {#sort} has read it
+    attr_reader :headers
+
+    # @param csv_file [String] path of the file to sort
+    # @param new_csv_file [String] path of the sorted file to write
+    # @param has_headers [Boolean] whether the first row is a header row
+    # @param csv_options [Hash] options passed to CSV.open; the file is read with its :encoding, 'bom|utf-8' by
+    #   default, and the sorted file written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(csv_file, new_csv_file, has_headers = true, csv_options = {})
       @csv_file = csv_file
       @new_csv_file = new_csv_file
       @has_headers = has_headers
       @csv_options = csv_options
       @csv_part_files = []
-      @files_to_delete = []
     end
 
-    def sort(batch_size = 100_000, &)
-      create_sorted_csv_part_files(batch_size, &)
-      merge_csv_part_files(&)
+    # Writes the sorted file. Temporary files are removed even when sorting fails.
+    # @param batch_size [Integer] rows held in memory and sorted at a time
+    # @yieldparam row1 [Array<String>]
+    # @yieldparam row2 [Array<String>]
+    # @yieldreturn [Integer] negative, 0 or positive like <=>; without a block rows are compared as arrays
+    # @return [void]
+    def sort(batch_size = 100_000, &block)
+      block ||= proc { |row1, row2| row1 <=> row2 }
+      @csv_part_files = []
+      create_sorted_csv_part_files(batch_size, &block)
+      merge_csv_part_files(&block)
+    ensure
+      delete_csv_part_files
     end
 
     private
 
-    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable-next Metrics/MethodLength
     def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
-      src1 = CSV.open(src_csv_file1, 'rb', **csv_options)
+      # part files were written with the write options, so reading them back with those changes nothing
+      src1 = CSV.open(src_csv_file1, 'rb', **write_options)
       begin
-        src2 = CSV.open(src_csv_file2, 'rb', **csv_options)
+        src2 = CSV.open(src_csv_file2, 'rb', **write_options)
         begin
-          dest = CSV.open(dest_csv_file, 'wb', **csv_options)
+          dest = CSV.open(dest_csv_file, 'wb', **write_options)
           begin
             if @headers
               dest << @headers
@@ -75,10 +99,9 @@ module CSVUtils
         src1.close
       end
     end
-    # rubocop:enable Metrics/MethodLength
 
     def create_sorted_csv_part_files(batch_size, &block)
-      src = CSV.open(csv_file, 'rb', **csv_options)
+      src = CSV.open(csv_file, 'rb', **EncodingOptions.read(csv_options))
       begin
         @headers = src.shift if has_headers
 
@@ -86,7 +109,7 @@ module CSVUtils
         create_batch_part_proc = proc do
           batch.sort!(&block)
           @csv_part_files << "#{new_csv_file}.part.#{@csv_part_files.size}"
-          CSV.open(@csv_part_files.last, 'wb', **csv_options) do |csv|
+          CSV.open(@csv_part_files.last, 'wb', **write_options) do |csv|
             csv << @headers if @headers
             batch.each { |row| csv << row }
           end
@@ -110,21 +133,37 @@ module CSVUtils
       while @csv_part_files.size > 1
         file_merge_cnt += 1
 
-        csv_part_file1 = @csv_part_files.shift
-        csv_part_file2 = @csv_part_files.shift
+        # inputs stay in the list until merged, so a failed merge still cleans them up
+        csv_part_file1, csv_part_file2 = @csv_part_files.first(2)
         @csv_part_files << "#{new_csv_file}.merge.#{file_merge_cnt}"
 
         merge_sort_csv_files(csv_part_file1, csv_part_file2, @csv_part_files.last, &)
 
-        File.unlink(csv_part_file1)
-        File.unlink(csv_part_file2)
+        @csv_part_files.shift(2).each { |file| File.unlink(file) }
       end
 
       if @csv_part_files.size.positive?
-        FileUtils.mv(@csv_part_files.last, new_csv_file)
+        FileUtils.mv(@csv_part_files.pop, new_csv_file)
       else
-        FileUtils.cp(@csv_file, new_csv_file)
+        write_headers_only
+      end
+    end
+
+    # a file without rows is written rather than copied, so it's in the same encoding as a sorted one
+    def write_headers_only
+      CSV.open(new_csv_file, 'wb', **write_options) do |csv|
+        csv << @headers if @headers
       end
     end
+
+    def write_options
+      EncodingOptions.write(csv_options)
+    end
+
+    # removes the temporary files left behind when sorting fails part way through
+    def delete_csv_part_files
+      @csv_part_files.each { |file| FileUtils.rm_f(file) }
+      @csv_part_files = []
+    end
   end
 end