csv-utils 0.3.25 → 0.6.0

data/lib/csv_utils/csv_iterator.rb

@@ -1,102 +1,144 @@
-# Search a CSV given a series of steps
-class CSVUtils::CSVIterator
-  include Enumerable
-
-  attr_reader :prev_row
-
-  class RowWrapper < Hash
-    attr_accessor :lineno
+# frozen_string_literal: true
+
+module CSVUtils
+  # Enumerates the rows of a CSV file as hashes keyed by header, with the line number of each row.
+  #
+  # @example
+  #   iterator = CSVUtils::CSVIterator.new('data.csv')
+  #   iterator.each { |row| puts "#{row.lineno}: #{row['name']}" }
+  #   iterator.select { |row| row['status'] == 'active' }
+  class CSVIterator
+    include Enumerable
+
+    # @return [Array<String>] the byte order marks stripped from the first header, as binary strings
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS.keys.freeze
+
+    # @return [Array<String>, nil] the raw row read before the current one, for debugging malformed rows
+    attr_reader :prev_row
+
+    # A row as a hash of header to value that knows its line number in the file.
+    class RowWrapper < Hash
+      # @return [Integer] line number of the row in the file, counting the header row as line 1
+      attr_accessor :lineno
+
+      # @param headers [Array<String>]
+      # @param row [Array<String>]
+      # @param lineno [Integer]
+      # @return [RowWrapper]
+      def self.create(headers, row, lineno)
+        row_wrapper = RowWrapper[headers.zip(row)]
+        row_wrapper.lineno = lineno
+        row_wrapper
+      end
 
-    def self.create(headers, row, lineno)
-      row_wrapper = RowWrapper[headers.zip(row)]
-      row_wrapper.lineno = lineno
-      row_wrapper
+      # Numbered "header: value" lines for the non-blank values, as csv-grep prints them.
+      # @return [String]
+      def to_pretty_s
+        reject { |_, v| v.nil? || v.strip.empty? }
+          .each_with_index
+          .map { |(k, v), idx| format('  %-3d %s: %s', idx + 1, k, v) }
+          .join("\n") + "\n"
+      end
     end
 
-    def to_pretty_s
-      reject { |_, v| v.nil? || v.strip.empty? }
-        .each_with_index
-        .map { |(k, v), idx| sprintf('  %-3d %s: %s', idx+1, k, v) }
-        .join("\n") + "\n"
+    # @param src_csv [String, CSV] path or CSV to read; it must support rewind
+    # @param csv_options [Hash] options passed to CSV.open for a path
+    # @param mode [String] file mode for a path; 'rb:BINARY' reads bytes as is
+    def initialize(src_csv, csv_options = {}, mode = 'rb')
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, mode, csv_options)
     end
-  end
 
-  def initialize(src_csv, csv_options = {}, mode = 'rb')
-    @src_csv = CSVUtils::CSVWrapper.new(src_csv, mode, csv_options)
-  end
+    # Yields each row from the start of the file.
+    # @param headers [Array<String>, nil] headers to use; by default the first row is read as the headers
+    # @yieldparam row [RowWrapper]
+    # @return [Enumerator, nil] an enumerator without a block
+    def each(headers = nil)
+      return enum_for(:each, headers) unless block_given?
 
-  def each(headers = nil)
-    @src_csv.rewind
+      @src_csv.rewind
 
-    lineno = 0
-    unless headers
-      headers = @src_csv.shift
-      strip_bom!(headers[0])
-      lineno += 1
-    end
+      lineno = 0
+      unless headers
+        headers = read_headers
+        lineno += 1
+      end
 
-    @prev_row = nil
-    while (row = @src_csv.shift)
-      lineno += 1
-      yield RowWrapper.create(headers, row, lineno)
-      @prev_row = row
+      @prev_row = nil
+      while (row = @src_csv.shift)
+        lineno += 1
+        yield RowWrapper.create(headers, row, lineno)
+        @prev_row = row
+      end
     end
-  end
 
-  def headers
-    @src_csv.rewind
-    headers = @src_csv.shift
-    strip_bom!(headers[0])
-    headers
-  end
-
-  def to_hash(key, value = nil)
-    raise("header #{key} not found in #{headers}") unless headers.include?(key)
-    raise("headers #{value} not found in #{headers}") if value && !headers.include?(value)
+    # The first row, without the byte order mark.
+    # @return [Array<String>] empty for an empty file
+    def headers
+      @src_csv.rewind
+      read_headers
+    end
 
-    value_proc =
-      if value
-        proc { |row| row[value] }
-      else
-        proc { |row| yield(row) }
+    # Builds a lookup hash from every row.
+    # @param key [String] header whose value is the hash key
+    # @param value [String, nil] header whose value is the hash value
+    # @yieldparam row [RowWrapper] used instead of value to compute the hash value
+    # @return [Hash]
+    # @raise [RuntimeError] when key or value isn't a header
+    def to_hash(key, value = nil, &)
+      raise("header #{key} not found in #{headers}") unless headers.include?(key)
+      raise("headers #{value} not found in #{headers}") if value && !headers.include?(value)
+
+      value_proc =
+        if value
+          proc { |row| row[value] }
+        else
+          proc(&)
+        end
+
+      to_h do |row|
+        [row[key], value_proc.call(row)]
       end
-
-    each_with_object({}) do |row, hsh|
-      hsh[row[key]] = value_proc.call(row)
     end
-  end
 
-  def size
-    @src_csv.rewind
-    @src_csv.shift
-    cnt = 0
-    while @src_csv.shift
-      cnt +=1
+    # Number of rows after the header row.
+    # @return [Integer]
+    def size
+      @src_csv.rewind
+      @src_csv.shift
+      cnt = 0
+      cnt += 1 while @src_csv.shift
+      cnt
     end
-    cnt
-  end
 
-  def each_batch(batch_size = 1_000)
-    batch = []
-
-    process_batch_proc = Proc.new do
-      yield batch
+    # Yields the rows in batches.
+    # @param batch_size [Integer] rows per batch; the last batch may be smaller
+    # @yieldparam batch [Array<RowWrapper>]
+    # @return [nil]
+    def each_batch(batch_size = 1_000)
       batch = []
-    end
 
-    each do |row|
-      batch << row
-      process_batch_proc.call if batch.size >= batch_size
-    end
+      process_batch_proc = proc do
+        yield batch
+        batch = []
+      end
 
-    process_batch_proc.call if batch.size > 0
+      each do |row|
+        batch << row
+        process_batch_proc.call if batch.size >= batch_size
+      end
 
-    nil
-  end
+      process_batch_proc.call if batch.size.positive?
 
-  private
+      nil
+    end
+
+    private
 
-  def strip_bom!(col)
-    col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+    # an empty file has no headers, and an empty first header cell is nil
+    def read_headers
+      headers = @src_csv.shift || []
+      headers[0] = ByteOrderMark.strip(headers[0]) if headers[0]
+      headers
+    end
   end
 end

data/lib/csv_utils/csv_options.rb

@@ -1,43 +1,44 @@
-# Auto detect a csv files options
+# frozen_string_literal: true
+
 module CSVUtils
+  # Detects the separators, byte order mark, encoding and column count of a CSV file from its first line.
+  #
+  # @example
+  #   options = CSVUtils::CSVOptions.new('data.csv')
+  #   CSV.open('data.csv', 'rb', col_sep: options.col_separator) if options.valid?
   class CSVOptions
+    # @return [Hash{String => String}] byte order marks, as binary strings, to the encoding they indicate
+    BYTE_ORDER_MARKS = ByteOrderMark::ENCODINGS
 
-    # this list is from https://en.wikipedia.org/wiki/Byte_order_mark
-    BYTE_ORDER_MARKS = {
-      "\xEF\xBB\xBF".force_encoding('ASCII-8BIT') => 'UTF-8',
-      "\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-16',
-      "\xFF\xFE".force_encoding('ASCII-8BIT') => 'UTF-16',
-      "\x00\x00\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-32',
-      "\xFF\xFE\x00\x00".force_encoding('ASCII-8BIT') => 'UTF-32'
-    }
-
+    # @return [Array<String>] column separators to look for, the first one found in the line wins
     COL_SEPARATORS = [
       "\x02",
       "\t",
       '|',
       ','
-    ]
+    ].freeze
 
+    # @return [Array<String>] row separators to look for, the first one found in the line wins
     ROW_SEPARATORS = [
       "\r\n",
       "\n",
       "\r"
-    ]
+    ].freeze
 
-    attr_reader :columns,
-                :byte_order_mark,
-                :encoding,
-                :col_separator,
-                :row_separator
-    
+    # @return [Integer, nil] number of headers, nil without a column separator
+    attr_reader :columns
+    # @return [String, nil] the byte order mark the file starts with, as a binary string
+    attr_reader :byte_order_mark
+    # @return [String] 'UTF-8', 'UTF-16' or 'UTF-32', from the byte order mark; 'UTF-8' without one
+    attr_reader :encoding
+    # @return [String, nil] one of {COL_SEPARATORS}
+    attr_reader :col_separator
+    # @return [String, nil] one of {ROW_SEPARATORS}
+    attr_reader :row_separator
 
+    # @param io [String, IO] path of the file, or an IO positioned at its first line
     def initialize(io)
-      line =
-        if io.is_a?(String)
-          File.open(io, 'rb', &:readline)
-        else
-          io.readline
-        end
+      line = read_first_line(io)
 
       @col_separator = auto_detect_col_sep(line)
       @row_separator = auto_detect_row_sep(line)
@@ -46,42 +47,79 @@ module CSVUtils
       @columns = get_number_of_columns(line) if @col_separator
     end
 
+    # Whether both separators were found. An empty file, or a single line without a newline, isn't valid.
+    # @return [Boolean]
     def valid?
       return false if @col_separator.nil? || @row_separator.nil?
 
       true
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_col_sep(line)
       COL_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def auto_detect_row_sep(line)
       ROW_SEPARATORS.detect { |sep| line.include?(sep) }
     end
 
+    # Parses the header line like a CSV row, so quoted headers can contain the separator,
+    # and falls back to a plain split when the line is malformed.
+    # @param line [String] the first line of the file
+    # @return [Array<String, nil>] headers without the byte order mark or row separator
     def get_headers(line)
-      headers = line.split(col_separator)
-      headers[0] = strip_byte_order_marks(headers[0])
-      headers
+      line = strip_byte_order_marks(line)
+      CSV.parse_line(line, **header_parse_options) || []
+    rescue CSV::MalformedCSVError
+      line.chomp(row_separator.to_s).split(col_separator, -1)
     end
 
+    # @api private
+    # @param line [String]
+    # @return [Integer]
     def get_number_of_columns(line)
       get_headers(line).size
     end
 
+    # @api private
+    # @param line [String]
+    # @return [String, nil]
     def get_byte_order_mark(line)
-      BYTE_ORDER_MARKS.keys.detect do |bom|
-        line =~ /\A#{bom}/
-      end
+      ByteOrderMark.detect(line)
     end
 
+    # @api private
+    # @param bom [String, nil]
+    # @return [String]
     def get_character_encoding(bom)
       BYTE_ORDER_MARKS[bom] || 'UTF-8'
     end
 
+    # @api private
+    # @param header [String]
+    # @return [String]
     def strip_byte_order_marks(header)
-      @byte_order_marks ? header.sub(@byte_order_marks, '') : header
+      ByteOrderMark.strip(header)
+    end
+
+    private
+
+    # an empty file reads as an empty line, which makes the options invalid
+    def read_first_line(io)
+      line = io.is_a?(String) ? File.open(io, 'rb', &:gets) : io.gets
+      line || ''
+    end
+
+    def header_parse_options
+      options = { col_sep: col_separator }
+      options[:row_sep] = row_separator if row_separator
+      options
     end
   end
 end

data/lib/csv_utils/csv_report.rb

@@ -1,15 +1,29 @@
-# Builds a csv file from csv rows
+# frozen_string_literal: true
+
 module CSVUtils
+  # Writes a CSV file from arrays or {CSVRow} objects.
+  #
+  # @example
+  #   CSVUtils::CSVReport.new('users.csv', UserCSVRow) do |report|
+  #     users.each { |user| report << user }
+  #   end
   class CSVReport
-    attr_reader :csv,
-                :must_close
+    # @return [CSV, #<<] where rows are written
+    attr_reader :csv
+    # @return [Boolean] whether the report opened the file and closes it after {#generate}
+    attr_reader :must_close
 
+    # @param csv [String, CSV, #<<] path to write, opened in 'wb' mode by default, or a CSV to append to
+    # @param headers [Array<String>, Class, nil] header row, or a {CSVRow} class to take the headers from
+    # @param csv_options [Hash] options passed to CSV.open for a path; :mode overrides the file mode
+    # @yieldparam report [CSVReport] when given, the block is passed to {#generate}
     def initialize(csv, headers = nil, csv_options = {}, &block)
       @csv =
         if csv.is_a?(String)
           @must_close = true
-          mode = csv_options.delete(:mode) || 'wb'
-          CSV.open(csv, mode, **csv_options)
+          opts = csv_options.dup
+          mode = opts.delete(:mode) || 'wb'
+          CSV.open(csv, mode, **opts)
         else
           @must_close = false
           csv
@@ -20,11 +34,18 @@ module CSVUtils
       generate(&block) if block
     end
 
+    # Yields the report, then closes the file if the report opened it, even when the block raises.
+    # @yieldparam report [CSVReport]
+    # @return [void]
     def generate
       yield self
+    ensure
       close if @must_close
     end
 
+    # Writes a row.
+    # @param csv_row [Array, #to_a] an array of values or a {CSVRow} object
+    # @return [CSV, #<<]
     def append(csv_row)
       @csv <<
         if csv_row.is_a?(Array)
@@ -35,10 +56,15 @@ module CSVUtils
     end
     alias << append
 
+    # Writes the header row.
+    # @param csv_row [Array<String>, #csv_headers] headers, or a {CSVRow} class or object to take them from
+    # @return [CSV, #<<]
     def add_headers(csv_row)
       append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
     end
 
+    # Closes the underlying CSV.
+    # @return [void]
     def close
       @csv.close
     end

data/lib/csv_utils/csv_row.rb

@@ -1,18 +1,46 @@
+# frozen_string_literal: true
+
 require 'inheritance-helper'
 
 module CSVUtils
+  # Mixin that declares how an object turns into a CSV row, for use with {CSVReport}.
+  # Columns are inherited by subclasses.
+  #
+  # @example
+  #   class UserCSVRow < User
+  #     include CSVUtils::CSVRow
+  #
+  #     csv_column :id, header: 'ID'
+  #     csv_column :name
+  #     csv_column(:email) { email.downcase }
+  #   end
   module CSVRow
+    # @api private
+    # @param base [Class]
+    # @return [void]
     def self.included(base)
       base.extend InheritanceHelper::Methods
       base.extend ClassMethods
     end
 
+    # Class level DSL for declaring columns.
     module ClassMethods
+      # Declared columns, keyed by name, in declaration order.
+      # @return [Hash{Symbol => Hash}]
       def csv_columns
         {}
       end
 
+      # Declares a column.
+      # @param header [Symbol, String] column name; also the default header and the method that provides the value
+      # @param options [Hash]
+      # @option options [String] :header header text
+      # @option options [Symbol] :method method that provides the value
+      # @option options [Proc] :proc evaluated on the object to provide the value
+      # @yield evaluated on the object to provide the value, like :proc
+      # @return [void]
       def csv_column(header, options = {}, &block)
+        options = options.dup
         options[:header] ||= header.to_s
 
         if block
@@ -24,6 +52,7 @@ module CSVUtils
         add_value_to_class_method(:csv_columns, header => options)
       end
 
+      # @return [Array<String>] the header of each column
       def csv_headers
         csv_columns.values.map { |column_options| csv_column_header(column_options) }
       end
@@ -35,11 +64,14 @@ module CSVUtils
       end
     end
 
+    # The value of each column.
+    # @return [Array]
     def csv_row
       self.class.csv_columns.values.map { |column_options| csv_column_value(column_options) }
     end
-    alias_method :to_a, :csv_row
+    alias to_a csv_row
 
+    # @return [Array<String>] the header of each column
     def csv_headers
       self.class.csv_headers
     end

data/lib/csv_utils/csv_row_matcher.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Matches rows (hashes of header to value) whose values match a regular expression.
+  #
+  # @example
+  #   matcher = CSVUtils::CSVRowMatcher.new(/error/i, ['status', 'message'])
+  #   CSVUtils::CSVIterator.new('logs.csv').select(&matcher)
+  class CSVRowMatcher
+    # @return [Regexp]
+    attr_accessor :regex
+    # @return [Array<String>, :all] headers to search, or :all for every value
+    attr_accessor :columns
+
+    # @param regex [Regexp]
+    # @param columns [Array<String>, :all] headers to search
+    def initialize(regex, columns = :all)
+      self.regex = regex
+      self.columns = columns
+    end
+
+    # Whether any searched value matches. Missing and nil values never match.
+    # @param row [Hash{String => String}]
+    # @return [Boolean]
+    def match?(row)
+      if columns == :all
+        row.each_value do |value|
+          return true if value&.match?(regex)
+        end
+      else
+        columns.each do |column_name|
+          value = row[column_name]
+          return true if value&.match?(regex)
+        end
+      end
+
+      false
+    end
+
+    # Lets the matcher be passed to Enumerable methods as a block, as in rows.select(&matcher).
+    # @return [Proc]
+    def to_proc
+      proc do |row|
+        match?(row)
+      end
+    end
+  end
+end