csv-utils 0.5.0 → 0.6.0

data/lib/csv_utils/csv_row_matcher.rb

@@ -1,15 +1,27 @@
 # frozen_string_literal: true
 
 module CSVUtils
+  # Matches rows (hashes of header to value) whose values match a regular expression.
+  #
+  # @example
+  #   matcher = CSVUtils::CSVRowMatcher.new(/error/i, ['status', 'message'])
+  #   CSVUtils::CSVIterator.new('logs.csv').select(&matcher)
   class CSVRowMatcher
-    attr_accessor :regex,
-                  :columns
+    # @return [Regexp]
+    attr_accessor :regex
+    # @return [Array<String>, :all] headers to search, or :all for every value
+    attr_accessor :columns
 
+    # @param regex [Regexp]
+    # @param columns [Array<String>, :all] headers to search
     def initialize(regex, columns = :all)
       self.regex = regex
       self.columns = columns
     end
 
+    # Whether any searched value matches. Missing and nil values never match.
+    # @param row [Hash{String => String}]
+    # @return [Boolean]
     def match?(row)
       if columns == :all
         row.each_value do |value|
@@ -25,6 +37,8 @@ module CSVUtils
       false
     end
 
+    # Lets the matcher be passed to Enumerable methods as a block, as in rows.select(&matcher).
+    # @return [Proc]
     def to_proc
       proc do |row|
         match?(row)

data/lib/csv_utils/csv_sort.rb

@@ -2,32 +2,54 @@
 
 require 'fileutils'
 
-# Utility class for sorting the rows for a csv file
 module CSVUtils
+  # Sorts a CSV file too large to load into memory with an external merge sort: batches of rows are
+  # sorted into temporary part files next to the new file, which are then merged in pairs.
+  #
+  # @example
+  #   CSVUtils::CSVSort.new('input.csv', 'sorted.csv').sort { |a, b| a[0].to_i <=> b[0].to_i }
   class CSVSort
-    attr_reader :csv_file,
-                :new_csv_file,
-                :has_headers,
-                :csv_options,
-                :headers
-
+    # @return [String] path of the file to sort
+    attr_reader :csv_file
+    # @return [String] path of the sorted file to write
+    attr_reader :new_csv_file
+    # @return [Boolean] whether the first row is a header row, kept at the top
+    attr_reader :has_headers
+    # @return [Hash] options passed to CSV.open
+    attr_reader :csv_options
+    # @return [Array<String>, nil] the header row, once {#sort} has read it
+    attr_reader :headers
+
+    # @param csv_file [String] path of the file to sort
+    # @param new_csv_file [String] path of the sorted file to write
+    # @param has_headers [Boolean] whether the first row is a header row
+    # @param csv_options [Hash] options passed to CSV.open
     def initialize(csv_file, new_csv_file, has_headers = true, csv_options = {})
       @csv_file = csv_file
       @new_csv_file = new_csv_file
       @has_headers = has_headers
       @csv_options = csv_options
       @csv_part_files = []
-      @files_to_delete = []
     end
 
-    def sort(batch_size = 100_000, &)
-      create_sorted_csv_part_files(batch_size, &)
-      merge_csv_part_files(&)
+    # Writes the sorted file. Temporary files are removed even when sorting fails.
+    # @param batch_size [Integer] rows held in memory and sorted at a time
+    # @yieldparam row1 [Array<String>]
+    # @yieldparam row2 [Array<String>]
+    # @yieldreturn [Integer] negative, 0 or positive like <=>; without a block rows are compared as arrays
+    # @return [void]
+    def sort(batch_size = 100_000, &block)
+      block ||= proc { |row1, row2| row1 <=> row2 }
+      @csv_part_files = []
+      create_sorted_csv_part_files(batch_size, &block)
+      merge_csv_part_files(&block)
+    ensure
+      delete_csv_part_files
     end
 
     private
 
-    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable-next Metrics/MethodLength
     def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
       src1 = CSV.open(src_csv_file1, 'rb', **csv_options)
       begin
@@ -75,7 +97,6 @@ module CSVUtils
         src1.close
       end
     end
-    # rubocop:enable Metrics/MethodLength
 
     def create_sorted_csv_part_files(batch_size, &block)
       src = CSV.open(csv_file, 'rb', **csv_options)
@@ -110,21 +131,26 @@ module CSVUtils
       while @csv_part_files.size > 1
         file_merge_cnt += 1
 
-        csv_part_file1 = @csv_part_files.shift
-        csv_part_file2 = @csv_part_files.shift
+        # inputs stay in the list until merged, so a failed merge still cleans them up
+        csv_part_file1, csv_part_file2 = @csv_part_files.first(2)
         @csv_part_files << "#{new_csv_file}.merge.#{file_merge_cnt}"
 
         merge_sort_csv_files(csv_part_file1, csv_part_file2, @csv_part_files.last, &)
 
-        File.unlink(csv_part_file1)
-        File.unlink(csv_part_file2)
+        @csv_part_files.shift(2).each { |file| File.unlink(file) }
       end
 
       if @csv_part_files.size.positive?
-        FileUtils.mv(@csv_part_files.last, new_csv_file)
+        FileUtils.mv(@csv_part_files.pop, new_csv_file)
       else
         FileUtils.cp(@csv_file, new_csv_file)
       end
     end
+
+    # removes the temporary files left behind when sorting fails part way through
+    def delete_csv_part_files
+      @csv_part_files.each { |file| FileUtils.rm_f(file) }
+      @csv_part_files = []
+    end
   end
 end

data/lib/csv_utils/csv_transformer.rb

@@ -1,63 +1,106 @@
 # frozen_string_literal: true
 
-# Transforms a CSV given a series of steps
 module CSVUtils
+  # Copies a CSV file through a pipeline of steps that filter, change and add to its rows in batches.
+  # Each step method returns the transformer so they can be chained, and {#process} runs them.
+  #
+  # Step blocks receive the row, the headers as they were when the step was added, and the value
+  # computed by the last {#additional_data} step for the current batch.
+  #
+  # @example
+  #   CSVUtils::CSVTransformer.new('input.csv', 'output.csv')
+  #     .read_headers
+  #     .select { |row, _headers, _data| row[0].to_i > 100 }
+  #     .append(['Email']) { |row, _headers, _data| ["#{row[1].downcase}@example.com"] }
+  #     .process
   class CSVTransformer
+    # @return [Array<String>, nil] the headers the output will have after the steps added so far
     attr_reader :headers
 
+    # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the transformer
+    # @param dest_csv [String, CSV, #<<] path or CSV to write
+    # @param csv_options [Hash] options passed to CSV.open for paths
     def initialize(src_csv, dest_csv, csv_options = {})
       @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
       @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
     end
 
+    # Reads the first row as the headers. Without it, every row is data and the output has no header row.
+    # @return [self]
     def read_headers
       @headers = @src_csv.shift
       self
     end
 
+    # Computes a value once per batch, passed to the steps that follow as their third argument.
+    # @yieldparam batch [Array<Array<String>>]
+    # @yieldparam headers [Array<String>, nil]
+    # @return [self]
     def additional_data(&block)
       steps << [:additional_data, @headers, block]
       self
     end
 
+    # Keeps the rows the block returns true for.
+    # @yieldparam row [Array<String>]
+    # @yieldparam headers [Array<String>, nil]
+    # @yieldparam additional_data [Object]
+    # @return [self]
     def select(&block)
       steps << [:select, @headers, block]
       self
     end
 
+    # Drops the rows the block returns true for.
+    # @yieldparam (see #select)
+    # @return [self]
     def reject(&block)
       steps << [:reject, @headers, block]
       self
     end
 
+    # Replaces each row with the block's result.
+    # @param new_headers [Array<String>, nil] the headers after this step
+    # @yieldparam (see #select)
+    # @yieldreturn [Array] the new row
+    # @return [self]
     def map(new_headers, &block)
       steps << [:map, @headers, block]
       @headers = new_headers
       self
     end
 
+    # Adds the block's values to the end of each row.
+    # @param additional_headers [Array<String>, nil] headers for the new values; nil drops the header row
+    # @yieldparam (see #select)
+    # @yieldreturn [Array] values to append
+    # @return [self]
     def append(additional_headers, &block)
       steps << [:append, @headers, block]
-
-      if additional_headers
-        @headers += additional_headers
-      else
-        @headers = nil
-      end
-
+      # without headers (read_headers not called) the output has no header row to extend
+      @headers = (@headers + additional_headers if @headers && additional_headers)
       self
     end
 
+    # Calls the block for each row without changing it.
+    # @yieldparam (see #select)
+    # @return [self]
     def each(&block)
       steps << [:each, @headers, block]
       self
     end
 
+    # Replaces the headers written to the output.
+    # @param headers [Array<String>, nil]
+    # @return [self]
     def set_headers(headers)
       @headers = headers
       self
     end
 
+    # Runs the steps over every row, writes the output and closes the files it opened, even when a step raises.
+    # @param batch_size [Integer] rows read and run through the steps at a time
+    # @return [void]
     def process(batch_size = 10_000)
       batch = []
 
@@ -79,7 +122,7 @@ module CSVUtils
       end
 
       steps_proc.call if batch.size.positive?
-
+    ensure
       @src_csv.close
       @dest_csv.close
     end
@@ -90,33 +133,34 @@ module CSVUtils
       @steps ||= []
     end
 
-    def process_step(step_type, current_headers, batch, &block)
-      case step_type
-      when :select
-        batch.select! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :reject
-        batch.reject! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :map
-        batch.map! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :append
-        batch.map! do |row|
-          row + block.call(row, current_headers, @additional_data)
-        end
-      when :additional_data
-        @additional_data = block.call(batch, current_headers)
-      when :each
-        batch.each do |row|
-          block.call(row, current_headers, @additional_data)
-        end
-      end
-
+    # each step type has a process_<type>_step method that changes the batch in place
+    def process_step(step_type, current_headers, batch, &)
+      send(:"process_#{step_type}_step", current_headers, batch, &)
       batch
     end
+
+    def process_select_step(current_headers, batch)
+      batch.select! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_reject_step(current_headers, batch)
+      batch.reject! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_map_step(current_headers, batch)
+      batch.map! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_append_step(current_headers, batch)
+      batch.map! { |row| row + yield(row, current_headers, @additional_data) }
+    end
+
+    def process_additional_data_step(current_headers, batch)
+      @additional_data = yield batch, current_headers
+    end
+
+    def process_each_step(current_headers, batch)
+      batch.each { |row| yield row, current_headers, @additional_data }
+    end
   end
 end

data/lib/csv_utils/csv_wrapper.rb

@@ -1,25 +1,38 @@
 # frozen_string_literal: true
 
-# Wraps a CSV object, if wrapper opens the csv file it will close it
 module CSVUtils
+  # Wraps a path or an open CSV so callers can read, write and close it the same way.
+  # It closes only the files it opened itself.
   class CSVWrapper
+    # @return [CSV, #shift, #<<] the CSV being wrapped
     attr_reader :csv
 
+    # @param csv [String, CSV, #shift, #<<] path to open, or a CSV (or array of rows) to use as is
+    # @param mode [String] file mode for a path
+    # @param csv_options [Hash] options passed to CSV.open for a path
     def initialize(csv, mode, csv_options)
       open(csv, mode, csv_options)
     end
 
+    # Wraps a path or CSV; with a block, yields the wrapper and closes it afterwards, even when the block raises.
+    # @param file [String, CSV] path or CSV
+    # @param mode [String] file mode for a path
+    # @param csv_options [Hash] options passed to CSV.open for a path
+    # @yieldparam csv [CSVWrapper]
+    # @return [CSVWrapper, Object] the wrapper, or the block's value
     def self.open(file, mode, csv_options = {})
       csv = new(file, mode, csv_options)
+      return csv unless block_given?
 
-      if block_given?
+      begin
         yield csv
+      ensure
         csv.close
-      else
-        csv
       end
     end
 
+    # @api private
+    # @return [CSV, #shift, #<<]
     def open(csv, mode, csv_options)
       if csv.is_a?(String)
         @close_when_done = true
@@ -30,18 +43,25 @@ module CSVUtils
       end
     end
 
+    # @param row [Array]
+    # @return [CSV, #<<]
     def <<(row)
       csv << row
     end
 
+    # @return [Array<String>, nil] the next row, nil at the end
     def shift
       csv.shift
     end
 
+    # Moves back to the first row.
+    # @return [void]
     def rewind
       csv.rewind
     end
 
+    # Closes the CSV if the wrapper opened it.
+    # @return [void]
     def close
       csv.close if close_when_done?
     end

data/lib/csv_utils/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Gem version, bumped by release-please
+  VERSION = '0.6.0'
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Doug Youch
 bindir: bin
 cert_chain: []
-date: 2026-01-31 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -15,31 +15,39 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-description: A Ruby library for CSV file processing featuring comparison, transformation,
-  sorting, and validation. Includes CLI tools for debugging malformed CSVs, auto-detection
-  of encodings and separators, and efficient handling of large files.
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Streaming tools for CSV files too big or too broken to load into memory.
+  Compare two sorted files into create, update and delete actions, sort with an external
+  merge sort, transform and extend rows in batches, and build reports from Ruby objects.
+  Detects separators, byte order marks and encodings, and ships command line tools
+  that pinpoint malformed rows, diff, grep, split and validate CSV files.
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
@@ -54,15 +62,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ci.yml"
-- ".gitignore"
-- ".rubocop.yml"
-- ".ruby-gemset"
-- ".ruby-version"
-- ARCHITECTURE.md
-- CLAUDE.md
-- Gemfile
-- Gemfile.lock
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/csv-change-eol
@@ -74,8 +74,8 @@ files:
 - bin/csv-readline
 - bin/csv-splitter
 - bin/csv-validator
-- csv-utils.gemspec
 - lib/csv-utils.rb
+- lib/csv_utils/byte_order_mark.rb
 - lib/csv_utils/csv_compare.rb
 - lib/csv_utils/csv_extender.rb
 - lib/csv_utils/csv_iterator.rb
@@ -86,12 +86,16 @@ files:
 - lib/csv_utils/csv_sort.rb
 - lib/csv_utils/csv_transformer.rb
 - lib/csv_utils/csv_wrapper.rb
-- script/console
+- lib/csv_utils/version.rb
 homepage: https://github.com/dougyouch/csv-utils
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/dougyouch/csv-utils
+  changelog_uri: https://github.com/dougyouch/csv-utils/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/dougyouch/csv-utils/issues
+  documentation_uri: https://rubydoc.info/gems/csv-utils
 rdoc_options: []
 require_paths:
 - lib
@@ -99,14 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 4.0.20
 specification_version: 4
-summary: Comprehensive CSV manipulation and debugging utilities for Ruby
+summary: Compare, sort, transform and debug large or malformed CSV files
 test_files: []

data/.github/workflows/ci.yml

@@ -1,53 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  lint:
-    name: RuboCop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.4'
-          bundler-cache: true
-
-      - name: Run RuboCop
-        run: bundle exec rubocop --format github
-
-  test:
-    name: Tests (Ruby ${{ matrix.ruby }})
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: ['3.2', '3.3', '3.4']
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Ruby ${{ matrix.ruby }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-
-      - name: Run tests
-        run: bundle exec rspec
-
-      - name: Upload coverage to Codecov
-        if: matrix.ruby == '3.4'
-        uses: codecov/codecov-action@v4
-        with:
-          files: coverage/coverage.xml
-          fail_ci_if_error: false
-          verbose: true
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

data/.gitignore

@@ -1,52 +0,0 @@
-*.gem
-*.rbc
-/.config
-/coverage/
-/InstalledFiles
-/pkg/
-/spec/reports/
-/spec/examples.txt
-/test/tmp/
-/test/version_tmp/
-/tmp/
-
-# Used by dotenv library to load environment variables.
-# .env
-
-## Specific to RubyMotion:
-.dat*
-.repl_history
-build/
-*.bridgesupport
-build-iPhoneOS/
-build-iPhoneSimulator/
-
-## Specific to RubyMotion (use of CocoaPods):
-#
-# We recommend against adding the Pods directory to your .gitignore. However
-# you should judge for yourself, the pros and cons are mentioned at:
-# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
-#
-# vendor/Pods/
-
-## Documentation cache and generated files:
-/.yardoc/
-/_yardoc/
-/doc/
-/rdoc/
-
-## Environment normalization:
-/.bundle/
-/vendor/bundle
-/lib/bundler/man/
-
-# for a library or gem, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
-
-# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
-.rvmrc
-
-*~