csv-utils 0.5.0 → 0.7.0

data/lib/csv_utils/csv_transformer.rb

@@ -1,63 +1,107 @@
 # frozen_string_literal: true
 
-# Transforms a CSV given a series of steps
 module CSVUtils
+  # Copies a CSV file through a pipeline of steps that filter, change and add to its rows in batches.
+  # Each step method returns the transformer so they can be chained, and {#process} runs them.
+  #
+  # Step blocks receive the row, the headers as they were when the step was added, and the value
+  # computed by the last {#additional_data} step for the current batch.
+  #
+  # @example
+  #   CSVUtils::CSVTransformer.new('input.csv', 'output.csv')
+  #     .read_headers
+  #     .select { |row, _headers, _data| row[0].to_i > 100 }
+  #     .append(['Email']) { |row, _headers, _data| ["#{row[1].downcase}@example.com"] }
+  #     .process
   class CSVTransformer
+    # @return [Array<String>, nil] the headers the output will have after the steps added so far
     attr_reader :headers
 
+    # @param src_csv [String, CSV, #shift] path or CSV to read; paths are opened and closed by the transformer
+    # @param dest_csv [String, CSV, #<<] path or CSV to write
+    # @param csv_options [Hash] options passed to CSV.open for paths; the source is read with its :encoding,
+    #   'bom|utf-8' by default, and the output written in the encoding values were decoded to (see {EncodingOptions})
     def initialize(src_csv, dest_csv, csv_options = {})
-      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
-      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+      @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', EncodingOptions.read(csv_options))
+      @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', EncodingOptions.write(csv_options))
     end
 
+    # Reads the first row as the headers. Without it, every row is data and the output has no header row.
+    # @return [self]
     def read_headers
       @headers = @src_csv.shift
       self
     end
 
+    # Computes a value once per batch, passed to the steps that follow as their third argument.
+    # @yieldparam batch [Array<Array<String>>]
+    # @yieldparam headers [Array<String>, nil]
+    # @return [self]
     def additional_data(&block)
       steps << [:additional_data, @headers, block]
       self
     end
 
+    # Keeps the rows the block returns true for.
+    # @yieldparam row [Array<String>]
+    # @yieldparam headers [Array<String>, nil]
+    # @yieldparam additional_data [Object]
+    # @return [self]
     def select(&block)
       steps << [:select, @headers, block]
       self
     end
 
+    # Drops the rows the block returns true for.
+    # @yieldparam (see #select)
+    # @return [self]
     def reject(&block)
       steps << [:reject, @headers, block]
       self
     end
 
+    # Replaces each row with the block's result.
+    # @param new_headers [Array<String>, nil] the headers after this step
+    # @yieldparam (see #select)
+    # @yieldreturn [Array] the new row
+    # @return [self]
     def map(new_headers, &block)
       steps << [:map, @headers, block]
       @headers = new_headers
       self
     end
 
+    # Adds the block's values to the end of each row.
+    # @param additional_headers [Array<String>, nil] headers for the new values; nil drops the header row
+    # @yieldparam (see #select)
+    # @yieldreturn [Array] values to append
+    # @return [self]
     def append(additional_headers, &block)
       steps << [:append, @headers, block]
-
-      if additional_headers
-        @headers += additional_headers
-      else
-        @headers = nil
-      end
-
+      # without headers (read_headers not called) the output has no header row to extend
+      @headers = (@headers + additional_headers if @headers && additional_headers)
       self
     end
 
+    # Calls the block for each row without changing it.
+    # @yieldparam (see #select)
+    # @return [self]
     def each(&block)
       steps << [:each, @headers, block]
       self
     end
 
+    # Replaces the headers written to the output.
+    # @param headers [Array<String>, nil]
+    # @return [self]
     def set_headers(headers)
       @headers = headers
       self
     end
 
+    # Runs the steps over every row, writes the output and closes the files it opened, even when a step raises.
+    # @param batch_size [Integer] rows read and run through the steps at a time
+    # @return [void]
     def process(batch_size = 10_000)
       batch = []
 
@@ -79,7 +123,7 @@ module CSVUtils
       end
 
       steps_proc.call if batch.size.positive?
-
+    ensure
       @src_csv.close
       @dest_csv.close
     end
@@ -90,33 +134,34 @@ module CSVUtils
       @steps ||= []
     end
 
-    def process_step(step_type, current_headers, batch, &block)
-      case step_type
-      when :select
-        batch.select! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :reject
-        batch.reject! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :map
-        batch.map! do |row|
-          block.call row, current_headers, @additional_data
-        end
-      when :append
-        batch.map! do |row|
-          row + block.call(row, current_headers, @additional_data)
-        end
-      when :additional_data
-        @additional_data = block.call(batch, current_headers)
-      when :each
-        batch.each do |row|
-          block.call(row, current_headers, @additional_data)
-        end
-      end
-
+    # each step type has a process_<type>_step method that changes the batch in place
+    def process_step(step_type, current_headers, batch, &)
+      send(:"process_#{step_type}_step", current_headers, batch, &)
       batch
     end
+
+    def process_select_step(current_headers, batch)
+      batch.select! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_reject_step(current_headers, batch)
+      batch.reject! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_map_step(current_headers, batch)
+      batch.map! { |row| yield row, current_headers, @additional_data }
+    end
+
+    def process_append_step(current_headers, batch)
+      batch.map! { |row| row + yield(row, current_headers, @additional_data) }
+    end
+
+    def process_additional_data_step(current_headers, batch)
+      @additional_data = yield batch, current_headers
+    end
+
+    def process_each_step(current_headers, batch)
+      batch.each { |row| yield row, current_headers, @additional_data }
+    end
   end
 end

data/lib/csv_utils/csv_wrapper.rb

@@ -1,25 +1,38 @@
 # frozen_string_literal: true
 
-# Wraps a CSV object, if wrapper opens the csv file it will close it
 module CSVUtils
+  # Wraps a path or an open CSV so callers can read, write and close it the same way.
+  # It closes only the files it opened itself.
   class CSVWrapper
+    # @return [CSV, #shift, #<<] the CSV being wrapped
     attr_reader :csv
 
+    # @param csv [String, CSV, #shift, #<<] path to open, or a CSV (or array of rows) to use as is
+    # @param mode [String] file mode for a path
+    # @param csv_options [Hash] options passed to CSV.open for a path
     def initialize(csv, mode, csv_options)
       open(csv, mode, csv_options)
     end
 
+    # Wraps a path or CSV; with a block, yields the wrapper and closes it afterwards, even when the block raises.
+    # @param file [String, CSV] path or CSV
+    # @param mode [String] file mode for a path
+    # @param csv_options [Hash] options passed to CSV.open for a path
+    # @yieldparam csv [CSVWrapper]
+    # @return [CSVWrapper, Object] the wrapper, or the block's value
     def self.open(file, mode, csv_options = {})
       csv = new(file, mode, csv_options)
+      return csv unless block_given?
 
-      if block_given?
+      begin
         yield csv
+      ensure
         csv.close
-      else
-        csv
       end
     end
 
+    # @api private
+    # @return [CSV, #shift, #<<]
     def open(csv, mode, csv_options)
       if csv.is_a?(String)
         @close_when_done = true
@@ -30,18 +43,25 @@ module CSVUtils
       end
     end
 
+    # @param row [Array]
+    # @return [CSV, #<<]
     def <<(row)
       csv << row
     end
 
+    # @return [Array<String>, nil] the next row, nil at the end
     def shift
       csv.shift
     end
 
+    # Moves back to the first row.
+    # @return [void]
     def rewind
       csv.rewind
     end
 
+    # Closes the CSV if the wrapper opened it.
+    # @return [void]
     def close
       csv.close if close_when_done?
     end

data/lib/csv_utils/encoding_options.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # The one place the library decides how files are decoded and encoded. Every class reads with the
+  # :encoding CSV option, 'bom|utf-8' by default, so values are UTF-8 strings whatever the locale, and
+  # writes in the encoding the values were decoded to, so reading 'Windows-1252:UTF-8' writes UTF-8.
+  # File modes never carry an encoding: CSV raises when the mode and the :encoding option both have one.
+  #
+  # @example
+  #   CSV.open(path, 'rb', **CSVUtils::EncodingOptions.read(col_sep: "\t"))
+  module EncodingOptions
+    # @return [String] the :encoding option used for reading when none is given
+    DEFAULT_ENCODING = 'bom|utf-8'
+
+    # Options for reading: the given options with the default :encoding when they have none.
+    # @param csv_options [Hash]
+    # @return [Hash]
+    def self.read(csv_options)
+      { encoding: DEFAULT_ENCODING }.merge(csv_options)
+    end
+
+    # Options for writing values read with csv_options: the :encoding is the one values were decoded to.
+    # @param csv_options [Hash]
+    # @return [Hash]
+    def self.write(csv_options)
+      csv_options.merge(encoding: decoded_encoding(read(csv_options)[:encoding]))
+    end
+
+    # The encoding of strings read with an :encoding option: the internal encoding of 'external:internal',
+    # otherwise the external one, without a 'bom|' prefix.
+    # @example
+    #   CSVUtils::EncodingOptions.decoded_encoding('Windows-1252:UTF-8') # => 'UTF-8'
+    #   CSVUtils::EncodingOptions.decoded_encoding('bom|utf-8') # => 'utf-8'
+    # @param encoding [String, Encoding]
+    # @return [String]
+    def self.decoded_encoding(encoding)
+      encoding.to_s.split(':').last.sub(/\Abom\|/i, '')
+    end
+  end
+end

data/lib/csv_utils/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module CSVUtils
+  # Gem version, bumped by release-please
+  VERSION = '0.7.0'
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Doug Youch
 bindir: bin
 cert_chain: []
-date: 2026-01-31 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -15,31 +15,39 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-description: A Ruby library for CSV file processing featuring comparison, transformation,
-  sorting, and validation. Includes CLI tools for debugging malformed CSVs, auto-detection
-  of encodings and separators, and efficient handling of large files.
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Streaming tools for CSV files too big or too broken to load into memory.
+  Compare two sorted files into create, update and delete actions, sort with an external
+  merge sort, transform and extend rows in batches, and build reports from Ruby objects.
+  Detects separators, byte order marks and encodings, and ships command line tools
+  that pinpoint malformed rows, diff, grep, split and validate CSV files.
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
@@ -54,15 +62,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ci.yml"
-- ".gitignore"
-- ".rubocop.yml"
-- ".ruby-gemset"
-- ".ruby-version"
-- ARCHITECTURE.md
-- CLAUDE.md
-- Gemfile
-- Gemfile.lock
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/csv-change-eol
@@ -74,8 +74,9 @@ files:
 - bin/csv-readline
 - bin/csv-splitter
 - bin/csv-validator
-- csv-utils.gemspec
 - lib/csv-utils.rb
+- lib/csv_utils/byte_order_mark.rb
+- lib/csv_utils/character_encoding.rb
 - lib/csv_utils/csv_compare.rb
 - lib/csv_utils/csv_extender.rb
 - lib/csv_utils/csv_iterator.rb
@@ -86,12 +87,17 @@ files:
 - lib/csv_utils/csv_sort.rb
 - lib/csv_utils/csv_transformer.rb
 - lib/csv_utils/csv_wrapper.rb
-- script/console
+- lib/csv_utils/encoding_options.rb
+- lib/csv_utils/version.rb
 homepage: https://github.com/dougyouch/csv-utils
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/dougyouch/csv-utils
+  changelog_uri: https://github.com/dougyouch/csv-utils/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/dougyouch/csv-utils/issues
+  documentation_uri: https://rubydoc.info/gems/csv-utils
 rdoc_options: []
 require_paths:
 - lib
@@ -99,14 +105,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 4.0.20
 specification_version: 4
-summary: Comprehensive CSV manipulation and debugging utilities for Ruby
+summary: Compare, sort, transform and debug large or malformed CSV files
 test_files: []

data/.github/workflows/ci.yml

@@ -1,53 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  lint:
-    name: RuboCop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.4'
-          bundler-cache: true
-
-      - name: Run RuboCop
-        run: bundle exec rubocop --format github
-
-  test:
-    name: Tests (Ruby ${{ matrix.ruby }})
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: ['3.2', '3.3', '3.4']
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Ruby ${{ matrix.ruby }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-
-      - name: Run tests
-        run: bundle exec rspec
-
-      - name: Upload coverage to Codecov
-        if: matrix.ruby == '3.4'
-        uses: codecov/codecov-action@v4
-        with:
-          files: coverage/coverage.xml
-          fail_ci_if_error: false
-          verbose: true
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

data/.gitignore

@@ -1,52 +0,0 @@
-*.gem
-*.rbc
-/.config
-/coverage/
-/InstalledFiles
-/pkg/
-/spec/reports/
-/spec/examples.txt
-/test/tmp/
-/test/version_tmp/
-/tmp/
-
-# Used by dotenv library to load environment variables.
-# .env
-
-## Specific to RubyMotion:
-.dat*
-.repl_history
-build/
-*.bridgesupport
-build-iPhoneOS/
-build-iPhoneSimulator/
-
-## Specific to RubyMotion (use of CocoaPods):
-#
-# We recommend against adding the Pods directory to your .gitignore. However
-# you should judge for yourself, the pros and cons are mentioned at:
-# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
-#
-# vendor/Pods/
-
-## Documentation cache and generated files:
-/.yardoc/
-/_yardoc/
-/doc/
-/rdoc/
-
-## Environment normalization:
-/.bundle/
-/vendor/bundle
-/lib/bundler/man/
-
-# for a library or gem, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
-
-# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
-.rvmrc
-
-*~

data/.rubocop.yml

@@ -1,81 +0,0 @@
-AllCops:
-  TargetRubyVersion: 3.2
-  NewCops: enable
-  SuggestExtensions: false
-  Exclude:
-    - 'bin/**/*'
-    - 'vendor/**/*'
-    - 'coverage/**/*'
-
-# Relaxed metrics for existing codebase
-Metrics/MethodLength:
-  Max: 35
-
-Metrics/AbcSize:
-  Max: 40
-
-Metrics/ClassLength:
-  Max: 200
-
-Metrics/CyclomaticComplexity:
-  Max: 15
-
-Metrics/PerceivedComplexity:
-  Max: 15
-
-Metrics/BlockLength:
-  Exclude:
-    - 'spec/**/*'
-    - '*.gemspec'
-
-# Style preferences
-Style/Documentation:
-  Enabled: false
-
-Style/FrozenStringLiteralComment:
-  EnforcedStyle: always
-
-Layout/LineLength:
-  Max: 130
-  Exclude:
-    - 'spec/**/*'
-
-# File naming - allow hyphenated gem name
-Naming/FileName:
-  Exclude:
-    - 'lib/csv-utils.rb'
-
-# Allow set_ prefix for transformer DSL methods
-Naming/AccessorMethodName:
-  Exclude:
-    - 'lib/csv_utils/csv_transformer.rb'
-
-# Allow Kernel#open for pipe support (intentional design)
-Security/Open:
-  Exclude:
-    - 'lib/csv_utils/csv_wrapper.rb'
-
-# Duplicate branches are intentional in comparison logic
-Lint/DuplicateBranch:
-  Exclude:
-    - 'lib/csv_utils/csv_compare.rb'
-    - 'lib/csv_utils/csv_sort.rb'
-
-# Allow empty blocks in specs (used for testing iteration behavior)
-Lint/EmptyBlock:
-  Exclude:
-    - 'spec/**/*'
-
-# Style relaxations for existing code patterns
-Style/OptionalBooleanParameter:
-  Enabled: false
-
-Style/StringConcatenation:
-  Enabled: false
-
-Style/FormatStringToken:
-  Enabled: false
-
-# Gemspec settings
-Gemspec/RequiredRubyVersion:
-  Enabled: false

data/.ruby-gemset

@@ -1 +0,0 @@
-csv-utils

data/.ruby-version

@@ -1 +0,0 @@
-3.4.2