csv-utils 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b1634cd26b21129c05cf1c5655e46ff3b81307135ce8dd686022cd8a4ff5adc
-  data.tar.gz: 53312f6841e32ddcae0e97cd9aab4513a47580c35a3ec39964f8933fdbbcff10
+  metadata.gz: 98303ab9b2df05bc501c1c66b66a62be5ade9d79ab38a5b8bda8eb52d91b26cc
+  data.tar.gz: 8adfd2144220de2cc4f23136ee4eb7314a3c16eeac68be87e1dc19b1ac7dc350
 SHA512:
-  metadata.gz: 5733c0ff8b730e957c46fc73b1080690dd79fb481b30f9591f4256507abc3ea46139d9758605c4c4e143e23781468b9adbd623812ceb105bb610a61943ffbc57
-  data.tar.gz: 4ce9256c44ed50fc289fbe2b0ce970b2798310d50b4f69dffe9fa57098b0acab0539049bd5c945b43768ea7cb3d97b2421e54bf4c069df7751bd3704b68db9ec
+  metadata.gz: a2a2b2067a9ca06920b171230a122eba479c1f91af3919e2965eaec6d073fff34d544221a92cffaa1b9546078960aee0c9b9031e7b652368e975cff9b196214c
+  data.tar.gz: '0786cfb3e75771ccb68bfa0e2cba42994c7c04a5c8be14432ae6467425536e7dfd4a4ef33403ae5bd129eafd871a07077610c000a78762052e0b055192c0cc16'

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.2.3'
+  s.version     = '0.3.0'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

data/lib/csv-utils.rb

@@ -7,4 +7,6 @@ module CSVUtils
   autoload :CSVReport, 'csv_utils/csv_report'
   autoload :CSVRow, 'csv_utils/csv_row'
   autoload :CSVSort, 'csv_utils/csv_sort'
+  autoload :CSVTransformer, 'csv_utils/csv_transformer'
+  autoload :CSVWrapper, 'csv_utils/csv_wrapper'
 end

data/lib/csv_utils/csv_extender.rb

@@ -1,20 +1,15 @@
 # Utility class for appending data to a csv file.
 class CSVUtils::CSVExtender
-  attr_reader :csv_file,
-              :new_csv_file,
-              :csv_options
-
-  def initialize(csv_file, new_csv_file, csv_options = {})
-    @csv_file = csv_file
-    @new_csv_file = new_csv_file
-    @csv_options = csv_options
+  def initialize(src_csv, dest_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+    @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
   end
 
   def append(additional_headers)
     process(additional_headers) do |current_headers|
-      while (row = src.shift)
+      while (row = @src_csv.shift)
         additional_columns = yield row, current_headers
-        dest << (row + additional_columns)
+        @dest_csv << (row + additional_columns)
       end
     end
   end
@@ -27,13 +22,13 @@ class CSVUtils::CSVExtender
         additional_rows = yield batch, current_headers
 
         batch.each_with_index do |row, idx|
-          dest << (row + additional_rows[idx])
+          @dest_csv << (row + additional_rows[idx])
         end
 
         batch = []
       end
 
-      while (row = src.shift)
+      while (row = @src_csv.shift)
         batch << row
 
         process_batch_proc.call if batch.size >= batch_size
@@ -43,6 +38,8 @@ class CSVUtils::CSVExtender
     end
   end
 
+  private
+
   def process(additional_headers)
     current_headers = append_headers(additional_headers)
 
@@ -51,26 +48,16 @@ class CSVUtils::CSVExtender
     close
   end
 
-  def src
-    @src ||= CSV.open(csv_file, 'rb', csv_options)
-  end
-
-  def dest
-    @dest ||= CSV.open(new_csv_file, 'wb', csv_options)
-  end
-
   def close
-    src.close
-    dest.close
+    @src_csv.close
+    @dest_csv.close
   end
 
-  private
-
   def append_headers(additional_headers)
     return nil unless additional_headers
 
-    current_headers = src.shift
-    dest << (current_headers + additional_headers)
+    current_headers = @src_csv.shift
+    @dest_csv << (current_headers + additional_headers)
     current_headers
   end
 end

data/lib/csv_utils/csv_transformer.rb

@@ -0,0 +1,119 @@
+# Transforms a CSV given a series of steps
+class CSVUtils::CSVTransformer
+  attr_reader :headers
+
+  def initialize(src_csv, dest_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+    @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+  end
+
+  def read_headers
+    @headers = @src_csv.shift
+    self
+  end
+
+  def additional_data(&block)
+    steps << [:additional_data, @headers, block]
+    self
+  end
+
+  def select(&block)
+    steps << [:select, @headers, block]
+    self
+  end
+
+  def reject(&block)
+    steps << [:reject, @headers, block]
+    self
+  end
+
+  def map(new_headers, &block)
+    steps << [:map, @headers, block]
+    @headers = new_headers
+    self
+  end
+
+  def append(additional_headers, &block)
+    steps << [:append, @headers, block]
+
+    if additional_headers
+      @headers += additional_headers
+    else
+      @headers = nil
+    end
+
+    self
+  end
+
+  def each(&block)
+    steps << [:each, @headers, block]
+    self
+  end
+
+  def set_headers(headers)
+    @headers = headers
+    self
+  end
+
+  def process(batch_size = 10_000, &block)
+    batch = []
+
+    @dest_csv << @headers if @headers
+
+    steps_proc = Proc.new do
+      steps.each do |step_type, current_headers, proc|
+        batch = process_step(step_type, current_headers, batch, &proc)
+      end
+
+      batch.each { |row| @dest_csv << row }
+
+      batch = []
+    end
+                   
+    while (row = @src_csv.shift)
+      batch << row
+      steps_proc.call if batch.size >= batch_size
+    end
+
+    steps_proc.call if batch.size > 0
+
+    @src_csv.close
+    @dest_csv.close
+  end
+
+  private
+
+  def steps
+    @steps ||= []
+  end
+
+
+  def process_step(step_type, current_headers, batch, &block)
+    case step_type
+    when :select
+      batch.select! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :reject
+      batch.reject! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :map
+      batch.map! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :append
+      batch.map! do |row|
+        row + block.call(row, current_headers, @additional_data)
+      end
+    when :additional_data
+      @additional_data = block.call(batch, current_headers)
+    when :each
+      batch.each do |row|
+        block.call(row, current_headers, @additional_data)
+      end
+    end
+
+    batch
+  end
+end

data/lib/csv_utils/csv_wrapper.rb

@@ -0,0 +1,47 @@
+# Wraps a CSV object, if wrapper opens the csv file it will close it
+class CSVUtils::CSVWrapper
+  attr_reader :csv
+
+  def initialize(csv, mode, csv_options)
+    open(csv, mode, csv_options)
+  end
+
+  def self.open(file, mode, csv_options = {})
+    csv = new(file, mode, csv_options)
+
+    if block_given?
+      yield csv
+      csv.close
+    else
+      csv
+    end
+  end
+
+  def open(csv, mode, csv_options)
+    if csv.is_a?(String)
+      @close_when_done = true
+      @csv = CSV.open(csv, mode, csv_options)
+    else
+      @close_when_done = false
+      @csv = csv
+    end
+  end
+
+  def <<(row)
+    csv << row
+  end
+
+  def shift
+    csv.shift
+  end
+
+  def close
+    csv.close if close_when_done?
+  end
+
+  private
+
+  def close_when_done?
+    @close_when_done
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-08 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -50,6 +50,8 @@ files:
 - lib/csv_utils/csv_report.rb
 - lib/csv_utils/csv_row.rb
 - lib/csv_utils/csv_sort.rb
+- lib/csv_utils/csv_transformer.rb
+- lib/csv_utils/csv_wrapper.rb
 - script/console
 homepage: https://github.com/dougyouch/csv-utils
 licenses: