csv-utils 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '083044bd714b955ff9d5f6a44cf6d4cb3344cf7f649b105d81a94b3ddb1c9425'
-  data.tar.gz: 5fa1cd9acacf10c275a23176e36722e189f8e6d8a85e1bfc7faf18eb45a1ca31
+  metadata.gz: f85c431ad42ed20382fbe91c3696153be9437a8ee755ede313be2d6f488b3770
+  data.tar.gz: b620cfb208a7a28573160103155564875b05990f965f9206ad89bd7cb6b5fcc7
 SHA512:
-  metadata.gz: de36fb6c80a68b33c92f3c943a665c419ac1287f5b4b2901c1a60f45964d4b13c2199f53e8dd779b2947d6c86439fa1b1b781cc5ba2225656b2d8b50f690e4cc
-  data.tar.gz: 0a6b5a9301f2386c2ad5bf3009ca77f949a92510ebb606844a1e6bf9fcc573f524966e68d1d8cec7db9778be3c93fb58a006fee9008e531c53a2c5391af6c0c3
+  metadata.gz: 02fe7a0d34f61c54a3788739cc5455dc685cad7b627a9091f2b6e5b3ed0323bf5d162cff53150e89a431c6ab42e76e014b1350c6b47f4aeed66ef727ffe76554
+  data.tar.gz: 117d50507b9661c1d70b89df658a97649ea2562604c9dd0764f160e9294586293edbf10e9d968da0107a2f072b0e1814d105c364f91ed2c22cca509fb48f2fd6

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.2.1'
+  s.version     = '0.2.2'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

data/lib/csv-utils.rb

@@ -6,4 +6,5 @@ module CSVUtils
   autoload :CSVOptions, 'csv_utils/csv_options'
   autoload :CSVReport, 'csv_utils/csv_report'
   autoload :CSVRow, 'csv_utils/csv_row'
+  autoload :CSVSort, 'csv_utils/csv_sort'
 end

data/lib/csv_utils/csv_sort.rb

@@ -0,0 +1,112 @@
+require 'fileutils'
+
+# Utility class for sorting the rows for a csv file
+class CSVUtils::CSVSort
+  attr_reader :csv_file,
+              :new_csv_file,
+              :has_headers,
+              :csv_options,
+              :headers
+
+  def initialize(csv_file, new_csv_file, has_headers = true, csv_options = {})
+    @csv_file = csv_file
+    @new_csv_file = new_csv_file
+    @has_headers = has_headers
+    @csv_options = csv_options
+    @csv_part_files = []
+    @files_to_delete = []
+  end
+
+  def sort(batch_size = 100_000, &block)
+    create_sorted_csv_part_files(batch_size, &block)
+    merge_csv_part_files(&block)
+  end
+
+  private
+
+  def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
+    src1 = CSV.open(src_csv_file1, 'rb', csv_options)
+    src2 = CSV.open(src_csv_file2, 'rb', csv_options)
+    dest = CSV.open(dest_csv_file, 'wb', csv_options)
+
+    if @headers
+      dest << @headers
+      src1.shift
+      src2.shift
+    end
+
+    row1 = src1.shift
+    row2 = src2.shift
+
+    append_row1_proc = Proc.new do
+      dest << row1
+      row1 = src1.shift
+    end
+
+    append_row2_proc = Proc.new do
+      dest << row2
+      row2 = src2.shift
+    end
+
+    while row1 || row2
+      if row1.nil?
+        append_row2_proc.call
+      elsif row2.nil?
+        append_row1_proc.call
+      elsif yield(row1, row2) <= 0
+        append_row1_proc.call
+      else
+        append_row2_proc.call
+      end
+    end
+
+    src1.close
+    src2.close
+    dest.close
+  end
+
+  def create_sorted_csv_part_files(batch_size, &block)
+    src = CSV.open(csv_file, 'rb', csv_options)
+
+    @headers = src.shift if has_headers
+
+    batch = []
+    create_batch_part_proc = Proc.new do
+      batch.sort!(&block)
+      @csv_part_files << "#{new_csv_file}.part.#{@csv_part_files.size}"
+      CSV.open(@csv_part_files.last, 'wb', csv_options) do |csv|
+        csv << @headers if @headers
+        batch.each { |row| csv << row }
+      end
+      batch = []
+    end
+
+    while (row = src.shift)
+      batch << row
+      create_batch_part_proc.call if batch.size >= batch_size
+    end
+
+    create_batch_part_proc.call if batch.size > 0
+
+    src.close
+  end
+
+  def merge_csv_part_files(&block)
+    file_merge_cnt = 0
+
+    while @csv_part_files.size > 1
+      file_merge_cnt += 1
+
+      csv_part_file1 = @csv_part_files.shift
+      csv_part_file2 = @csv_part_files.shift
+      @csv_part_files << "#{new_csv_file}.merge.#{file_merge_cnt}"
+
+      merge_sort_csv_files(csv_part_file1, csv_part_file2, @csv_part_files.last, &block)
+
+      File.unlink(csv_part_file1)
+      File.unlink(csv_part_file2)
+    end
+
+    FileUtils.mv(@csv_part_files.last, new_csv_file)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-04 00:00:00.000000000 Z
+date: 2020-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -49,6 +49,7 @@ files:
 - lib/csv_utils/csv_options.rb
 - lib/csv_utils/csv_report.rb
 - lib/csv_utils/csv_row.rb
+- lib/csv_utils/csv_sort.rb
 - script/console
 homepage: https://github.com/dougyouch/csv-utils
 licenses: