csv-utils 0.3.14 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a15793e118aa3bd4175b59c6a17e7800dd322390f95ce55af29170a6422ae0c3
-  data.tar.gz: 481c775dc66bc47fec11f5ab39f98a1a35020cb00f6e565761b47badc4d53ad8
+  metadata.gz: cda417e397f1791d177bb6eadd107edfdcd31a6680a3d2fef07b5be6df91dd6c
+  data.tar.gz: 7dee11a2b59738a8ad1ea9f692ef2da6472bab65039fbeed630c34d369a952c0
 SHA512:
-  metadata.gz: e987380b48bc309fbccb7bfd4311c8664077c8bac6ae31cf0c8c83f8035366f40fbdf60c8bf5b476e2a84e1899d6adada995fe81cd08531376511501700e2dca
-  data.tar.gz: 020e3c5d08bbc1025b72fbde2de838efd45c24cf41ba89c9fb93cceb619e4184ec0c1671bf26b3ae1d9dad8978ce206359c53672bce843a3ac8651cc6ccf0412
+  metadata.gz: af9f536d693cdcac8db949cbec1981b38637149927910b59aa570becef637cbae54c5371bc9d6031698c993e536c063b0ba89a0ca1be6c6d6ca7056e4326cc83
+  data.tar.gz: f48853f7756f3a7e36c34e37fd38ce611db880fb5e083340eedf817800f094dcd7900b89f3f7edab2eb8eac6e2c837e9209ec3dba1cc79d95d2f065f36b15d45

data/.ruby-version

@@ -1 +1 @@
-3.0.0
+3.1.0

data/Gemfile.lock

@@ -2,51 +2,51 @@ GEM
   remote: http://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.4.4)
-    docile (1.3.5)
-    inheritance-helper (0.1.5)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    inheritance-helper (0.2.5)
+    parallel (1.22.1)
+    parser (3.1.1.0)
       ast (~> 2.4.1)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    regexp_parser (2.1.1)
-    rexml (3.2.4)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.2.1)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.26.1)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.16.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
-    unicode-display_width (2.0.0)
+    simplecov_json_formatter (0.1.4)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
-  x86_64-darwin-20
+  x86_64-darwin-21
 
 DEPENDENCIES
   inheritance-helper
@@ -56,4 +56,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.2.3
+   2.3.3

data/bin/csv-diff

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+options = {
+  unique_headers: [],
+  ignore_headers: [],
+  sort_batch_size: 1_000_000
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: ' + File.basename(__FILE__) + ' [options] <csv file>'
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-u', '--unique HEADERS', 'Comman separated list of headers that genrate a unique key per a row, use 1st column by default') do |v|
+    options[:unique_headers] = v.split(',')
+  end
+
+  opts.on('-i', '--ignore HEADERS', 'Comman separated list of headers to ignore during row comparison') do |v|
+    options[:ignore_headers] = v.split(',')
+  end
+
+  opts.on('--sort-batch-size SIZE', Integer, 'Number of rows to load into memory while sorting') do |v|
+    opts[:sort_batch_size] = v
+  end
+end.parse!
+
+require 'csv-utils'
+
+csv1 = CSVUtils::CSVIterator.new(ARGV[0])
+csv2 = CSVUtils::CSVIterator.new(ARGV[1])
+
+unless csv1.first.keys == csv2.first.keys
+  $stderr.puts("headers do not match #{ARGV[0]} headers #{csv1.first.keys}, #{ARGV[1]} headers #{csv2.first.keys}")
+  exit 1
+end
+
+unknown_unique_headers = options[:unique_headers] - csv1.first.keys
+unless unknown_unique_headers.empty?
+  $stderr.puts("specified unique headers are unknown #{unknown_unique_headers}")
+  exit 1
+end
+
+unknown_ignore_headers = options[:ignore_headers] - csv1.first.keys
+unless unknown_ignore_headers.empty?
+  $stderr.puts("specified headers to ignore are unknown #{unknown_ignore_headers}")
+  exit 1
+end
+
+options[:unique_headers] = [csv1.first.keys.first] if options[:unique_headers].empty?
+
+puts "uniqueness header(s) are #{options[:unique_headers].join(', ')}"
+
+unique_header_indexes = []
+csv1.first.keys.each_with_index do |header, idx|
+  unique_header_indexes << idx if options[:unique_headers].include?(header)
+end
+
+sort_compare_proc = proc do |csv1_row, csv2_row|
+  result = 0
+  unique_header_indexes.each do |idx|
+    result = csv1_row[idx] <=> csv2_row[idx]
+    break unless result == 0
+  end
+
+  if result == 0
+    csv1_row.each_with_index do |csv1_col, idx|
+      csv2_col = csv2_row[idx]
+      result = csv1_col <=> csv2_col
+      break unless result == 0
+    end
+  end
+
+  result
+end
+
+csv1_sorted_file_name = ARGV[0] + '.sorted'
+csv2_sorted_file_name = ARGV[1] + '.sorted'
+
+puts "sorting #{ARGV[0]}"
+sorter = CSVUtils::CSVSort.new(ARGV[0], csv1_sorted_file_name)
+sorter.sort(options[:sort_batch_size], &sort_compare_proc)
+
+puts "sorting #{ARGV[1]}"
+sorter = CSVUtils::CSVSort.new(ARGV[1], csv2_sorted_file_name)
+sorter.sort(options[:sort_batch_size], &sort_compare_proc)
+
+options[:unique_headers] = [csv1.first.keys.first]
+update_comparison_columns = csv1.first.keys - options[:unique_headers]
+update_comparison_columns -= options[:ignore_headers]
+
+comparer = CSVUtils::CSVCompare.new(csv1_sorted_file_name, update_comparison_columns) do |csv1_row, csv2_row|
+  result = 0
+  options[:unique_headers].each do |header|
+    result = csv1_row[header] <=> csv2_row[header]
+    break unless result == 0
+  end
+
+  result
+end
+
+stats = Hash.new(0)
+puts "comparing #{ARGV[0]} with #{ARGV[1]}"
+diff_file_name = 'diff-results-' + ARGV[0]
+CSV.open(diff_file_name, 'wb') do |out|
+  out << ['Result'] + csv1.first.keys
+  comparer.compare(csv2_sorted_file_name) do |action, record|
+    stats[action] += 1
+    out << [action] + record.values
+  end
+end
+puts "differences found #{stats}"
+
+File.unlink(csv1_sorted_file_name)
+File.unlink(csv2_sorted_file_name)
+
+if stats.empty?
+  puts "files were identical"
+  File.unlink(diff_file_name)
+else
+  puts "results can be found in #{diff_file_name}"
+end

data/bin/csv-duplicate-finder

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+options = {
+  ignore_columns: []
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: ' + File.basename(__FILE__) + ' [options] <csv file>'
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-i', '--ignore HEADERS', 'Comman separated list of headers to ignore') do |v|
+    options[:ignore_columns] = v.split(',')
+  end
+end.parse!
+
+require 'digest/sha2'
+require 'json'
+require 'csv-utils'
+
+csv = CSVUtils::CSVIterator.new(ARGV[0])
+
+missing_headers = options[:ignore_columns] - csv.first.keys
+unless missing_headers.empty?
+  raise("unkown headers #{missing_headers.join(', ')} configured ingnore headers")
+end
+
+hashed_rows = {}
+
+csv.each_with_index do |row, idx|
+  options[:ignore_columns].each do |ignore_column|
+    row[ignore_column] = ''
+  end
+  key = Digest::SHA256.hexdigest(row.to_json)
+  hashed_rows[key] ||= []
+  hashed_rows[key] << idx
+end
+
+duplicate_rows = {}
+
+hashed_rows.reject! { |key, row_numbers| row_numbers.size <= 1 }
+
+hashed_rows.each do |key, row_numbers|
+  hashed_rows[key] = {}
+  row_numbers.each do |row_number|
+    duplicate_rows[row_number] = key
+    hashed_rows[key][row_number] = nil
+  end
+end
+
+csv.each_with_index do |row, idx|
+  next unless (key = duplicate_rows[idx])
+
+  hashed_rows[key][idx] = row
+end
+
+CSV.open('duplicates-' + File.basename(ARGV[0]), 'wb') do |out|
+  out << ['duplicate_key', 'row_no'] + csv.first.keys
+
+  hashed_rows.each do |key, rows|
+    rows.each do |idx, row|
+      out << [key, idx] + row.values
+    end
+  end
+end

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.3.14'
+  s.version     = '0.3.16'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.16
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-23 00:00:00.000000000 Z
+date: 2022-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -28,6 +28,8 @@ description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
+- csv-diff
+- csv-duplicate-finder
 - csv-explorer
 - csv-find-error
 - csv-readline
@@ -43,6 +45,8 @@ files:
 - LICENSE
 - README.md
 - bin/csv-change-eol
+- bin/csv-diff
+- bin/csv-duplicate-finder
 - bin/csv-explorer
 - bin/csv-find-error
 - bin/csv-readline
@@ -78,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: CSV Utils