csv-utils 0.3.12 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7b43801f0f7e62d34d8a77508964f0b6ea054a3ac2c8d17bf42d22aa0b245bd
-  data.tar.gz: ccd3282e4c7b1e1985439e24a159fe09c570b18446adee2b8db4776bc6024852
+  metadata.gz: 0b511a7a2cac6926477ed165212fa2009947dc84040f7d0451f7fbd7fdd9543b
+  data.tar.gz: 6661363cabaebcd2f21a41159f7ec0712b7729da5dde0cc7b1818415d8f0bc81
 SHA512:
-  metadata.gz: 1684319de3c4f728f55e1c201790dd5cc3aa110e09d14a4043fcdf3b70c02568fe898c4f2d7b839e633f461294f9870fae0a8cf2d3d785d173aa66f9c8ba2ab1
-  data.tar.gz: 6e7689f6f544e5280dbf144dc5a23a4b088aeff6c06931c0a856f6cd16353048e4ed343da0406550d2e7f48abd8af6e1ef31f6114ec9f3435aa56815c5fea24b
+  metadata.gz: cb860bdbe29726e44e1af6c528623daf7acc186f48eecefdbd47530d4ae7ff3472a2c7044cef8de079e70d4d9ada0b1bfbff65923fc3fcb9c6236d3d2e9606d8
+  data.tar.gz: fe7a67353de3c0e4dd5e36b71345abb373c19ca3e23bc0a8624f7f4147058911e487db17218c35e2ae2e77dfaed61185ca5ffea77736c410a3b18d1334514cd8

data/.ruby-version

@@ -1 +1 @@
-3.0.0
+3.1.0

data/Gemfile.lock

@@ -2,51 +2,51 @@ GEM
   remote: http://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.4.4)
-    docile (1.3.5)
-    inheritance-helper (0.1.5)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    inheritance-helper (0.2.5)
+    parallel (1.22.1)
+    parser (3.1.1.0)
       ast (~> 2.4.1)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    regexp_parser (2.1.1)
-    rexml (3.2.4)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.2.1)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.26.1)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.16.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
-    unicode-display_width (2.0.0)
+    simplecov_json_formatter (0.1.4)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
-  x86_64-darwin-20
+  x86_64-darwin-21
 
 DEPENDENCIES
   inheritance-helper
@@ -56,4 +56,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.2.3
+   2.3.3

data/bin/csv-duplicate-finder

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+options = {
+  ignore_columns: []
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: ' + File.basename(__FILE__) + ' [options] <csv file>'
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-i', '--ignore HEADERS', 'Comman separated list of headers to ignore') do |v|
+    options[:ignore_columns] = v.split(',')
+  end
+end.parse!
+
+require 'digest/sha2'
+require 'json'
+require 'csv-utils'
+
+csv = CSVUtils::CSVIterator.new(ARGV[0])
+
+missing_headers = options[:ignore_columns] - csv.first.keys
+unless missing_headers.empty?
+  raise("unkown headers #{missing_headers.join(', ')} configured ingnore headers")
+end
+
+hashed_rows = {}
+
+csv.each_with_index do |row, idx|
+  options[:ignore_columns].each do |ignore_column|
+    row[ignore_column] = ''
+  end
+  key = Digest::SHA256.hexdigest(row.to_json)
+  hashed_rows[key] ||= []
+  hashed_rows[key] << idx
+end
+
+duplicate_rows = {}
+
+hashed_rows.reject! { |key, row_numbers| row_numbers.size <= 1 }
+
+hashed_rows.each do |key, row_numbers|
+  hashed_rows[key] = {}
+  row_numbers.each do |row_number|
+    duplicate_rows[row_number] = key
+    hashed_rows[key][row_number] = nil
+  end
+end
+
+csv.each_with_index do |row, idx|
+  next unless (key = duplicate_rows[idx])
+
+  hashed_rows[key][idx] = row
+end
+
+CSV.open('duplicates-' + File.basename(ARGV[0]), 'wb') do |out|
+  out << ['duplicate_key', 'row_no'] + csv.first.keys
+
+  hashed_rows.each do |key, rows|
+    rows.each do |idx, row|
+      out << [key, idx] + row.values
+    end
+  end
+end

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.3.12'
+  s.version     = '0.3.15'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

data/lib/csv_utils/csv_iterator.rb

@@ -44,7 +44,10 @@ class CSVUtils::CSVIterator
   end
 
   def headers
-    first.keys
+    @src_csv.rewind
+    headers = @src_csv.shift
+    strip_bom!(headers[0])
+    headers
   end
 
   def to_hash(key, value = nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.15
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-27 00:00:00.000000000 Z
+date: 2022-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -28,6 +28,7 @@ description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
+- csv-duplicate-finder
 - csv-explorer
 - csv-find-error
 - csv-readline
@@ -43,6 +44,7 @@ files:
 - LICENSE
 - README.md
 - bin/csv-change-eol
+- bin/csv-duplicate-finder
 - bin/csv-explorer
 - bin/csv-find-error
 - bin/csv-readline
@@ -78,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: CSV Utils