csv-utils 0.3.4 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e649e8e220856a0676e01ef58811ab9df3822a1757eae4e776a45256063c510
-  data.tar.gz: fad8bcac595659bc5d91b4ebc19ee8f18ebbdf910568e75ff812768e1c23515f
+  metadata.gz: b5b6f30da64b899586ef7d801904b8da9b01b6053f9d975f0702edcc5a7b65e0
+  data.tar.gz: e7e5006c6b63a9b8472e4658c748071619d09adf0ca73a84109b43b8862f1bcd
 SHA512:
-  metadata.gz: 4e4a26c3494c466a0099f72db534b8bd73910444c6eadf1b1faa78213d225b6752e9e8b60818bbeabe57f4542bd235da1969a9b6a3bad968328622531f89f077
-  data.tar.gz: d35d7a46f4f58a0fe87e10c6bcfec3ededc840d5d3c7ce0f1e5162e22f6f9ddb1516ed23a59b93877529ae326d597dfbde5089d9676393d488e476f6a9ad0924
+  metadata.gz: 7ded9a4318f44f77b6c2ccf72a844a7cec734745052c62be5e19c2dbadc8e989b3c89528058fabfd63cbcb59a1575293b9cc87a343bb4d68dbad3051ba04616d
+  data.tar.gz: dd8f0d7cd953eee05489423d0ef55e0b486500d2499d442d7d523636ced12858a2e9cec7ad60ce9d0126516121674d3c1e2a879ea89bb3a93cf7100709204b9d

data/.ruby-version

@@ -1 +1 @@
-2.6.3
+3.0.0

data/Gemfile.lock

@@ -1,50 +1,52 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    ast (2.4.1)
-    diff-lcs (1.3)
-    docile (1.3.2)
+    ast (2.4.2)
+    diff-lcs (1.4.4)
+    docile (1.3.5)
     inheritance-helper (0.1.5)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
     rainbow (3.0.0)
-    rake (13.0.1)
-    regexp_parser (1.7.1)
+    rake (13.0.3)
+    regexp_parser (2.1.1)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.86.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.11.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.0.3)
-      parser (>= 2.7.0.1)
-    ruby-progressbar (1.10.1)
-    simplecov (0.18.5)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-    simplecov-html (0.12.2)
-    unicode-display_width (1.7.0)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
+    unicode-display_width (2.0.0)
 
 PLATFORMS
-  ruby
+  x86_64-darwin-20
 
 DEPENDENCIES
   inheritance-helper
@@ -54,4 +56,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   1.17.3
+   2.2.3

data/bin/csv-explorer

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'csv-utils'
+require 'irb'
+
+CSV_FILE = ARGV.shift
+
+def csv
+  @csv ||= CSVUtils::CSVIterator.new(CSV_FILE)
+end
+
+IRB.start

data/bin/csv-find-error

@@ -3,10 +3,12 @@
 require 'csv'
 require 'shellwords'
 
+prev_row = nil
 begin
-  CSV.open(ARGV[0], 'rb').each { }
+  CSV.open(ARGV[0], 'rb').each { |row| prev_row = row }
 rescue CSV::MalformedCSVError => e
   puts e.class.to_s + ': ' + e.message
+  puts "previous row was #{prev_row}"
   if e.message =~ /line (\d+)/
     lineno = $1.to_i
     cmd = "csv-readline #{Shellwords.escape(ARGV[0])} #{lineno}"

data/bin/csv-validator

@@ -35,20 +35,24 @@ id_column_name = ARGV[1]
 headers = csv.shift
 strip_bom!(headers[0])
 
-id_column_num = nil
-if id_column_name
-  unless headers.include?(id_column_name)
-    $stderr.puts("header #{id_column_name} not found in current set of headers")
-    exit 1
-  end
-
-  id_column_num = headers.index(id_column_name)
+id_column_name ||= headers[0]
+unless headers.include?(id_column_name)
+  $stderr.puts("header #{id_column_name} not found in current set of headers")
+  exit 1
 end
 
+id_column_num = headers.index(id_column_name)
+
 out = nil
-if id_column_num
-  out = CSV.open('utf8-correctsion.csv', 'wb')
-  out << [id_column_name, 'Row', 'Col', 'Header', 'Value']
+out_proc = Proc.new do |row|
+  out ||=
+    begin
+      out = CSV.open('utf8-correctsion.csv', 'wb')
+      out << [id_column_name, 'Row', 'Col', 'Header', 'Value']
+      out
+    end
+
+  out << row
 end
 
 csv_lineno = 1
@@ -61,12 +65,12 @@ while (row = csv.shift)
   end
 
   row.each_with_index do |col, idx|
-    next if utf8?(col)
+    next if col.nil? || utf8?(col)
 
     $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: none UTF-8 characters found in \"#{col}\""
     if (col_utf8_encoded = convert_to_utf8(col, detect_encoding(col)))
       puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{detect_encoding(col)} \"#{col_utf8_encoded}\""
-      out << [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
+      out_proc.call [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
     else
       $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
     end

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.3.4'
+  s.version     = '0.3.9'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

data/lib/csv-utils.rb

@@ -2,7 +2,9 @@ require 'csv'
 
 # Collection of tools for working with CSV files.
 module CSVUtils
+  autoload :CSVCompare, 'csv_utils/csv_compare'
   autoload :CSVExtender, 'csv_utils/csv_extender'
+  autoload :CSVIterator, 'csv_utils/csv_iterator'
   autoload :CSVOptions, 'csv_utils/csv_options'
   autoload :CSVReport, 'csv_utils/csv_report'
   autoload :CSVRow, 'csv_utils/csv_row'

data/lib/csv_utils/csv_compare.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# CSVUtils::CSVCompare purpose is to determine which rows in the secondary_data_file need to be created, deleted or updated
+# **requires both CSV files to be sorted on the same columns, CSVUtils::CSVSort can accomplish this
+# In order to receive updates, update_comparison_columns must configured or use inheritance and change the update_row? method
+class CSVUtils::CSVCompare
+  # primary_data_file is the source of truth
+  # compare_proc used to compare the id column(s)
+  # update_comparison_columns column(s) to compare for equality, ex: updated_at, timestamp, hash
+  #  caveat: update_comparison_columns need to be in both csv files
+  attr_reader :primary_data_file,
+              :update_comparison_columns,
+              :compare_proc
+
+  def initialize(primary_data_file, update_comparison_columns=nil, &block)
+    @primary_data_file = primary_data_file
+    @update_comparison_columns = update_comparison_columns
+    @compare_proc = block
+  end
+
+  def compare(secondary_data_file)
+    src = CSV.open(primary_data_file)
+    src_headers = src.shift
+    dest = CSV.open(secondary_data_file)
+    dest_headers = dest.shift
+
+    read_next_src = true
+    read_next_dest = true
+
+    while(!src.eof? || !dest.eof?)
+      src_record = next_record_from_file(src_headers, src) if read_next_src
+      dest_record = next_record_from_file(dest_headers, dest) if read_next_dest
+
+      if ! src_record
+        read_next_src = false
+        read_next_dest = true
+
+        yield :delete, dest_record
+      elsif ! dest_record
+        read_next_src = true
+        read_next_dest = false
+
+        yield :create, src_record
+      elsif compare_proc.call(src_record, dest_record) == 0
+        read_next_src = true
+        read_next_dest = true
+
+        yield(:update, src_record) if update_row?(src_record, dest_record)
+      elsif compare_proc.call(src_record, dest_record) > 0
+        read_next_src = false
+        read_next_dest = true
+
+        yield :delete, dest_record
+      else
+        read_next_src = true
+        read_next_dest = false
+
+        yield :create, src_record
+      end
+    end
+
+    src.close
+    dest.close
+  end
+
+  private
+
+  def next_record_from_file(headers, file)
+    return nil if file.eof?
+
+    Hash[headers.zip(file.shift)]
+  end
+
+  def update_row?(src_record, dest_record)
+    return false unless update_comparison_columns
+
+    update_comparison_columns.each do |column_name|
+      return true unless src_record[column_name] == dest_record[column_name]
+    end
+
+    false
+  end
+end

data/lib/csv_utils/csv_iterator.rb

@@ -0,0 +1,51 @@
+# Search a CSV given a series of steps
+class CSVUtils::CSVIterator
+  include Enumerable
+
+  attr_reader :prev_row
+
+  class RowWrapper < Hash
+    attr_accessor :lineno
+
+    def self.create(headers, row, lineno)
+      row_wrapper = RowWrapper[headers.zip(row)]
+      row_wrapper.lineno = lineno
+      row_wrapper
+    end
+
+    def to_pretty_s
+      reject { |_, v| v.strip.empty? }
+        .each_with_index
+        .map { |(k, v), idx| sprintf('  %-3d %s: %s', idx+1, k, v) }
+        .join("\n") + "\n"
+    end
+  end
+
+  def initialize(src_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+  end
+
+  def each(headers = nil)
+    @src_csv.rewind
+
+    lineno = 0
+    unless headers
+      headers = @src_csv.shift
+      strip_bom!(headers[0])
+      lineno += 1
+    end
+
+    @prev_row = nil
+    while (row = @src_csv.shift)
+      lineno += 1
+      yield RowWrapper.create(headers, row, lineno)
+      @prev_row = row
+    end
+  end
+
+  private
+
+  def strip_bom!(col)
+    col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+  end
+end

data/lib/csv_utils/csv_report.rb

@@ -9,19 +9,20 @@ module CSVUtils
         if csv.is_a?(String)
           @must_close = true
           mode = csv_options.delete(:mode) || 'wb'
-          CSV.open(csv, mode, csv_options)
+          CSV.open(csv, mode, **csv_options)
         else
           @must_close = false
           csv
         end
 
-      generate(headers, &block) if block
+      add_headers(headers) if headers
+
+      generate(&block) if block
     end
 
-    def generate(headers = nil)
-      add_headers(headers) if headers
+    def generate
       yield self
-      @csv.close if @must_close
+      close if @must_close
     end
 
     def append(csv_row)
@@ -37,5 +38,9 @@ module CSVUtils
     def add_headers(csv_row)
       append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
     end
+
+    def close
+      @csv.close
+    end
   end
 end

data/lib/csv_utils/csv_sort.rb

@@ -25,9 +25,9 @@ class CSVUtils::CSVSort
   private
 
   def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
-    src1 = CSV.open(src_csv_file1, 'rb', csv_options)
-    src2 = CSV.open(src_csv_file2, 'rb', csv_options)
-    dest = CSV.open(dest_csv_file, 'wb', csv_options)
+    src1 = CSV.open(src_csv_file1, 'rb', **csv_options)
+    src2 = CSV.open(src_csv_file2, 'rb', **csv_options)
+    dest = CSV.open(dest_csv_file, 'wb', **csv_options)
 
     if @headers
       dest << @headers
@@ -66,7 +66,7 @@ class CSVUtils::CSVSort
   end
 
   def create_sorted_csv_part_files(batch_size, &block)
-    src = CSV.open(csv_file, 'rb', csv_options)
+    src = CSV.open(csv_file, 'rb', **csv_options)
 
     @headers = src.shift if has_headers
 
@@ -74,7 +74,7 @@ class CSVUtils::CSVSort
     create_batch_part_proc = Proc.new do
       batch.sort!(&block)
       @csv_part_files << "#{new_csv_file}.part.#{@csv_part_files.size}"
-      CSV.open(@csv_part_files.last, 'wb', csv_options) do |csv|
+      CSV.open(@csv_part_files.last, 'wb', **csv_options) do |csv|
         csv << @headers if @headers
         batch.each { |row| csv << row }
       end
@@ -107,6 +107,6 @@ class CSVUtils::CSVSort
       File.unlink(csv_part_file2)
     end
 
-    FileUtils.mv(@csv_part_files.last, new_csv_file)
+    FileUtils.mv(@csv_part_files.last || @csv_file, new_csv_file)
   end
 end

data/lib/csv_utils/csv_wrapper.rb

@@ -20,7 +20,7 @@ class CSVUtils::CSVWrapper
   def open(csv, mode, csv_options)
     if csv.is_a?(String)
       @close_when_done = true
-      @csv = CSV.open(csv, mode, csv_options)
+      @csv = CSV.open(csv, mode, **csv_options)
     else
       @close_when_done = false
       @csv = csv
@@ -35,6 +35,10 @@ class CSVUtils::CSVWrapper
     csv.shift
   end
 
+  def rewind
+    csv.rewind
+  end
+
   def close
     csv.close if close_when_done?
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.9
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2021-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -28,6 +28,7 @@ description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
+- csv-explorer
 - csv-find-error
 - csv-readline
 - csv-validator
@@ -42,12 +43,15 @@ files:
 - LICENSE
 - README.md
 - bin/csv-change-eol
+- bin/csv-explorer
 - bin/csv-find-error
 - bin/csv-readline
 - bin/csv-validator
 - csv-utils.gemspec
 - lib/csv-utils.rb
+- lib/csv_utils/csv_compare.rb
 - lib/csv_utils/csv_extender.rb
+- lib/csv_utils/csv_iterator.rb
 - lib/csv_utils/csv_options.rb
 - lib/csv_utils/csv_report.rb
 - lib/csv_utils/csv_row.rb
@@ -74,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: CSV Utils