csv-utils 0.3.2 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b3f6bdded232bf3be2009d4bbb5ab99e083cd9f48dff5fabe89324f5217550a
-  data.tar.gz: 86b130f177d173a74bc1611c5677cc6bad5053e953a1a5fabfeabee494e372c8
+  metadata.gz: '0248ce0ef1db03a751c864fdf52194b25a21f21ef90cbeaf8d0ae07075cbaf51'
+  data.tar.gz: c48e4f907963b1ace89fecb095f2a7ee17ec1670db2371dd4e015005800b757b
 SHA512:
-  metadata.gz: dd0a299cbe4b153f122d605bbb1b2ab08726194d58010e909090048f1cef0e384d08c8a3a59683b8108c1dc3766deacac46a9063af35e57502f6fe3c92b855ff
-  data.tar.gz: e4233e0c38338d24a6105d465745092146a65e03f297442fabada13ecf380db70740196726ec2d40a8f92828b6cfbc6945c0274701091cb2687af549126e34fa
+  metadata.gz: 848fa57de005d00bdf24cd07b533680cd9307c07a19b2fb955391737e36407dfc6be22edef55742fb0c90e8ed5273d4c584840e7d7a657404c1242947a88692b
+  data.tar.gz: 71209d88db0d67ed606e4db56e3cf667081074523373951fea44698537af0661cf63c79eb9b971419bb13bfc2a8b1b9c10c2f89e911b913b4f6c4bc8d00ee061

data/bin/csv-explorer

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'csv-utils'
+require 'irb'
+
+CSV_FILE = ARGV.shift
+
+def csv
+  @csv ||= CSVUtils::CSVIterator.new(CSV_FILE)
+end
+
+IRB.start

data/bin/csv-find-error

@@ -3,10 +3,12 @@
 require 'csv'
 require 'shellwords'
 
+prev_row = nil
 begin
-  CSV.open(ARGV[0], 'rb').each { }
+  CSV.open(ARGV[0], 'rb').each { |row| prev_row = row }
 rescue CSV::MalformedCSVError => e
   puts e.class.to_s + ': ' + e.message
+  puts "previous row was #{prev_row}"
   if e.message =~ /line (\d+)/
     lineno = $1.to_i
     cmd = "csv-readline #{Shellwords.escape(ARGV[0])} #{lineno}"

data/bin/csv-validator

@@ -25,11 +25,32 @@ def detect_encoding(col)
   CharDet.detect(col)['encoding']
 end
 
+def strip_bom!(col)
+  col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+end
+
 csv = CSV.open(ARGV[0], 'rb')
-out = CSV.open(ARGV[1], 'wb') if ARGV[1]
+id_column_name = ARGV[1]
 
 headers = csv.shift
-out << headers if out
+strip_bom!(headers[0])
+
+id_column_num = nil
+if id_column_name
+  unless headers.include?(id_column_name)
+    $stderr.puts("header #{id_column_name} not found in current set of headers")
+    exit 1
+  end
+
+  id_column_num = headers.index(id_column_name)
+end
+
+out = nil
+if id_column_num
+  out = CSV.open('utf8-correctsion.csv', 'wb')
+  out << [id_column_name, 'Row', 'Col', 'Header', 'Value']
+end
+
 csv_lineno = 1
 
 while (row = csv.shift)
@@ -39,21 +60,17 @@ while (row = csv.shift)
     $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
   end
 
-  converted = false
   row.each_with_index do |col, idx|
-    next if utf8?(col)
+    next if col.nil? || utf8?(col)
 
-    $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}): none UTF-8 characters found in \"#{col}\""
+    $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: none UTF-8 characters found in \"#{col}\""
     if (col_utf8_encoded = convert_to_utf8(col, detect_encoding(col)))
-      converted = true
-      puts "row(#{csv_lineno}),col(#{idx + 1}): converted to UTF-8 from #{detect_encoding(col)} \"#{col_utf8_encoded}\""
-      row[idx] = col_utf8_encoded
+      puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: converted to UTF-8 from #{detect_encoding(col)} \"#{col_utf8_encoded}\""
+      out << [row[id_column_num], csv_lineno, (idx + 1), headers[idx], col_utf8_encoded]
     else
-      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}): unknown character encoding"
+      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}) #{headers[idx]}: unknown character encoding"
     end
   end
-
-  out << row if out && converted
 end
 
 csv.close

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.3.2'
+  s.version     = '0.3.7'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

data/lib/csv-utils.rb

@@ -3,6 +3,7 @@ require 'csv'
 # Collection of tools for working with CSV files.
 module CSVUtils
   autoload :CSVExtender, 'csv_utils/csv_extender'
+  autoload :CSVIterator, 'csv_utils/csv_iterator'
   autoload :CSVOptions, 'csv_utils/csv_options'
   autoload :CSVReport, 'csv_utils/csv_report'
   autoload :CSVRow, 'csv_utils/csv_row'

data/lib/csv_utils/csv_iterator.rb

@@ -0,0 +1,51 @@
+# Search a CSV given a series of steps
+class CSVUtils::CSVIterator
+  include Enumerable
+
+  attr_reader :prev_row
+
+  class RowWrapper < Hash
+    attr_accessor :lineno
+
+    def self.create(headers, row, lineno)
+      row_wrapper = RowWrapper[headers.zip(row)]
+      row_wrapper.lineno = lineno
+      row_wrapper
+    end
+
+    def to_pretty_s
+      reject { |_, v| v.strip.empty? }
+        .each_with_index
+        .map { |(k, v), idx| sprintf('  %-3d %s: %s', idx+1, k, v) }
+        .join("\n") + "\n"
+    end
+  end
+
+  def initialize(src_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+  end
+
+  def each(headers = nil)
+    @src_csv.rewind
+
+    lineno = 0
+    unless headers
+      headers = @src_csv.shift
+      strip_bom!(headers[0])
+      lineno += 1
+    end
+
+    @prev_row = nil
+    while (row = @src_csv.shift)
+      lineno += 1
+      yield RowWrapper.create(headers, row, lineno)
+      @prev_row = row
+    end
+  end
+
+  private
+
+  def strip_bom!(col)
+    col.sub!("\xEF\xBB\xBF".force_encoding('ASCII-8BIT'), '')
+  end
+end

data/lib/csv_utils/csv_report.rb

@@ -21,7 +21,7 @@ module CSVUtils
     def generate(headers = nil)
       add_headers(headers) if headers
       yield self
-      @csv.close if @must_close
+      close if @must_close
     end
 
     def append(csv_row)
@@ -37,5 +37,9 @@ module CSVUtils
     def add_headers(csv_row)
       append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
     end
+
+    def close
+      @csv.close
+    end
   end
 end

data/lib/csv_utils/csv_wrapper.rb

@@ -35,6 +35,10 @@ class CSVUtils::CSVWrapper
     csv.shift
   end
 
+  def rewind
+    csv.rewind
+  end
+
   def close
     csv.close if close_when_done?
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.7
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inheritance-helper
@@ -28,6 +28,7 @@ description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
 - csv-change-eol
+- csv-explorer
 - csv-find-error
 - csv-readline
 - csv-validator
@@ -42,12 +43,14 @@ files:
 - LICENSE
 - README.md
 - bin/csv-change-eol
+- bin/csv-explorer
 - bin/csv-find-error
 - bin/csv-readline
 - bin/csv-validator
 - csv-utils.gemspec
 - lib/csv-utils.rb
 - lib/csv_utils/csv_extender.rb
+- lib/csv_utils/csv_iterator.rb
 - lib/csv_utils/csv_options.rb
 - lib/csv_utils/csv_report.rb
 - lib/csv_utils/csv_row.rb