csv-utils 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98303ab9b2df05bc501c1c66b66a62be5ade9d79ab38a5b8bda8eb52d91b26cc
-  data.tar.gz: 8adfd2144220de2cc4f23136ee4eb7314a3c16eeac68be87e1dc19b1ac7dc350
+  metadata.gz: '019dcd269f036bc21e93019e567e8a0223d8436e87c56519d74af02383640bdf'
+  data.tar.gz: 34b4e8035a533e897c395943e892de0bae16fbdc3847a4990cc0281225d21bd4
 SHA512:
-  metadata.gz: a2a2b2067a9ca06920b171230a122eba479c1f91af3919e2965eaec6d073fff34d544221a92cffaa1b9546078960aee0c9b9031e7b652368e975cff9b196214c
-  data.tar.gz: '0786cfb3e75771ccb68bfa0e2cba42994c7c04a5c8be14432ae6467425536e7dfd4a4ef33403ae5bd129eafd871a07077610c000a78762052e0b055192c0cc16'
+  metadata.gz: e770276baa097fa30551266882910818f331890c6e9bfd7fa92ab01654826a14ad3b67f151f2ce858c816d6d628170174fcc872038636998c15c818dc129130a
+  data.tar.gz: a1689e7404f5d9b70f092b7cda83c8f200df0af1725d27ecc222d798bafcbb1ea1612cbff74dbef493fe427a12ff8330f7828ed608becb4a3d3cd7267179319e

data/bin/csv-validator

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'rchardet'
+
+def utf8?(str)
+  str
+    .force_encoding('utf-8')
+    .valid_encoding?
+end
+
+def convert_to_utf8(str, current_encoding)
+  str.force_encoding(current_encoding)
+  return nil unless str.valid_encoding?
+
+  str.encode('utf-8')
+end
+
+def detect_encoding(col)
+  CharDet.detect(col)['encoding']
+end
+
+csv = CSV.open(ARGV[0], 'rb')
+
+headers = csv.shift
+csv_lineno = 1
+
+while (row = csv.shift)
+  csv_lineno += 1
+
+  unless row.size == headers.size
+    $stderr.puts "row(#{csv_lineno}): invalid number of columns, expected #{headers.size} got #{row.size}"
+  end
+
+  row.each_with_index do |col, idx|
+    next if utf8?(col)
+
+    $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}): none UTF-8 characters found in \"#{col}\""
+    if (col_utf8_encoded = convert_to_utf8(col, detect_encoding(col)))
+      puts "row(#{csv_lineno}),col(#{idx + 1}): converted to UTF-8 from #{detect_encoding(col)} \"#{col_utf8_encoded}\""
+    else
+      $stderr.puts "row(#{csv_lineno}),col(#{idx + 1}): unknown character encoding"
+    end
+  end
+end
+
+csv.close

data/csv-utils.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.3.0'
+  s.version     = '0.3.1'
   s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Doug Youch
@@ -30,6 +30,7 @@ executables:
 - csv-change-eol
 - csv-find-error
 - csv-readline
+- csv-validator
 extensions: []
 extra_rdoc_files: []
 files:
@@ -43,6 +44,7 @@ files:
 - bin/csv-change-eol
 - bin/csv-find-error
 - bin/csv-readline
+- bin/csv-validator
 - csv-utils.gemspec
 - lib/csv-utils.rb
 - lib/csv_utils/csv_extender.rb