data_kit 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fdf7cb54e3359090864cc1f2021822a91b382ba
-  data.tar.gz: 065fc5738cad3b21dfd62a79474b326ee2722bc4
+  metadata.gz: a079b89a65b3f20db63058e070d6b12d19db436f
+  data.tar.gz: 78868c7945412096389eca3d05e15a495d73f08b
 SHA512:
-  metadata.gz: 4d0283c91997c819063e7f7b38aee1cba5a97f8f6c43acc1d321d50945a5728607e2bcf44ae994ba6d7731b46e0e016bc13d0ef765a49348c3f6a6987b6e1297
-  data.tar.gz: 5f8b249b33841efcd8d38e4a899ce7a4465b6dcd08c05d77bcb4f42f82bc24f9f957cf319c5cf8c84f83e8c84c2bea854e93206d1fee901f229bc8c690c989c5
+  metadata.gz: f1a80f34c222c89ee9af223796c06d50087dfddd9def99d4d1b6edd4c650ec6fd0f1dc8770dbf5c8f6c62433a8987e4f157e945ac9d0b188507a6e51feb67562
+  data.tar.gz: 8f4c7dac994fe348c69eba8b3b2a62105c33f8f4565efdc2effd3a4f41b3582f34b0b8ed8d86b89fa9a1986e1f152147563967f43e32426d255c5fb910728ec5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    data_kit (0.0.7)
+    data_kit (0.0.8)
       rcsv
       timeliness
 

data/lib/data_kit/csv/parser.rb

@@ -17,6 +17,7 @@ module DataKit
       def each_row(&block)
         handle.rewind
         Rcsv.parse(handle, :header => :skip, :columns => columns, :row_as_hash => true) do |row|
+          puts row.inspect
           yield row
         end
       end
@@ -39,7 +40,10 @@ module DataKit
           @handle = File.open(path)
         end
 
-        @handle.set_encoding(Encoding.find("UTF-8"))
+        @handle.set_encoding(
+          Encoding.find("BINARY"), Encoding.find("UTF-8"),
+          {:invalid => :replace, :undef => :replace, :replace => ''}
+        )
       end
 
       def set_headers

data/lib/data_kit/patches/rcsv.rb

@@ -32,15 +32,18 @@ class Rcsv
 
     initial_position = csv_data.pos
 
+    first_line = csv_data.each_line.first
+    field_count = first_line.split(raw_options[:col_sep]).length
+
     case options[:header]
     when :use
-      header = self.raw_parse(StringIO.new(csv_data.each_line.first), raw_options).first
+      header = self.raw_parse(StringIO.new(first_line), raw_options).first
       raw_options[:offset_rows] += 1
     when :skip
-      header = (0..(csv_data.each_line.first.split(raw_options[:col_sep]).count)).to_a
+      header = (0..field_count).to_a
       raw_options[:offset_rows] += 1
     when :none
-      header = (0..(csv_data.each_line.first.split(raw_options[:col_sep]).count)).to_a
+      header = (0..field_count).to_a
     end
 
     raw_options[:row_as_hash] = options[:row_as_hash] # Setting after header parsing

data/lib/data_kit/version.rb

@@ -1,3 +1,3 @@
 module DataKit
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/spec/csv/parser_spec.rb

@@ -9,6 +9,10 @@ describe DataKit::CSV::Parser do
     data_path('carriage_returns.csv')
   }
 
+  let(:vc_companies_path) {
+    data_path('vc_backed_companies.csv')
+  }
+
   it "should initialize" do
     csv = DataKit::CSV::Parser.new(path)
 
@@ -47,4 +51,15 @@ describe DataKit::CSV::Parser do
 
     count.should == 10
   end
+
+  it "should parse CSVs with unknown encodings" do
+    csv = DataKit::CSV::Parser.new(File.open(vc_companies_path))
+
+    count = 0
+    csv.each_row do |row|
+      count += 1
+    end
+
+    count.should == 2
+  end
 end

data/spec/fixtures/vc_backed_companies.csv

@@ -0,0 +1 @@
+Company Name,LOCATION,Still operating? Yes/No,Employees ,Phone,Email,LinkDin Profile,Website Url

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_kit
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Mode Analytics
@@ -155,6 +155,7 @@ files:
 - spec/fixtures/carriage_returns.csv
 - spec/fixtures/standard.csv
 - spec/fixtures/utf8.csv
+- spec/fixtures/vc_backed_companies.csv
 - spec/spec_helper.rb
 homepage: http://www.modeanalytics.com/
 licenses:
@@ -197,4 +198,5 @@ test_files:
 - spec/fixtures/carriage_returns.csv
 - spec/fixtures/standard.csv
 - spec/fixtures/utf8.csv
+- spec/fixtures/vc_backed_companies.csv
 - spec/spec_helper.rb