data_kit 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a079b89a65b3f20db63058e070d6b12d19db436f
-  data.tar.gz: 78868c7945412096389eca3d05e15a495d73f08b
+  metadata.gz: e62f0fdbbac2331ba30c312f2456e271d11ef544
+  data.tar.gz: 47f8462a6ee5e5d5e00e5a9d1d11b92b8ebc6506
 SHA512:
-  metadata.gz: f1a80f34c222c89ee9af223796c06d50087dfddd9def99d4d1b6edd4c650ec6fd0f1dc8770dbf5c8f6c62433a8987e4f157e945ac9d0b188507a6e51feb67562
-  data.tar.gz: 8f4c7dac994fe348c69eba8b3b2a62105c33f8f4565efdc2effd3a4f41b3582f34b0b8ed8d86b89fa9a1986e1f152147563967f43e32426d255c5fb910728ec5
+  metadata.gz: 4403a287ef848eabea8074fcdc7739d28021c8efc86b314cbeb817e57e565f3a601049d10a72c89f21fb071b37b1a6ebedaa09c0919aa6ec27ccbeb90e88b7b5
+  data.tar.gz: c6f27fc161251157dbafb00b28913d3eeb39ec9cc8947233db21476e38afa737392daa1ca872b0288476668cbd6f82905dddc7e15f4199fe2a3bf48b0b74f2dd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    data_kit (0.0.8)
+    data_kit (0.0.9)
       rcsv
       timeliness
 

data/lib/data_kit/converters/number.rb

@@ -17,11 +17,15 @@ module DataKit
 
         def reformat(value)
           if value.is_a?(String)
-            value.gsub(/(\p{Sc}|\,)/, '')
+            value.encode('UTF-8', encoding_opts).gsub(/(\p{Sc}|\,)/, '')
           else
             value
           end
         end
+
+        def encoding_opts
+          {:invalid => :replace, :undef => :replace, :replace => '?'}
+        end
       end
     end
   end

data/lib/data_kit/csv/parser.rb

@@ -17,7 +17,6 @@ module DataKit
       def each_row(&block)
         handle.rewind
         Rcsv.parse(handle, :header => :skip, :columns => columns, :row_as_hash => true) do |row|
-          puts row.inspect
           yield row
         end
       end

data/lib/data_kit/csv/schema_analyzer.rb

@@ -27,7 +27,6 @@ module DataKit
           if random.rand <= sampling_rate
             analysis.increment_sample
             row.keys.each do |field_name|
-              row[field_name].force_encoding('UTF-8')
               analysis.insert(field_name.to_s, row[field_name])
             end
           end

data/lib/data_kit/version.rb

@@ -1,3 +1,3 @@
 module DataKit
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/spec/converters/number_spec.rb

@@ -17,4 +17,9 @@ describe DataKit::Converters::Integer do
       DataKit::Converters::Number.convert(reformatted).should == result
     end
   end
+
+  it "should reformat strings with unknown encodings" do
+    str = "9350 Waxie WayÊSuite"
+    DataKit::Converters::Number.reformat(str).should == str
+  end
 end

data/spec/csv/converter_spec.rb

@@ -17,6 +17,10 @@ describe DataKit::CSV::Converter do
     DataKit::CSV::Parser.new(data_path('utf8.csv'))
   }
 
+  let(:asciicsv) {
+    DataKit::CSV::Parser.new(data_path('vc_backed_companies.csv'))
+  }
+
   it "should initialize and execute" do
     analysis = DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => 1)
     converter = DataKit::CSV::Converter.new(csv, analysis, target)
@@ -47,4 +51,15 @@ describe DataKit::CSV::Converter do
     CSV.open(target).each { |row| row_count += 1 }
     row_count.should == 11
   end
+
+  it "should convert rows with invalid UTF-8 characters" do
+    analysis = DataKit::CSV::SchemaAnalyzer.analyze(asciicsv, :sampling_rate => 1)
+    converter = DataKit::CSV::Converter.new(csv, analysis, target)
+
+    converter.execute
+
+    row_count = 0
+    CSV.open(target).each { |row| row_count += 1 }
+    row_count.should == 11
+  end
 end

data/spec/csv/schema_analyzer_spec.rb

@@ -52,9 +52,6 @@ describe DataKit::CSV::SchemaAnalyzer do
     analysis.row_count.should == 10
     analysis.sample_count.should be < 10
     analysis.use_type_hints.should == true
-
-    puts analysis.type_hints.inspect
-
   end
 
   it "should execute an analysis without type hints" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_kit
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Mode Analytics