data_kit 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eed0243b61997177b645f36dd755c121dd36c177
-  data.tar.gz: a9631c21ab9bf41820e8f4d28bec5bcf7a0c43b5
+  metadata.gz: dd32e91aed4ac35a8dbb104e577d9d7454025733
+  data.tar.gz: 31e8c8ef04ba50e94bffb8a29104362a14e56e08
 SHA512:
-  metadata.gz: b8468fc7bda9be6701312139e1d2cd3ad5933c0d1ddd0d38f4e9e373ba1dd27b176e1a1928c4a1790cd8d8e1873fdbecae9adc86266478f438595d0f8d32f1a0
-  data.tar.gz: 4f5dcdab6b2a42d05ba8a71bf38391b8109ba4adb6a46e5077f73dbfad796a9fd5706b5d07343a22e56aa438982bac5b27e889a964e4444b7327e3958a0340ed
+  metadata.gz: 77effe4f14b155f04c3f7d177a4ba0072cb02d558a2916bb66d8709484422e28e34153090b467a828dfc3ebf14d967d97f0b66cf5ba9a6c9ccbca14cc62952ab
+  data.tar.gz: e94f0b6657c0b7a105a6dd324df4ce8824c21d6acce11eee50a55973ef108873641e340597ef989cedda71043a10466e28f05540fa8a23ad6cb2176c8e2a002e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    data_kit (0.0.2)
+    data_kit (0.0.4)
       rcsv
       timeliness
 

data/lib/data_kit/csv/parser.rb

@@ -3,10 +3,6 @@ require 'rcsv'
 module DataKit
   module CSV
     class Parser
-      # Encode streams from BINARY into UTF-8
-      InternalEnc = Encoding.find("UTF-8")
-      ExternalEnc = Encoding.find("BINARY")
-
       attr_reader :path
       attr_reader :handle
       attr_reader :headers
@@ -43,7 +39,7 @@ module DataKit
           @handle = File.open(path)
         end
 
-        @handle.set_encoding(ExternalEnc, InternalEnc)
+        @handle.set_encoding(Encoding.find("UTF-8"))
       end
 
       def set_headers

data/lib/data_kit/csv/schema_analyzer.rb

@@ -20,6 +20,7 @@ module DataKit
           if random.rand <= sampling_rate
             analysis.increment_sample
             row.keys.each do |field_name|
+              row[field_name].force_encoding('UTF-8')
               analysis.insert(field_name.to_s, row[field_name])
             end
           end

data/lib/data_kit/version.rb

@@ -1,3 +1,3 @@
 module DataKit
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/spec/csv/converter_spec.rb

@@ -13,6 +13,10 @@ describe DataKit::CSV::Converter do
     DataKit::CSV::Parser.new(data_path('standard.csv'))
   }
 
+  let(:utf8csv) {
+    DataKit::CSV::Parser.new(data_path('utf8.csv'))
+  }
+
   it "should initialize and execute" do
     analysis = DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => 1)
     converter = DataKit::CSV::Converter.new(csv, analysis, target)
@@ -24,7 +28,7 @@ describe DataKit::CSV::Converter do
     row_count.should == 11
   end
 
-  it "should convert using the convience method" do
+  it "should convert using the convenience method" do
     analysis = DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => 1)
     converter = DataKit::CSV::Converter.convert(csv, analysis, target)
 
@@ -32,4 +36,15 @@ describe DataKit::CSV::Converter do
     CSV.open(target).each { |row| row_count += 1 }
     row_count.should == 11
   end
+
+  it "should convert rows with utf8 characters" do
+    analysis = DataKit::CSV::SchemaAnalyzer.analyze(utf8csv, :sampling_rate => 1)
+    converter = DataKit::CSV::Converter.new(csv, analysis, target)
+
+    converter.execute
+
+    row_count = 0
+    CSV.open(target).each { |row| row_count += 1 }
+    row_count.should == 11
+  end
 end

data/spec/fixtures/utf8.csv

@@ -0,0 +1,100 @@
+id,first_name,last_name,email,country,ip_address
+1,Juan,Powell,jpowell@gabtune.mil,Belize,174.115.161.126
+2,Elizabeth,Thomas,ethomas@vidoo.biz,San Marino,237.239.51.102
+3,Andrea,Nelson,anelson@yakijo.org,Macedonia,5.158.57.111
+4,Matthew,Jones,mjones@photofeed.com,Chad,183.112.197.55
+5,Patrick,Wright,pwright@oyoyo.edu,Oman,164.108.233.48
+6,Sharon,Riley,sriley@rhybox.net,Czech Republic,166.230.65.220
+7,Albert,Arnold,aarnold@gigashots.net,Cook Islands,114.85.247.243
+8,John,Scott,jscott@yambee.name,Georgia,94.199.125.106
+9,Philip,Reynolds,preynolds@skipstorm.mil,Comoros,10.215.98.115
+10,Mark,Anderson,manderson@browsebug.info,Spain,42.191.208.43
+11,Paula,Gomez,pgomez@edgeify.net,Montserrat,19.144.148.158
+12,Antonio,Cook,acook@jaxspan.info,Saint Martin,120.195.173.63
+13,Terry,Duncan,tduncan@izio.name,Bangladesh,91.238.167.163
+14,Anthony,Russell,arussell@zoomcast.com,Albania,34.18.207.137
+15,Ruby,Lane,rlane@pixope.org,"Palestinian Territory, Occupied",195.14.7.172
+16,Robin,Spencer,rspencer@leenti.mil,Panama,184.7.185.174
+17,Melissa,Washington,mwashington@katz.com,"Korea, North",224.91.93.188
+18,Barbara,Nelson,bnelson@dablist.gov,Cambodia,192.247.213.41
+19,Janet,Mason,jmason@wikizz.mil,Guinea-Bissau,30.17.150.124
+20,Michelle,Green,mgreen@twiyo.biz,United States Virgin Islands,22.139.120.215
+21,David,Smith,dsmith@quinu.net,United States Virgin Islands,148.157.14.156
+22,Anna,Martin,amartin@zoonoodle.gov,Cameroon,123.80.1.226
+23,Judith,Holmes,jholmes@mudo.gov,Jamaica,110.154.217.110
+24,Carolyn,Riley,criley@meemm.gov,Ethiopia,246.252.115.105
+25,Carolyn,Lynch,clynch@voonyx.org,South Africa,133.159.182.203
+26,Kimberly,Torres,ktorres@bluezoom.biz,Armenia,3.201.239.96
+27,Paula,Andrews,pandrews@brainbox.com,Turks and Caicos Islands,161.2.154.206
+28,Joe,Ramos,jramos@kimia.info,Australia,189.102.57.100
+29,Shirley,Fowler,sfowler@ntag.gov,Mexico,144.132.244.41
+30,Louise,Morales,lmorales@flashspan.org,Mali,177.85.64.237
+31,Patrick,Vasquez,pvasquez@oozz.com,Jordan,52.197.37.130
+32,Gloria,Carpenter,gcarpenter@jabbercube.net,Seychelles,74.26.218.240
+33,Bruce,Anderson,banderson@yabox.com,Mali,2.51.248.44
+34,Richard,Hunter,rhunter@brightbean.name,Guam,57.38.118.134
+35,Keith,Kennedy,kkennedy@pixoboo.mil,Vietnam,253.240.67.38
+36,Charles,Ramirez,cramirez@meejo.org,Malawi,148.110.48.106
+37,Lawrence,Willis,lwillis@devpulse.net,Åland,226.179.18.113
+38,Arthur,Frazier,afrazier@jabbertype.gov,Jamaica,191.8.127.69
+39,Joseph,Jordan,jjordan@brainverse.edu,Belarus,35.133.77.128
+40,Steve,Robertson,srobertson@yata.gov,Guinea-Bissau,214.196.72.59
+41,Christine,Williams,cwilliams@dablist.name,Grenada,207.37.134.135
+42,Brandon,Payne,bpayne@oyondu.net,Seychelles,103.172.50.228
+43,Charles,Fernandez,cfernandez@zava.edu,Palau,156.40.43.254
+44,Lisa,James,ljames@dazzlesphere.org,Mayotte,88.191.96.192
+45,Helen,Stephens,hstephens@devbug.edu,Nauru,65.240.201.247
+46,Donna,Barnes,dbarnes@npath.edu,Guyana,53.79.21.160
+47,Willie,Berry,wberry@chatterbridge.edu,Ascension Island,27.173.100.171
+48,Jerry,Fernandez,jfernandez@realcube.edu,China,54.147.218.106
+49,Jessica,Stewart,jstewart@gabvine.biz,Djibouti,220.164.122.4
+50,Paul,Stevens,pstevens@skalith.com,Qatar,181.118.119.183
+51,Jeffrey,Frazier,jfrazier@oyoba.info,Saint Martin,247.167.19.66
+52,Emily,Bell,ebell@lazz.info,Venezuela,56.23.2.77
+53,Wayne,Banks,wbanks@dabshots.org,Denmark,210.58.240.119
+54,Helen,Stone,hstone@mita.gov,Belgium,25.191.200.47
+55,Laura,Stone,lstone@oloo.com,Yugoslavia,98.117.184.246
+56,Amanda,Ward,award@yotz.com,Estonia,197.191.111.153
+57,Carlos,Duncan,cduncan@einti.edu,Moldova,88.215.57.115
+58,Catherine,Holmes,cholmes@photojam.gov,Rwanda,171.27.114.159
+59,Janice,Hernandez,jhernandez@kwimbee.biz,Qatar,180.13.57.100
+60,Judith,Vasquez,jvasquez@janyx.biz,Niue,151.169.187.79
+61,Dennis,Henderson,dhenderson@browsedrive.gov,Mozambique,220.48.212.254
+62,Jeremy,Ford,jford@meezzy.biz,Sudan,248.135.67.205
+63,Dennis,Hunter,dhunter@rooxo.net,Latvia,176.187.214.230
+64,Stephanie,Montgomery,smontgomery@babbleblab.net,France,137.214.192.40
+65,Henry,Olson,holson@tazz.info,Puerto Rico,176.207.8.142
+66,Jesse,Edwards,jedwards@dabfeed.mil,Argentina,147.16.35.23
+67,Randy,Peterson,rpeterson@wikizz.biz,Antigua and Barbuda,179.119.53.133
+68,Joan,Holmes,jholmes@mydo.name,United States Virgin Islands,0.59.105.147
+69,Jane,Vasquez,jvasquez@edgepulse.name,Bosnia and Herzegovina,179.67.30.193
+70,Heather,Hernandez,hhernandez@feedfish.com,Falkland Islands (Malvinas),154.132.147.163
+71,Linda,Webb,lwebb@feedfire.name,Gabon,118.24.235.210
+72,Richard,Daniels,rdaniels@jetwire.org,Jordan,42.10.7.222
+73,Terry,Fernandez,tfernandez@twitterbeat.org,Cook Islands,173.98.100.195
+74,Virginia,Willis,vwillis@jabberbean.biz,Argentina,130.235.152.57
+75,Richard,Austin,raustin@livefish.com,Zambia,167.130.14.74
+76,Carol,Griffin,cgriffin@fatz.mil,Grenada,1.189.205.12
+77,Brandon,Gutierrez,bgutierrez@zoomcast.edu,Vatican City State (Holy See),37.247.25.100
+78,Samuel,Ellis,sellis@yodo.net,Pakistan,10.183.35.75
+79,Stephanie,Fields,sfields@podcat.gov,Finland,133.98.9.101
+80,Rebecca,Kim,rkim@brainlounge.mil,Guadeloupe,155.122.101.233
+81,Debra,Warren,dwarren@dazzlesphere.org,Cameroon,41.110.64.145
+82,Brenda,Henderson,bhenderson@jetpulse.mil,Swaziland,216.4.186.151
+83,Victor,Hughes,vhughes@vipe.name,Macedonia,144.105.122.32
+84,Gerald,Romero,gromero@quinu.mil,France,114.177.65.254
+85,Raymond,Gomez,rgomez@realbridge.edu,Egypt,26.255.191.232
+86,Anne,Chavez,achavez@roodel.biz,Croatia,193.55.29.211
+87,Patrick,Willis,pwillis@dynabox.com,Georgia,156.207.126.24
+88,Alice,Gardner,agardner@aivee.info,Solomon Islands,71.46.147.126
+89,Jane,Harvey,jharvey@cogibox.edu,Svalbard and Jan Mayen Islands,25.33.240.168
+90,Rachel,Brown,rbrown@rhyloo.com,Equatorial Guinea,220.113.103.62
+91,Brenda,Hudson,bhudson@wikizz.biz,Brazil,67.187.212.52
+92,Lisa,Johnston,ljohnston@devshare.info,Qatar,13.43.69.191
+93,Shawn,Gonzales,sgonzales@fatz.gov,Sri Lanka,162.236.113.63
+94,Christine,Stanley,cstanley@voonyx.biz,American Samoa,77.123.162.28
+95,Annie,Thomas,athomas@ntags.mil,Guadeloupe,91.226.131.120
+96,Wayne,Rodriguez,wrodriguez@topicshots.com,Antarctica,238.217.34.126
+97,Daniel,Hall,dhall@skippad.edu,Gibraltar,231.242.152.83
+98,Virginia,Watson,vwatson@chatterpoint.mil,Northern Mariana Islands,188.211.155.117
+99,Benjamin,Henderson,bhenderson@zoovu.com,Netherlands Antilles,192.19.128.146

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_kit
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Mode Analytics
@@ -154,6 +154,7 @@ files:
 - spec/dataset/schema_spec.rb
 - spec/fixtures/carriage_returns.csv
 - spec/fixtures/standard.csv
+- spec/fixtures/utf8.csv
 - spec/spec_helper.rb
 homepage: http://www.modeanalytics.com/
 licenses:
@@ -195,5 +196,5 @@ test_files:
 - spec/dataset/schema_spec.rb
 - spec/fixtures/carriage_returns.csv
 - spec/fixtures/standard.csv
+- spec/fixtures/utf8.csv
 - spec/spec_helper.rb
-has_rdoc: