remote_table 2.0.1 → 2.0.2

data/CHANGELOG

@@ -1,3 +1,13 @@
+2.0.2 / 2012-06-20
+
+* Secrets revealed
+
+  * Indeed it is OK to omit the file:// as long as you are using an absolute path to a local file... e.g., RemoteTable.new('/path/to/foo')
+
+* Bug fixes
+
+  * Don't bork UTF-8 by incorrectly treating \xAD as a soft hyphen
+
 2.0.1 / 2012-05-16
 
 * Enhancements

data/lib/remote_table.rb

@@ -147,6 +147,9 @@ class RemoteTable
   # @example Local
   #   file:///Users/myuser/Desktop/holidays.csv
   #
+  # @example Local using an absolute path
+  #   /Users/myuser/Desktop/holidays.csv
+  #
   # @example Remote
   #   http://data.brighterplanet.com/countries.csv
   #

data/lib/remote_table/plaintext.rb

@@ -3,17 +3,30 @@ require 'unix_utils'
 
 class RemoteTable
   # Helper methods that act on plaintext files before they are parsed
-  module Plaintext      
-    CONSIDERED_HARMFUL = [
-      '\xef\xbb\xbf',   # UTF-8 byte order mark
-      '\xc2\xad',       # soft hyphen, often inserted by MS Office (html: ­)
-      '\xad'            # any remaining soft hyphens (sometimes seen in windows-1252)
-    ]
+  module Plaintext
+    class << self
+      # @private
+      # Code for the soft hyphen, often inserted by MS Office (html: &shy;)
+      def soft_hyphen(encoding)
+        case encoding
+        when /775/, /85[02578]/
+          '\xF0'
+        when /utf-?8/i
+          '\xc2\xad'
+        else # iso-8859-1, latin1, windows-1252, etc...
+          '\xad'
+        end
+      end
+    end
+
+    # UTF-8 byte order mark
+    UTF8_BOM = '\xef\xbb\xbf'
     EOL_TO_UNIX = 's/\r\n|\n|\r/\n/g'
 
     # Remove bytes that are both useless and harmful in the vast majority of cases.
     def delete_harmful!
-      local_copy.in_place :perl, "s/#{CONSIDERED_HARMFUL.join('//g; s/')}//g"
+      harmful = [ Plaintext.soft_hyphen(internal_encoding), UTF8_BOM ]
+      local_copy.in_place :perl, "s/#{harmful.join('//g; s/')}//g"
     end
     
     # No matter what the file encoding is SUPPOSED to be, run it through the system iconv binary to make sure it's UTF-8

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = "2.0.1"
+  VERSION = "2.0.2"
 end

data/test/support/airports.utf8.csv

@@ -0,0 +1,5 @@
+id,name,city,country_name,iata_code,icao_code,latitude,longitude,altitude,timezone,daylight_savings
+6060,"Capitán Av. German Quiroga G. Airport","San Borja","Bolivia","SRJ","SLSB",-14.8592,-66.7375,633,-4,"U"
+6058,"Capitán Av. Selin Zeitun Lopez Airport","Riberalta","Bolivia","RIB","SLRI",-11,-66,462,-4,"U"
+6056,"Capitán de Av. Emilio Beltrán Airport","Guayaramerín","Bolivia","GYA","SLGY",-10.8206,-65.3456,557,-4,"U"
+6054,"Obando Airport","Puerto Inírida","Colombia","PDA","SKPD",3.85,-67.91,460,-5,"U"

data/test/test_old_syntax.rb

@@ -44,7 +44,7 @@ describe RemoteTable do
     
     it "ignore UTF-8 byte order marks" do
       t = RemoteTable.new :url => 'http://www.freebase.com/type/exporttypeinstances/base/horses/horse_breed?page=0&filter_mode=type&filter_view=table&show%01p%3D%2Ftype%2Fobject%2Fname%01index=0&show%01p%3D%2Fcommon%2Ftopic%2Fimage%01index=1&show%01p%3D%2Fcommon%2Ftopic%2Farticle%01index=2&sort%01p%3D%2Ftype%2Fobject%2Ftype%01p%3Dlink%01p%3D%2Ftype%2Flink%2Ftimestamp%01index=false&=&exporttype=csv-8'
-      t.rows.first['name'].must_equal 'Tawleed'
+      t.rows.first['Name'].must_equal 'Tawleed'
     end
     
     # this will die with an error about libcurl if your curl doesn't support ssl

data/test/test_remote_table.rb

@@ -8,6 +8,17 @@ describe RemoteTable do
     t[5]["Requirements"].must_equal "Secure encryption of all data"
   end
 
+  it "doesn't screw up UTF-8" do
+    t = RemoteTable.new "file://#{File.expand_path('../support/airports.utf8.csv', __FILE__)}"
+    t[3]['city'].must_equal "Puerto Inírida"
+  end
+
+  it "likes paths as much as urls for local files" do
+    by_url = RemoteTable.new "file://#{File.expand_path('../support/airports.utf8.csv', __FILE__)}"
+    by_path = RemoteTable.new File.expand_path('../support/airports.utf8.csv', __FILE__)
+    by_path.rows.must_equal by_url.rows
+  end
+
   it "does its best to download urls without http://" do
     t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
     t[5]["Requirements"].must_equal "Secure encryption of all data"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: remote_table
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-16 00:00:00.000000000 Z
+date: 2012-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -159,6 +159,7 @@ files:
 - remote_table.gemspec
 - test/fixtures/data.yml
 - test/helper.rb
+- test/support/airports.utf8.csv
 - test/support/list-en1-semic-3.neooffice.binary.ods
 - test/support/list-en1-semic-3.neooffice.iso-8859-1.csv
 - test/support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64
@@ -198,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: remotetable
-rubygems_version: 1.8.21
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma
@@ -206,6 +207,7 @@ summary: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (com
 test_files:
 - test/fixtures/data.yml
 - test/helper.rb
+- test/support/airports.utf8.csv
 - test/support/list-en1-semic-3.neooffice.binary.ods
 - test/support/list-en1-semic-3.neooffice.iso-8859-1.csv
 - test/support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64