remote_table 0.2.22 → 0.2.23

data/VERSION

@@ -1 +1 @@
-0.2.22
+0.2.23

data/lib/remote_table/file/csv.rb

@@ -3,6 +3,7 @@ class RemoteTable
     def each_row(&block)
       backup_file!
       convert_file_to_utf8!
+      remove_useless_characters!
       skip_rows!
       FasterCSV.foreach(path, fastercsv_options) do |row|
         ordered_hash = ActiveSupport::OrderedHash.new

data/lib/remote_table/file/fixed_width.rb

@@ -3,6 +3,7 @@ class RemoteTable
     def each_row(&block)
       backup_file!
       convert_file_to_utf8!
+      remove_useless_characters!
       crop_rows!
       skip_rows!
       cut_columns!

data/lib/remote_table/file/html.rb

@@ -3,6 +3,7 @@ class RemoteTable
     def each_row(&block)
       backup_file!
       convert_file_to_utf8!
+      remove_useless_characters!
       html_headers = (headers.is_a?(Array)) ? headers : nil
       Nokogiri::HTML(unescaped_html_without_soft_hyphens, nil, 'UTF-8').xpath(row_xpath).each do |row|
         values = row.xpath(column_xpath).map { |td| td.content.gsub(/\s+/, ' ').strip }
@@ -29,7 +30,7 @@ class RemoteTable
     # should we be doing this in ruby?
     def unescaped_html_without_soft_hyphens
       str = CGI.unescapeHTML IO.read(path)
-      str.gsub! /­|\302\255/, ''
+      str.gsub! /­/, ''
       str
     end
   end

data/lib/remote_table/file.rb

@@ -63,8 +63,16 @@ class RemoteTable
       FileUtils.mv "#{path}.tmp", path
     end
     
+    USELESS_CHARACTERS = [
+      '\xef\xbb\xbf',   # UTF-8 byte order mark
+      '\xc2\xad'        # soft hyphen, often inserted by MS Office (html: ­)
+    ]
+    def remove_useless_characters!
+      RemoteTable.backtick_with_reporting "perl -pe 's/#{USELESS_CHARACTERS.join '//g; s/'}//g' #{path} > #{path}.tmp"
+      FileUtils.mv "#{path}.tmp", path
+    end
+    
     def convert_file_to_utf8!
-      return if encoding == 'UTF-8' or encoding == 'UTF8'
       RemoteTable.backtick_with_reporting "iconv -c -f #{encoding} -t UTF-8 #{path} > #{path}.tmp"
       FileUtils.mv "#{path}.tmp", path
     end

data/remote_table.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{remote_table}
-  s.version = "0.2.22"
+  s.version = "0.2.23"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-17}
+  s.date = %q{2010-05-21}
   s.description = %q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [

data/test/remote_table_test.rb

@@ -128,6 +128,14 @@ class RemoteTableTest < Test::Unit::TestCase
   end
   
   if ENV['ALL'] == 'true' or ENV['NEW'] == 'true'
+  end
+  
+  if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+    should "ignore UTF-8 byte order marks" do
+      t = RemoteTable.new :url => 'http://www.freebase.com/type/exporttypeinstances/base/horses/horse_breed?page=0&filter_mode=type&filter_view=table&show%01p%3D%2Ftype%2Fobject%2Fname%01index=0&show%01p%3D%2Fcommon%2Ftopic%2Fimage%01index=1&show%01p%3D%2Fcommon%2Ftopic%2Farticle%01index=2&sort%01p%3D%2Ftype%2Fobject%2Ftype%01p%3Dlink%01p%3D%2Ftype%2Flink%2Ftimestamp%01index=false&=&exporttype=csv-8'
+      assert_equal 'Tawleed', t.rows.first['name']
+    end
+    
     should "be able to apply errata files" do
       t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-G.htm",
                           :encoding => 'windows-1252',
@@ -140,9 +148,7 @@ class RemoteTableTest < Test::Unit::TestCase
       assert_equal 'GRUMMAN', g1['Manufacturer']
       assert_equal 'G159 Gulfstream I (TC4 Academe, VC4)', g1['Model']
     end
-  end
-  
-  if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+
     # this will die with an error about libcurl if your curl doesn't support ssl
     should "connect using HTTPS if available" do
       t = RemoteTable.new(:url => 'https://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA')

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 22
-  version: 0.2.22
+  - 23
+  version: 0.2.23
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-17 00:00:00 -04:00
+date: 2010-05-21 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency