remote_table 2.1.2 → 3.0.0.alpha

data/lib/remote_table/local_copy.rb

@@ -78,7 +78,7 @@ class RemoteTable
     def encoded_io
       @encoded_io || @encoded_io_mutex.synchronize do
         @encoded_io ||= if ::RUBY_VERSION >= '1.9'
-          ::File.open path, 'rb', :internal_encoding => t.internal_encoding, :external_encoding => RemoteTable::EXTERNAL_ENCODING
+          ::File.open path, 'rb', :internal_encoding => t.encoding, :external_encoding => RemoteTable::EXTERNAL_ENCODING
         else
           ::File.open path, 'rb'
         end

data/lib/remote_table/plaintext.rb

@@ -25,7 +25,7 @@ class RemoteTable
 
     # Remove bytes that are both useless and harmful in the vast majority of cases.
     def delete_harmful!
-      harmful = [ Plaintext.soft_hyphen(internal_encoding), UTF8_BOM ]
+      harmful = [ Plaintext.soft_hyphen(encoding), UTF8_BOM ]
       local_copy.in_place :perl, "s/#{harmful.join('//g; s/')}//g"
     end
     
@@ -35,12 +35,12 @@ class RemoteTable
     #   iconv -c -t UTF-8//TRANSLIT -f WINDOWS-1252
     def transliterate_whole_file_to_utf8!
       if ::UnixUtils.available?('iconv')
-        local_copy.in_place :iconv, RemoteTable::EXTERNAL_ENCODING_ICONV, internal_encoding
+        local_copy.in_place :iconv, RemoteTable::EXTERNAL_ENCODING_ICONV, encoding
       else
         ::Kernel.warn %{[remote_table] iconv not available in your $PATH, not performing transliteration}
       end
       # now that we've force-transliterated to UTF-8, act as though this is what the user had specified
-      @internal_encoding = RemoteTable::EXTERNAL_ENCODING
+      @encoding = RemoteTable::EXTERNAL_ENCODING
     end
     
     # No matter what the EOL are SUPPOSED to be, run it through Perl with a regex that will convert all EOLS to \n

data/lib/remote_table/processed_by_roo.rb

@@ -6,25 +6,27 @@ class RemoteTable
 
     # Yield each row using Roo.
     def _each
+      # sometimes Roo forgets to require iconv.
+      require 'iconv'
       require 'roo'
 
       spreadsheet = roo_class.new local_copy.path, nil, :ignore
       if sheet
         spreadsheet.default_sheet = sheet
       end
-
+      
       first_row = if crop
         crop.first + 1
       else
         skip + 1
       end
-
+        
       last_row = if crop
         crop.last
       else
         spreadsheet.last_row
       end
-
+      
       if not headers
 
         # create an array to represent this row
@@ -46,7 +48,7 @@ class RemoteTable
         end
 
       else
-
+        
         # create a hash to represent this row
         current_headers = ::ActiveSupport::OrderedHash.new
         if headers == :first_row

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = "2.1.2"
+  VERSION = '3.0.0.alpha'
 end

data/remote_table.gemspec

@@ -16,9 +16,9 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
-
+  
   s.add_runtime_dependency 'activesupport', '>=2.3.4'
-  s.add_runtime_dependency 'roo', '>= 1.11'
+  s.add_runtime_dependency 'roo', '>= 1.10.3'
   s.add_runtime_dependency 'fixed_width-multibyte', '>=0.2.3'
   s.add_runtime_dependency 'i18n' # activesupport?
   s.add_runtime_dependency 'unix_utils', '>=0.0.8'
@@ -26,10 +26,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'hash_digest'
 
   s.add_development_dependency 'errata', '>=0.2.0'
-  s.add_development_dependency 'georuby'
-  s.add_development_dependency 'dbf'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'minitest-reporters'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yard'
+  s.add_development_dependency 'pry'
 end

data/test/data/color.csv

@@ -0,0 +1,3 @@
+en,es,ru
+green,verde,зеленый
+red,rojo,красный

data/test/data/ranges.csv

@@ -0,0 +1,4 @@
+description,range
+great,A-B
+ok,C
+bad,D-F

data/test/test_errata.rb

@@ -50,8 +50,8 @@ describe RemoteTable do
                           :encoding => 'windows-1252',
                           :row_xpath => '//table[2]//table[1]//tr[3]//tr',
                           :column_xpath => 'td',
-                          :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
-                                       :responder => AircraftGuru.new }
+                          :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+                                       :responder => AircraftGuru.new)
       g1 = t.rows.detect { |row| row['Model'] =~ /Gulfstream I/ }
       g1.wont_be_nil
       g1['Manufacturer'].must_equal 'GULFSTREAM AEROSPACE'

data/test/test_local.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require 'helper'
+
+describe RemoteTable do
+  describe 'used on local files' do
+    it "understands relative paths" do
+      RemoteTable.new('test/data/color.csv').to_a.must_equal RemoteTable.new(File.expand_path('../../test/data/color.csv', __FILE__)).to_a
+    end
+  end
+end

data/test/test_old_syntax.rb

@@ -67,19 +67,6 @@ describe RemoteTable do
       t.rows.last['Model'].must_equal 'EZ King Cobra'
     end
     
-    it "hash rows without paying attention to order" do
-      x = ActiveSupport::OrderedHash.new
-      x[:a] = 1
-      x[:b] = 2
-    
-      y = ActiveSupport::OrderedHash.new
-      y[:b] = 2
-      y[:a] = 1
-    
-      Marshal.dump(x).wont_equal Marshal.dump(y)
-      RemoteTable::Transform.row_hash(y).must_equal RemoteTable::Transform.row_hash(x)
-    end
-    
     it "open a Google Docs url (as a CSV)" do
       t = RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA')
       t.rows.first['PAD district name'].must_equal 'Gulf Coast'

data/test/test_parser.rb

@@ -0,0 +1,24 @@
+require 'helper'
+
+describe RemoteTable do
+  describe ":parser option" do
+    it "takes a parser object that responds to #parse(row) and returns an array of rows" do
+      class GradeRangeParser
+        def parse(row)
+          row['range'].split('-').map do |subrange|
+            virtual_row = row.dup
+            virtual_row.delete 'range'
+            virtual_row['grade'] = subrange
+            virtual_row
+          end
+        end
+      end
+      t = RemoteTable.new "file://#{File.expand_path('../data/ranges.csv', __FILE__)}", parser: GradeRangeParser.new
+      t[0].must_equal 'description' => 'great', 'grade' => 'A'
+      t[1].must_equal 'description' => 'great', 'grade' => 'B'
+      t[2].must_equal 'description' => 'ok', 'grade' => 'C'
+      t[3].must_equal 'description' => 'bad', 'grade' => 'D'
+      t[4].must_equal 'description' => 'bad', 'grade' => 'F'
+    end
+  end
+end

data/test/test_remote.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require 'helper'
+require 'tempfile'
+
+describe RemoteTable do
+  describe 'used on remote files' do
+    it "open an XLSX" do
+      t = RemoteTable.new 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
+      t[5]["Requirements"].must_equal "Secure encryption of all data"
+    end
+
+    it "does its best to download urls without http://" do
+      t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
+      t[5]["Requirements"].must_equal "Secure encryption of all data"
+    end
+
+    it "add a row hash to every row" do
+      t = RemoteTable.new(:url => 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx')
+      t[5].row_hash.must_equal "06d8a738551c17735e2731e25c8d0461"
+    end
+
+    it "open a google doc" do
+      t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
+      t[0]['name'].must_equal 'Seamus Abshere'
+    end
+
+    it "open a csv with custom headers" do
+      t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :headers => %w{ col1 col2 col3 }
+      t[0]['col2'].must_equal 'name'
+      t[1]['col2'].must_equal 'Seamus Abshere'
+    end
+
+    it "return an ordered hash" do
+      t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
+      t[0].class.must_equal ::ActiveSupport::OrderedHash
+    end
+
+    it "open a csv inside a zip file" do
+      t = RemoteTable.new  'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
+                           :filename => 'Annex Tables/Annex 3/Table A-93.csv',
+                           :skip => 1,
+                           :select => proc { |row| row['Vehicle Age'].strip =~ /^\d+$/ }
+      t[0]['LDGV'].must_equal '9.09%'
+    end
+
+    it 'not blow up if each is called twice' do
+      t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
+      count = 0
+      t.each { |row| count += 1 }
+      first_run = count
+      t.each { |row| count += 1}
+      count.must_equal first_run*2
+    end
+
+    it 'allow itself to be cleared for save memory' do
+      t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
+      t.to_a
+      t.send(:cache).length.must_be :>, 0
+      t.free
+      t.send(:cache).length.must_equal 0
+    end
+
+    # fixes ArgumentError: invalid byte sequence in UTF-8
+    it %{safely strip soft hyphens and read windows-1252 html} do
+      t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table[2]//table[1]//tr[3]//tr', :column_xpath => 'td', :encoding => 'windows-1252'
+      t.rows.detect { |row| row['Model'] == 'A300B4600' }.wont_equal nil
+    end
+
+    it %{transliterate characters from ISO-8859-1} do
+      t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
+      t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }.wont_equal nil
+    end
+
+    it %{read xml with css selectors} do
+      t = RemoteTable.new 'http://www.nanonull.com/TimeService/TimeService.asmx/getCityTime?city=Chicago', :format => :xml, :row_css => 'string', :headers => false
+      /(AM|PM)/.match(t[0][0]).wont_equal nil
+    end
+
+    it %{optionally stream rows instead of caching them} do
+      t = RemoteTable.new 'http://www.earthtools.org/timezone/40.71417/-74.00639', :format => :xml, :row_xpath => '//timezone/isotime', :headers => false, :streaming => true
+      time1 = t[0][0]
+      /\d\d\d\d-\d\d-\d\d/.match(time1).wont_equal nil
+      sleep 1
+      time2 = t[0][0]
+      time1.wont_equal time2
+    end
+
+    it %{recode as UTF-8 even ISO-8859-1 (or any other encoding)} do
+      t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';', :encoding => 'ISO-8859-1'
+      t[1][0].must_equal %{ÅLAND ISLANDS}
+    end
+
+    it %{parse a big CSV that is not UTF-8} do
+      t = RemoteTable.new 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat', :headers => false#, :encoding => 'UTF-8'
+      t[0][1].must_equal 'Goroka'
+    end
+
+    it "read only certain rows of an XLSX" do
+      t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => false
+      t[0][0].must_equal "Permissioning and access groups for all content"
+      t[4][0].must_equal "Manage Multiple Incentive Programs for Participants"
+
+      t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => %w{ col1 }
+      t[0]['col1'].must_equal "Permissioning and access groups for all content"
+      t[4]['col1'].must_equal "Manage Multiple Incentive Programs for Participants"
+    end
+
+    it "doesn't get confused by :format => nil" do
+      t = RemoteTable.new :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls', :format => nil
+      t[0]['Class'].must_equal 'TWO SEATERS'
+    end
+  end
+end

data/test/test_remote_table.rb

@@ -3,130 +3,32 @@ require 'helper'
 require 'tempfile'
 
 describe RemoteTable do
-  it "open an XLSX" do
-    t = RemoteTable.new 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
-    t[5]["Requirements"].must_equal "Secure encryption of all data"
-  end
-
   it "doesn't screw up UTF-8" do
-    t = RemoteTable.new "file://#{File.expand_path('../support/airports.utf8.csv', __FILE__)}"
+    t = RemoteTable.new "file://#{File.expand_path('../data/airports.utf8.csv', __FILE__)}"
     t[3]['city'].must_equal "Puerto Inírida"
   end
 
   it "likes paths as much as urls for local files" do
-    by_url = RemoteTable.new "file://#{File.expand_path('../support/airports.utf8.csv', __FILE__)}"
-    by_path = RemoteTable.new File.expand_path('../support/airports.utf8.csv', __FILE__)
+    by_url = RemoteTable.new "file://#{File.expand_path('../data/airports.utf8.csv', __FILE__)}"
+    by_path = RemoteTable.new File.expand_path('../data/airports.utf8.csv', __FILE__)
     by_path.rows.must_equal by_url.rows
   end
 
-  it "does its best to download urls without http://" do
-    t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
-    t[5]["Requirements"].must_equal "Secure encryption of all data"
-  end
-
-  it "add a row hash to every row" do
-    t = RemoteTable.new(:url => 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx')
-    t[5].row_hash.must_equal "06d8a738551c17735e2731e25c8d0461"
-  end
-
-  it "open a google doc" do
-    t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
-    t[0]['name'].must_equal 'Seamus Abshere'
-  end
-
-  it "open a csv with custom headers" do
-    t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :headers => %w{ col1 col2 col3 }
-    t[0]['col2'].must_equal 'name'
-    t[1]['col2'].must_equal 'Seamus Abshere'
-  end
-
-  it "open a yaml" do
-    t = RemoteTable.new "file://#{File.expand_path('../fixtures/data.yml', __FILE__)}"
-    t[0]['name'].must_equal 'Seamus Abshere'
-    t[0]['city'].must_equal 'Madison'
-    t[1]['name'].must_equal 'Derek Kastner'
-    t[1]['city'].must_equal 'Lansing'
-  end
-
-  it "return an ordered hash" do
-    t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
-    t[0].class.must_equal ::ActiveSupport::OrderedHash
-  end
-
-  it "pass through fastercsv options" do
-    f = Tempfile.new 'pass-through-fastercsv-options'
-    f.write %{3,Title example,Body example with a <a href="">link</a>,test category}
-    f.flush
-    t = RemoteTable.new "file://#{f.path}", :quote_char => %{'}, :headers => nil # this should really be "headers => false"
-    t[0][2].must_equal %{Body example with a <a href="">link</a>}
-    f.close
-  end
-
-  it "open a csv inside a zip file" do
-    t = RemoteTable.new  'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-                         :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-                         :skip => 1,
-                         :select => proc { |row| row['Vehicle Age'].strip =~ /^\d+$/ }
-    t[0]['LDGV'].must_equal '9.09%'
-  end
-
-  it 'not blow up if each is called twice' do
-    t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
-    count = 0
-    t.each { |row| count += 1 }
-    first_run = count
-    t.each { |row| count += 1}
-    count.must_equal first_run*2
-  end
-
-  it 'allow itself to be cleared for save memory' do
-    t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
-    t.to_a
-    t.send(:cache).length.must_be :>, 0
-    t.free
-    t.send(:cache).length.must_equal 0
-  end
-
-  # fixes ArgumentError: invalid byte sequence in UTF-8
-  it %{safely strip soft hyphens and read windows-1252 html} do
-    t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table[2]//table[1]//tr[3]//tr', :column_xpath => 'td', :encoding => 'windows-1252'
-    t.rows.detect { |row| row['Model'] == 'A300B4600' }.wont_equal nil
-  end
-
-  it %{transliterate characters from ISO-8859-1} do
-    t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
-    t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }.wont_equal nil
-  end
-
-  it %{read xml with css selectors} do
-    t = RemoteTable.new 'http://www.nanonull.com/TimeService/TimeService.asmx/getCityTime?city=Chicago', :format => :xml, :row_css => 'string', :headers => false
-    /(AM|PM)/.match(t[0][0]).wont_equal nil
-  end
-
-  it %{optionally stream rows instead of caching them} do
-    t = RemoteTable.new 'http://www.earthtools.org/timezone/40.71417/-74.00639', :format => :xml, :row_xpath => '//timezone/isotime', :headers => false, :streaming => true
-    time1 = t[0][0]
-    /\d\d\d\d-\d\d-\d\d/.match(time1).wont_equal nil
-    sleep 1
-    time2 = t[0][0]
-    time1.wont_equal time2
-  end
-
   {
-  # IMPOSSIBLE "../support/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls" => {:format=>"xls",         :encoding=>"binary"},
-  "../support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx"         => {:format=>"xlsx"},
-  "../support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xls"          => {:format=>"xls"},
-  "../support/list-en1-semic-3.neooffice.binary.ods"                        => {:format=>"ods"},
-  "../support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64"         => {:format=>"fixed_width", :encoding=>"iso-8859-1", :schema => [['name', 63, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
-  "../support/list-en1-semic-3.neooffice.utf-8.fixed_width-62"              => {:format=>"fixed_width", :schema => [['name', 61, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
-  # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html"          => {:format=>"html" },
-  # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html"     => {:format=>"html", :encoding=>"iso-8859-1"},
-  # TODO "../support/list-en1-semic-3.neooffice.utf-8.html"                        => {:format=>"html" },
-  "../support/list-en1-semic-3.neooffice.utf-8.xml"                         => {:format=>"xml", :row_css=>'Row', :column_css => 'Data', :select => proc { |row| row[1].to_s =~ /[A-Z]{2}/ }},
-  "../support/list-en1-semic-3.neooffice.iso-8859-1.csv"                    => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
-  "../support/list-en1-semic-3.original.iso-8859-1.csv"                     => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
-  "../support/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma"       => {:format=>"csv", :encoding=>"MACROMAN"}, # comma because no option in excel
-  "../support/list-en1-semic-3.neooffice.utf-8.csv"                         => {:format=>"csv", :delimiter => ';'}
+  # IMPOSSIBLE "../data/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls" => {:format=>"xls",         :encoding=>"binary"},
+  "../data/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx"         => {:format=>"xlsx"},
+  "../data/list-en1-semic-3.office-2011-for-mac-sp1.binary.xls"          => {:format=>"xls"},
+  "../data/list-en1-semic-3.neooffice.binary.ods"                        => {:format=>"ods"},
+  "../data/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64"         => {:format=>"fixed_width", :encoding=>"iso-8859-1", :schema => [['name', 63, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
+  "../data/list-en1-semic-3.neooffice.utf-8.fixed_width-62"              => {:format=>"fixed_width", :schema => [['name', 61, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
+  # TODO "../data/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html"          => {:format=>"html" },
+  # TODO "../data/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html"     => {:format=>"html", :encoding=>"iso-8859-1"},
+  # TODO "../data/list-en1-semic-3.neooffice.utf-8.html"                        => {:format=>"html" },
+  "../data/list-en1-semic-3.neooffice.utf-8.xml"                         => {:format=>"xml", :row_css=>'Row', :column_css => 'Data', :select => proc { |row| row[1].to_s =~ /[A-Z]{2}/ }},
+  "../data/list-en1-semic-3.neooffice.iso-8859-1.csv"                    => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
+  "../data/list-en1-semic-3.original.iso-8859-1.csv"                     => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
+  "../data/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma"       => {:format=>"csv", :encoding=>"MACROMAN"}, # comma because no option in excel
+  "../data/list-en1-semic-3.neooffice.utf-8.csv"                         => {:format=>"csv", :delimiter => ';'}
   }.each do |k, v|
     it %{open #{k} with encoding #{v[:encoding] || 'default'}} do
       options = v.merge(:headers => false, :skip => 2)
@@ -141,57 +43,20 @@ describe RemoteTable do
     end
   end
 
-  it %{recode as UTF-8 even ISO-8859-1 (or any other encoding)} do
-    t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';', :encoding => 'ISO-8859-1'
-    t[1][0].must_equal %{ÅLAND ISLANDS}
-  end
-
-  it %{parse a big CSV that is not UTF-8} do
-    t = RemoteTable.new 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat', :headers => false#, :encoding => 'UTF-8'
-    t[0][1].must_equal 'Goroka'
-  end
-
-  it "read only certain rows of an XLSX" do
-    t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => false
-    t[0][0].must_equal "Permissioning and access groups for all content"
-    t[4][0].must_equal "Manage Multiple Incentive Programs for Participants"
-
-    t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => %w{ col1 }
-    t[0]['col1'].must_equal "Permissioning and access groups for all content"
-    t[4]['col1'].must_equal "Manage Multiple Incentive Programs for Participants"
-  end
-
-  it "doesn't get confused by :format => nil" do
-    t = RemoteTable.new :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls', :format => nil
-    t[0]['Class'].must_equal 'TWO SEATERS'
-  end
-
-  {
-    'foo.ods' => :ods,
-    'foo.open_office' => :ods,
-    'foo.xlsx' => :xlsx,
-    'foo.excelx' => :xlsx,
-    'foo.xls' => :xls,
-    'foo.excel' => :xls,
-    'foo.csv' => :delimited,
-    'foo.tsv' => :delimited,
-    'foo.delimited' => :delimited,
-    'foo.fixed_width' => :fixed_width,
-    'foo.htm' => :html,
-    'foo.html' => :html,
-    'foo.xml' => :xml,
-    'foo.yaml' => :yaml,
-    'foo.yml' => :yaml,
-    'foo.shp' => :shp
-  }.each do |basename, format|
-    it "detects the #{format} format from the filename #{basename}" do
-      RemoteTable.guess_format(basename).must_equal format
-    end
+  it "pass through fastercsv options" do
+    f = Tempfile.new 'pass-through-fastercsv-options'
+    f.write %{3,Title example,Body example with a <a href="">link</a>,test category}
+    f.flush
+    t = RemoteTable.new "file://#{f.path}", :quote_char => %{'}, :headers => nil # this should really be "headers => false"
+    t[0][2].must_equal %{Body example with a <a href="">link</a>}
+    f.close
   end
 
-  it "detects the correct extension name without confusion from basename" do
-    [ 'foo.xls', 'xlsx.xls', 'foo_xls' ].each do |basename|
-      RemoteTable.guess_format(basename).must_equal :xls
-    end
+  it "open a yaml" do
+    t = RemoteTable.new "file://#{File.expand_path('../data/data.yml', __FILE__)}"
+    t[0]['name'].must_equal 'Seamus Abshere'
+    t[0]['city'].must_equal 'Madison'
+    t[1]['name'].must_equal 'Derek Kastner'
+    t[1]['city'].must_equal 'Lansing'
   end
 end