remote_table 1.2.2 → 1.2.3

data/.gitattributes

@@ -0,0 +1 @@
+list-en1-semic-3* -crlf -diff -merge

data/README.rdoc

@@ -6,6 +6,13 @@ Open local or remote XLSX, XLS, ODS, CSV and fixed-width files.
 
 Used by http://data.brighterplanet.com and the data_miner gem (http://github.com/seamusabshere/data_miner)
 
+==Requirements
+
+* POSIX operating system (not windows)
+* curl, iconv, perl, cat, cut, tail, etc. accessible from /usr/local/bin:/usr/bin:/bin
+
+As this library matures, those should go away.
+
 ==Example
 
     ?> t = RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', 'filename' => '98guide6.csv'
@@ -136,6 +143,11 @@ More examples:
                                 [ 'spacer',  1 ],
                                 [ 'header1', 10, { :type => :string } ]]
 
+==Helpful hints
+
+* ASCII-8BIT is the same as BINARY
+* ISO-8859-1 is the same as Latin1
+
 ==Custom parsers
 
 See the test file and also data_miner examples of custom parsers.

data/lib/remote_table.rb

@@ -1,3 +1,8 @@
+if ::RUBY_VERSION < '1.9' and $KCODE != 'UTF8'
+  $stderr.puts "[remote_table] Ruby 1.8 detected, setting $KCODE to UTF8 so that ActiveSupport::Multibyte works properly."
+  $KCODE = 'UTF8'
+end
+
 require 'active_support'
 require 'active_support/version'
 %w{

data/lib/remote_table/executor.rb

@@ -16,6 +16,9 @@ class RemoteTable
 
     def backtick_with_reporting(cmd, raise_on_error = false)
       cmd = cmd.gsub /\n/m, ' '
+      if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include? 'backtick'
+        $stderr.puts "[remote_table] Executing #{cmd}"
+      end
       pid = ::POSIX::Spawn.spawn({ 'PATH' => '/usr/local/bin:/usr/bin:/bin' }, cmd)
       stat = ::Process::waitpid pid
       if raise_on_error and not stat.success?

data/lib/remote_table/format.rb

@@ -24,17 +24,24 @@ class RemoteTable
       @t = t
     end
     
-    def utf8(str)
+    def transliterate_to_utf8(str)
+      return if str.nil?
+      $stderr.puts "[remote_table translit] Before:  #{str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
+      transliterated_str = if ::RUBY_VERSION >= '1.9'
+        str.ensure_encoding t.properties.external_encoding, :external_encoding => t.properties.internal_encoding, :invalid_characters => :transcode
+      else
+        ::Iconv.conv(t.properties.external_encoding_iconv, t.properties.internal_encoding, str.to_s + ' ')[0..-2]
+      end
+      $stderr.puts "[remote_table translit] After:   #{transliterated_str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
+      transliterated_str
+    end
+
+    def assume_utf8(str)
+      return if str.nil?
       if ::RUBY_VERSION >= '1.9'
-        str.ensure_encoding 'UTF-8', :external_encoding => t.properties.encoding, :invalid_characters => :transcode
+        str.encode! t.properties.external_encoding
       else
-        return str if t.properties.encoding[0] =~ /utf.?8/i
-        begin
-          ::Iconv.conv('UTF-8//TRANSLIT', t.properties.encoding[0], str.to_s + ' ')[0..-2]
-        rescue ::Iconv::IllegalSequence
-          $stderr.puts "[remote_table] Unable to transliterate #{str} into UTF-8 given #{t.properties.encoding[0]}"
-          str
-        end
+        str
       end
     end
     

data/lib/remote_table/format/delimited.rb

@@ -17,19 +17,21 @@ class RemoteTable
       include Textual
       def each(&blk)
         remove_useless_characters!
+        fix_newlines!
+        transliterate_whole_file_to_utf8!
         skip_rows!
-        CSV.foreach(t.local_file.path, fastercsv_options) do |row|
+        CSV.new(t.local_file.encoded_io, fastercsv_options).each do |row|
           if row.is_a?(CSV::Row)
-            output = row.inject(::ActiveSupport::OrderedHash.new) do |memo, (key, value)|
-              if key.present?
-                value = '' if value.nil?
-                memo[key] = utf8 value
+            hash = row.inject(::ActiveSupport::OrderedHash.new) do |memo, (k, v)|
+              if k.present?
+                memo[k] = v.to_s
               end
               memo
             end
-            yield output if t.properties.keep_blank_rows or output.any? { |k, v| v.present? }
-          else
-            yield row if t.properties.keep_blank_rows or row.any? { |v| v.present? }
+            yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
+          elsif row.is_a?(::Array)
+            array = row.map { |v| v.to_s }
+            yield array if t.properties.keep_blank_rows or array.any? { |v| v.present? }
           end
         end
       ensure

data/lib/remote_table/format/fixed_width.rb

@@ -1,17 +1,20 @@
-require 'slither'
+require 'fixed_width'
+
 class RemoteTable
   class Format
     class FixedWidth < Format
       include Textual
       def each(&blk)
         remove_useless_characters!
+        fix_newlines!
+        transliterate_whole_file_to_utf8!
         crop_rows!
         skip_rows!
         cut_columns!
         parser.parse[:rows].each do |row|
           row.reject! { |k, v| k.blank? }
           row.each do |k, v|
-            row[k] = utf8 v
+            row[k] = v.strip
           end
           yield row if t.properties.keep_blank_rows or row.any? { |k, v| v.present? }
         end
@@ -22,16 +25,20 @@ class RemoteTable
       private
       
       def parser
-        @parser ||= ::Slither::Parser.new definition, t.local_file.path
+        return @parser if @parser.is_a?(::FixedWidth::Parser)
+        if ::FixedWidth::Section.private_instance_methods.map(&:to_sym).include?(:unpacker)
+          raise "[remote_table] You need a different (newer) version of the FixedWidth gem that supports multibyte encoding, sometime after https://github.com/timonk/fixed_width/pull/1 was incorporated"
+        end
+        @parser = ::FixedWidth::Parser.new definition, t.local_file.encoded_io
       end
       
       def definition
         @definition ||= if t.properties.schema_name.is_a?(::String) or t.properties.schema_name.is_a?(::Symbol)
-          ::Slither.send :definition, t.properties.schema_name
+          ::FixedWidth.send :definition, t.properties.schema_name
         elsif t.properties.schema.is_a?(::Array)
           everything = lambda { |_| true }
           srand # in case this was forked by resque
-          ::Slither.define(rand.to_s) do |d|
+          ::FixedWidth.define(rand.to_s) do |d|
             d.rows do |row|
               row.trap(&everything)
               t.properties.schema.each do |name, width, options|

data/lib/remote_table/format/mixins/processed_by_nokogiri.rb

@@ -4,7 +4,9 @@ class RemoteTable
   class Format
     module ProcessedByNokogiri
       def each
+        raise "[remote_table] Need :row_css or :row_xpath in order to process XML or HTML" unless t.properties.row_css or t.properties.row_xpath
         remove_useless_characters!
+        transliterate_whole_file_to_utf8!
         first_row = true
         keys = t.properties.headers if t.properties.headers.is_a?(::Array)
         xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, 'UTF-8')
@@ -15,7 +17,7 @@ class RemoteTable
             row.xpath(t.properties.column_xpath)
           else
             [row]
-          end.map { |cell| cell.content.gsub(/\s+/, ' ').strip }
+          end.map { |cell| assume_utf8 cell.content.gsub(/\s+/, ' ').strip }
           if first_row and t.properties.use_first_row_as_header?
             keys = values
             first_row = false
@@ -57,7 +59,7 @@ class RemoteTable
 
       # should we be doing this in ruby?
       def unescaped_xml_without_soft_hyphens
-        str = ::CGI.unescapeHTML utf8(::IO.read(t.local_file.path))
+        str = ::CGI.unescapeHTML t.local_file.encoded_io.read
         # get rid of MS Office baddies
         str.gsub! '­', ''
         str

data/lib/remote_table/format/mixins/processed_by_roo.rb

@@ -6,9 +6,9 @@ class RemoteTable
         spreadsheet = roo_class.new t.local_file.path, nil, :ignore
         spreadsheet.default_sheet = t.properties.sheet.is_a?(::Numeric) ? spreadsheet.sheets[t.properties.sheet] : t.properties.sheet
         if t.properties.output_class == ::Array
-          (first_data_row..spreadsheet.last_row).each do |y|
+          (first_row..spreadsheet.last_row).each do |y|
             output = (1..spreadsheet.last_column).map do |x|
-              spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
+              assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
             end
             yield output if t.properties.keep_blank_rows or output.any? { |v| v.present? }
           end
@@ -16,18 +16,19 @@ class RemoteTable
           keys = {}
           if t.properties.use_first_row_as_header?
             (1..spreadsheet.last_column).each do |x|
-              keys[x] = spreadsheet.cell(header_row, x)
-              keys[x] = spreadsheet.cell(header_row - 1, x) if keys[x].blank? # look up
+              keys[x] = spreadsheet.cell(first_row, x)
+              keys[x] = spreadsheet.cell(first_row - 1, x) if keys[x].blank? # look up
+              keys[x] = assume_utf8 keys[x]
             end
           else
             (1..spreadsheet.last_column).each do |x|
-              keys[x] = t.properties.headers[x - 1]
+              keys[x] = assume_utf8 t.properties.headers[x - 1]
             end
           end
-          (first_data_row..spreadsheet.last_row).each do |y|
+          (first_row+1..spreadsheet.last_row).each do |y|
             output = (1..spreadsheet.last_column).inject(::ActiveSupport::OrderedHash.new) do |memo, x|
               if keys[x].present?
-                memo[keys[x]] = spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
+                memo[keys[x]] = assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
               end
               memo
             end
@@ -40,13 +41,9 @@ class RemoteTable
 
       private
 
-      def header_row
+      def first_row
         1 + t.properties.skip
       end
-
-      def first_data_row
-        1 + header_row
-      end
     end
   end
 end

data/lib/remote_table/format/mixins/textual.rb

@@ -6,11 +6,22 @@ class RemoteTable
       USELESS_CHARACTERS = [
         '\xef\xbb\xbf',   # UTF-8 byte order mark
         '\xc2\xad',       # soft hyphen, often inserted by MS Office (html: ­)
-        '\xad',
-        # '\xa0'
       ]
       def remove_useless_characters!
         ::RemoteTable.executor.bang t.local_file.path, "perl -pe 's/#{USELESS_CHARACTERS.join '//g; s/'}//g'"
+        if t.properties.internal_encoding =~ /windows.?1252/i
+          # soft hyphen again, as I have seen it appear in windows 1252
+          ::RemoteTable.executor.bang t.local_file.path, %q{perl -pe 's/\xad//g'}
+        end
+      end
+      
+      def transliterate_whole_file_to_utf8!
+        ::RemoteTable.executor.bang t.local_file.path, "iconv -c -f #{::Escape.shell_single_word t.properties.internal_encoding} -t #{::Escape.shell_single_word t.properties.external_encoding_iconv}"
+        t.properties.update 'encoding' => t.properties.external_encoding
+      end
+      
+      def fix_newlines!
+        ::RemoteTable.executor.bang t.local_file.path, %q{perl -pe 's/\r\n|\n|\r/\n/g'}
       end
       
       def skip_rows!

data/lib/remote_table/local_file.rb

@@ -15,8 +15,20 @@ class RemoteTable
       @path
     end
     
+    def encoded_io
+      @encoded_io ||= if ::RUBY_VERSION >= '1.9'
+        ::File.open path, 'rb', :internal_encoding => t.properties.internal_encoding, :external_encoding => t.properties.external_encoding
+      else
+        ::File.open path, 'rb'
+      end
+    end
+    
     def delete
+      if @encoded_io.respond_to?(:closed?) and !@encoded_io.closed?
+        @encoded_io.close
+      end
       ::FileUtils.rm_rf staging_dir_path
+      @encoded_io = nil
       @path = nil
       @staging_dir_path = nil
     end

data/lib/remote_table/properties.rb

@@ -3,8 +3,15 @@ class RemoteTable
   # Represents the properties of a RemoteTable, whether they are explicitly set by the user or inferred automatically.
   class Properties
     attr_reader :t
+    attr_reader :current_options
+    
     def initialize(t)
       @t = t
+      @current_options = t.options.dup
+    end
+    
+    def update(options)
+      current_options.update options
     end
     
     # The parsed URI of the file to get.
@@ -22,19 +29,19 @@ class RemoteTable
     # * call each
     # Defaults to false.
     def streaming
-      t.options['streaming'] || false
+      current_options['streaming'] || false
     end
 
     # Defaults to true.
     def warn_on_multiple_downloads
-      t.options['warn_on_multiple_downloads'] != false
+      current_options['warn_on_multiple_downloads'] != false
     end
     
     # The headers specified by the user
     #
     # Default: :first_row
     def headers
-      t.options['headers'].nil? ? :first_row : t.options['headers']
+      current_options['headers'].nil? ? :first_row : current_options['headers']
     end
     
     def use_first_row_as_header?
@@ -49,60 +56,65 @@ class RemoteTable
     #
     # Default: 0
     def sheet
-      t.options['sheet'] || 0
+      current_options['sheet'] || 0
     end
     
     # Whether to keep blank rows
     #
     # Default: false
     def keep_blank_rows
-      t.options['keep_blank_rows'] || false
+      current_options['keep_blank_rows'] || false
     end
     
     # Form data to send in with the download request
     def form_data
-      t.options['form_data']
+      current_options['form_data']
     end
     
     # How many rows to skip
     #
     # Default: 0
     def skip
-      t.options['skip'].to_i
+      current_options['skip'].to_i
     end
     
-    # Likely external encoding
-    #
-    # Default: "UTF-8"
-    def encoding
-      @encoding ||= ::Array.wrap(t.options['encoding'] || [ 'ISO-8859-1', 'US-ASCII', 'WINDOWS-1252', 'ASCII-8BIT', 'UTF-8' ])
+    def internal_encoding
+      (current_options['encoding'] || 'UTF-8').upcase
+    end
+    
+    def external_encoding
+      'UTF-8'
+    end
+    
+    def external_encoding_iconv
+      'UTF-8//TRANSLIT'
     end
     
     # The delimiter
     #
     # Default: ","
     def delimiter
-      t.options['delimiter'] || ','
+      current_options['delimiter'] || ','
     end
     
     # The XPath used to find rows
     def row_xpath
-      t.options['row_xpath']
+      current_options['row_xpath']
     end
     
     # The XPath used to find columns
     def column_xpath
-      t.options['column_xpath']
+      current_options['column_xpath']
     end
 
     # The CSS selector used to find rows
     def row_css
-      t.options['row_css']
+      current_options['row_css']
     end
     
     # The CSS selector used to find columns
     def column_css
-      t.options['column_css']
+      current_options['column_css']
     end
     
     # The compression type.
@@ -111,8 +123,8 @@ class RemoteTable
     #
     # Can be specified as: "gz", "zip", "bz2", "exe" (treated as "zip")
     def compression
-      clue = if t.options['compression']
-        t.options['compression'].to_s
+      clue = if current_options['compression']
+        current_options['compression'].to_s
       else
         ::File.extname uri.path
       end
@@ -134,8 +146,8 @@ class RemoteTable
     #
     # Can be specified as: "tar"
     def packing
-      clue = if t.options['packing']
-        t.options['packing'].to_s
+      clue = if current_options['packing']
+        current_options['packing'].to_s
       else
         ::File.extname(uri.path.sub(/\.#{compression}\z/, ''))
       end
@@ -150,7 +162,7 @@ class RemoteTable
     # Example:
     #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', 'glob' => '/*.csv'
     def glob
-      t.options['glob']
+      current_options['glob']
     end
     
     # The filename, which can be used to pick a file out of an archive.
@@ -158,17 +170,17 @@ class RemoteTable
     # Example:
     #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', 'filename' => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv'
     def filename
-      t.options['filename']
+      current_options['filename']
     end
     
     # Cut columns up to this character
     def cut
-      t.options['cut']
+      current_options['cut']
     end
     
     # Crop rows after this line
     def crop
-      t.options['crop']
+      current_options['crop']
     end
     
     # The fixed-width schema, given as an array
@@ -183,31 +195,31 @@ class RemoteTable
     #                                  [  'spacer',  12 ],
     #                                  [  'header6', 10, { :type => :string } ]])
     def schema
-      t.options['schema']
+      current_options['schema']
     end
     
-    # The name of the fixed-width schema according to Slither
+    # The name of the fixed-width schema according to FixedWidth
     def schema_name
-      t.options['schema_name']
+      current_options['schema_name']
     end
     
     # A proc to call to decide whether to return a row.
     def select
-      t.options['select']
+      current_options['select']
     end
     
     # A proc to call to decide whether to return a row.
     def reject
-      t.options['reject']
+      current_options['reject']
     end
     
     # A hash of options to create a new Errata instance (see the Errata gem at http://github.com/seamusabshere/errata) to be used on every row.
     def errata
-      return unless t.options.has_key? 'errata'
-      @errata ||= if t.options['errata'].is_a? ::Hash
-        ::Errata.new t.options['errata']
+      return unless current_options.has_key? 'errata'
+      @errata ||= if current_options['errata'].is_a? ::Hash
+        ::Errata.new current_options['errata']
       else
-        t.options['errata']
+        current_options['errata']
       end
     end
     
@@ -220,8 +232,8 @@ class RemoteTable
     # Can be specified as: "xlsx", "xls", "csv", "ods", "fixed_width", "html"
     def format
       return Format::Delimited if uri.host == 'spreadsheets.google.com'
-      clue = if t.options['format']
-        t.options['format'].to_s
+      clue = if current_options['format']
+        current_options['format'].to_s
       else
         ::File.extname t.local_file.path
       end

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = "1.2.2"
+  VERSION = "1.2.3"
 end

data/remote_table.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   
   s.add_dependency 'activesupport', '>=2.3.4'
   s.add_dependency 'roo', '~>1.9'
-  s.add_dependency 'slither', '>=0.99.4'
+  s.add_dependency 'fixed_width-multibyte' # TODO replace with fixed_width once timon gets off vacation
   s.add_dependency 'i18n' # activesupport?
   s.add_dependency 'builder' # roo?
   s.add_dependency 'zip' # roo
@@ -31,9 +31,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'escape', '>=0.0.4'
   s.add_dependency 'posix-spawn'
   s.add_dependency 'ensure-encoding'
-  unless RUBY_VERSION >= '1.9'
-    s.add_dependency 'fastercsv', '>=1.5.0'
-  end
+  s.add_dependency 'fastercsv', '>=1.5.0'
 
   s.add_development_dependency 'errata', '>=0.2.0'
   s.add_development_dependency 'test-unit'

data/test/helper.rb

@@ -4,11 +4,26 @@ Bundler.setup
 require 'test/unit'
 require 'shoulda'
 require 'ruby-debug'
-require 'tempfile'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'remote_table'))
 
 class Test::Unit::TestCase
+  def setup
+    if RUBY_VERSION >= '1.9'
+      @old_default_internal = Encoding.default_internal
+      @old_default_external = Encoding.default_external
+      # totally random choices here
+      Encoding.default_internal = 'EUC-JP'
+      Encoding.default_external = 'Shift_JIS'
+    end
+  end
+  
+  def teardown
+    if RUBY_VERSION >= '1.9'
+      Encoding.default_internal = @old_default_internal
+      Encoding.default_external = @old_default_external
+    end
+  end
 end

data/test/test_old_syntax.rb

@@ -16,7 +16,7 @@ $test2_rows.freeze
 class TestOldSyntax < Test::Unit::TestCase
   should "open an XLSX like an array (numbered columns)" do
     t = RemoteTable.new(:url => 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :headers => false)
-    assert_equal "Secure encryption of all data", t.rows[5][0]
+    assert_equal "Software-As-A-Service", t.rows[5][0]
   end
 
   should "open an XLSX with custom headers" do

data/test/test_old_transform.rb

@@ -1,36 +1,31 @@
 require 'helper'
 
-class FuelOilParser
+class NaturalGasParser
   def initialize(options = {})
     # nothing
   end
-  def add_hints!(bus)
-    bus[:sheet] = 'Data 1'
-    bus[:skip] = 2
-    bus[:select] = lambda { |row| row['year'] > 1989 }
-  end
   def apply(row)
     virtual_rows = []
-    row.keys.grep(/(.+) Residual Fuel Oil/) do |location_column_name|
-      first_part = $1
-      next if (cost = row[location_column_name]).blank? or (date = row['Date']).blank?
-      if first_part.start_with?('U.S.')
-        locatable = "united_states (Country)"
-      elsif first_part.include?('PADD')
-        /\(PADD (.*)\)/.match(first_part)
-        padd_part = $1
-        next if padd_part == '1' # skip PADD 1 because we always prefer subdistricts
-        locatable = "#{padd_part} (PetroleumAdministrationForDefenseDistrict)"
+    row.keys.grep(/\A(.*) Natural Gas/) do |location_column_name|
+      match_1 = $1
+      next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+      if match_1 == 'U.S.'
+        locatable_id = 'US'
+        locatable_type = 'Country'
       else
-        locatable = "#{first_part} (State)"
+        locatable_id = match_1 # name
+        locatable_type = 'State'
       end
       date = Time.parse(date)
-      virtual_rows << {
-        'locatable' => locatable,
-        'cost' => cost,
-        'year' => date.year,
-        'month' => date.month
-      }
+      new_row = ActiveSupport::OrderedHash.new
+      new_row['locatable_id'] = locatable_id
+      new_row['locatable_type'] = locatable_type
+      new_row['price'] = price
+      new_row['year'] = date.year
+      new_row['month'] = date.month
+      row_hash = RemoteTable::Transform.row_hash new_row
+      new_row['row_hash'] = row_hash
+      virtual_rows << new_row
     end
     virtual_rows
   end
@@ -38,12 +33,12 @@ end
 
 class TestOldTransform < Test::Unit::TestCase
   should "open an XLS with a parser" do
-    ma_1990_01 = {"month"=>1, "cost"=>"54.0", "locatable"=>"Massachusetts (State)", "year"=>1990}
-    ga_1990_01 = {"month"=>1, "cost"=>"50.7", "locatable"=>"Georgia (State)", "year"=>1990}
-
-    t = RemoteTable.new(:url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls',
-                        :transform => { :class => FuelOilParser })
-    assert t.rows.include?(ma_1990_01)
-    assert t.rows.include?(ga_1990_01)
+    t = RemoteTable.new(:url => 'http://tonto.eia.doe.gov/dnav/ng/xls/ng_pri_sum_a_EPG0_FWA_DMcf_a.xls',
+           :sheet => 'Data 1',
+           :skip => 2,
+           :select => lambda { |row| row['year'].to_i > 1989 },
+           :transform => { :class => NaturalGasParser })
+    assert_equal 'Country', t[0]['locatable_type']
+    assert_equal 'US', t[0]['locatable_id']
   end
-end
+end

data/test/test_remote_table.rb

@@ -63,12 +63,12 @@ class TestRemoteTable < Test::Unit::TestCase
     
   # fixes ArgumentError: invalid byte sequence in UTF-8
   should %{safely strip soft hyphens and read windows-1252 html} do
-    t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table/tr[2]/td/table/tr', :column_xpath => 'td'
+    t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table/tr[2]/td/table/tr', :column_xpath => 'td', :encoding => 'windows-1252'
     assert t.rows.detect { |row| row['Model'] == 'A300B4600' }
   end
   
   should %{transliterate characters from ISO-8859-1} do
-    t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv'
+    t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
     assert t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }
   end
   
@@ -86,15 +86,42 @@ class TestRemoteTable < Test::Unit::TestCase
     assert(time1 != time2)
   end
   
-  should %{not die when it reads Åland Islands} do
-    t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';'
-    assert_nothing_raised do
-      t[1][0]
+  {
+  # IMPOSSIBLE "../support/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls" => {:format=>"xls",         :encoding=>"binary"},
+  "../support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx"         => {:format=>"xlsx"},
+  "../support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xls"          => {:format=>"xls"},
+  "../support/list-en1-semic-3.neooffice.binary.ods"                        => {:format=>"ods"},
+  "../support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64"         => {:format=>"fixed_width", :encoding=>"iso-8859-1", :schema => [['name', 63, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
+  "../support/list-en1-semic-3.neooffice.utf-8.fixed_width-62"              => {:format=>"fixed_width", :schema => [['name', 61, { :type => :string }], ['iso_3166', 2, { :type => :string }]]},
+  # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html"          => {:format=>"html" },
+  # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html"     => {:format=>"html", :encoding=>"iso-8859-1"},
+  # TODO "../support/list-en1-semic-3.neooffice.utf-8.html"                        => {:format=>"html" },
+  "../support/list-en1-semic-3.neooffice.utf-8.xml"                         => {:format=>"xml", :row_css=>'Row', :column_css => 'Data', :select => lambda { |row| row[1].to_s =~ /[A-Z]{2}/ }},
+  "../support/list-en1-semic-3.neooffice.iso-8859-1.csv"                    => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
+  "../support/list-en1-semic-3.original.iso-8859-1.csv"                     => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
+  "../support/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma"       => {:format=>"csv", :encoding=>"MACROMAN"}, # comma because no option in excel
+  "../support/list-en1-semic-3.neooffice.utf-8.csv"                         => {:format=>"csv", :delimiter => ';'}
+  }.each do |k, v|
+    should %{open #{k} with encoding #{v[:encoding] || 'default'}} do
+      options = v.merge(:headers => false, :skip => 2)
+      t = RemoteTable.new "file://#{File.expand_path(k, __FILE__)}", options
+      a = %{ÅLAND ISLANDS}
+      b = (t[1].is_a?(::Array) ? t[1][0] : t[1]['name'])
+      if RUBY_VERSION >= '1.9'
+        assert_equal 'UTF-8', a.encoding.to_s
+        assert_equal 'UTF-8', b.encoding.to_s
+      end
+      assert_equal a, b
     end
   end
   
+  should %{recode as UTF-8 even ISO-8859-1 (or any other encoding)} do
+    t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';', :encoding => 'ISO-8859-1'
+    assert_equal %{ÅLAND ISLANDS}, t[1][0]
+  end
+  
   should %{parse a big CSV that is not UTF-8} do
-    t = RemoteTable.new 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat', :headers => false
+    t = RemoteTable.new 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat', :headers => false#, :encoding => 'UTF-8'
     assert_equal 'Goroka', t[0][1]
   end
 end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: remote_table
 version: !ruby/object:Gem::Version 
-  hash: 27
   prerelease: 
-  segments: 
-  - 1
-  - 2
-  - 2
-  version: 1.2.2
+  version: 1.2.3
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -16,7 +11,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-05 00:00:00 Z
+date: 2011-05-21 00:00:00 -05:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
@@ -26,11 +22,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 3
-        - 4
         version: 2.3.4
   type: :runtime
   version_requirements: *id001
@@ -42,27 +33,18 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 1
-        - 9
         version: "1.9"
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: slither
+  name: fixed_width-multibyte
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 411
-        segments: 
-        - 0
-        - 99
-        - 4
-        version: 0.99.4
+        version: "0"
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -73,9 +55,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id004
@@ -87,9 +66,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id005
@@ -101,9 +77,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id006
@@ -115,11 +88,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 1
-        - 4
-        - 1
         version: 1.4.1
   type: :runtime
   version_requirements: *id007
@@ -131,9 +99,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id008
@@ -145,9 +110,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id009
@@ -159,11 +121,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 0
-        - 0
-        - 4
         version: 0.0.4
   type: :runtime
   version_requirements: *id010
@@ -175,9 +132,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id011
@@ -189,9 +143,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id012
@@ -203,11 +154,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 5
-        - 0
         version: 1.5.0
   type: :runtime
   version_requirements: *id013
@@ -219,11 +165,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 0
-        - 2
-        - 0
         version: 0.2.0
   type: :development
   version_requirements: *id014
@@ -235,9 +176,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id015
@@ -249,23 +187,17 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id016
 - !ruby/object:Gem::Dependency 
-  name: ruby-debug
+  name: ruby-debug19
   prerelease: false
   requirement: &id017 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id017
@@ -280,6 +212,7 @@ extra_rdoc_files: []
 
 files: 
 - .document
+- .gitattributes
 - .gitignore
 - Gemfile
 - LICENSE
@@ -305,11 +238,26 @@ files:
 - lib/remote_table/version.rb
 - remote_table.gemspec
 - test/helper.rb
+- test/support/list-en1-semic-3.neooffice.binary.ods
+- test/support/list-en1-semic-3.neooffice.iso-8859-1.csv
+- test/support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64
+- test/support/list-en1-semic-3.neooffice.utf-8.csv
+- test/support/list-en1-semic-3.neooffice.utf-8.fixed_width-62
+- test/support/list-en1-semic-3.neooffice.utf-8.html
+- test/support/list-en1-semic-3.neooffice.utf-8.xml
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xls
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html
+- test/support/list-en1-semic-3.original.iso-8859-1.csv
 - test/test_big.rb
 - test/test_errata.rb
 - test/test_old_syntax.rb
 - test/test_old_transform.rb
 - test/test_remote_table.rb
+has_rdoc: true
 homepage: https://github.com/seamusabshere/remote_table
 licenses: []
 
@@ -323,28 +271,36 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: remotetable
-rubygems_version: 1.7.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Open local or remote XLSX, XLS, ODS, CSV and fixed-width files.
 test_files: 
 - test/helper.rb
+- test/support/list-en1-semic-3.neooffice.binary.ods
+- test/support/list-en1-semic-3.neooffice.iso-8859-1.csv
+- test/support/list-en1-semic-3.neooffice.iso-8859-1.fixed_width-64
+- test/support/list-en1-semic-3.neooffice.utf-8.csv
+- test/support/list-en1-semic-3.neooffice.utf-8.fixed_width-62
+- test/support/list-en1-semic-3.neooffice.utf-8.html
+- test/support/list-en1-semic-3.neooffice.utf-8.xml
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xls
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma
+- test/support/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html
+- test/support/list-en1-semic-3.original.iso-8859-1.csv
 - test/test_big.rb
 - test/test_errata.rb
 - test/test_old_syntax.rb