remote_table 1.1.10 → 1.2.0

data/lib/remote_table.rb

@@ -4,11 +4,19 @@ require 'active_support/version'
   active_support/core_ext/hash
   active_support/core_ext/string
   active_support/core_ext/module
-  active_support/core_ext/array/wrap
+  active_support/core_ext/array
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
 end if ::ActiveSupport::VERSION::MAJOR == 3
 
+class Hash
+  attr_accessor :row_hash
+end
+
+class Array
+  attr_accessor :row_hash
+end
+
 class RemoteTable
   autoload :Format, 'remote_table/format'
   autoload :Properties, 'remote_table/properties'
@@ -53,40 +61,52 @@ class RemoteTable
     @options.freeze
   end
   
+  # not thread safe
   def each(&blk)
-    to_a.each { |row| yield row }
+    if fully_cached?
+      cache.each(&blk)
+    else
+      mark_download!
+      retval = format.each do |row|
+        row.row_hash = ::RemoteTable.hasher.hash row
+        transformer.transform(row).each do |virtual_row|
+          if properties.errata
+            next if properties.errata.rejects? virtual_row
+            properties.errata.correct! virtual_row
+          end
+          next if properties.select and !properties.select.call(virtual_row)
+          next if properties.reject and properties.reject.call(virtual_row)
+          cache.push virtual_row unless properties.streaming
+          yield virtual_row
+        end
+      end
+      fully_cached! unless properties.streaming
+      retval
+    end
   end
   alias :each_row :each
   
   def to_a
-    return @to_a if @to_a.is_a? ::Array
-    @to_a = []
-    format.each do |row|
-      row['row_hash'] = ::RemoteTable.hasher.hash row
-      # allow the transformer to return multiple "virtual rows" for every real row
-      ::Array.wrap(transformer.transform(row)).each do |virtual_row|
-        if properties.errata
-          next if properties.errata.rejects? virtual_row
-          properties.errata.correct! virtual_row
-        end
-        next if properties.select and !properties.select.call(virtual_row)
-        next if properties.reject and properties.reject.call(virtual_row)
-        @to_a.push virtual_row
-      end
+    if fully_cached?
+      cache.dup
+    else
+      map { |row| row }
     end
-    @to_a
   end
   alias :rows :to_a
   
   # Get a row by row number
   def [](row_number)
-    to_a[row_number]
+    if fully_cached?
+      cache[row_number]
+    else
+      to_a[row_number]
+    end
   end
   
   # clear the row cache to save memory
   def free
-    @to_a.clear if @to_a.is_a?(::Array)
-    @to_a = nil
+    cache.clear
     ::GC.start
     nil
   end
@@ -120,4 +140,28 @@ class RemoteTable
   def transformer
     @transformer ||= Transformer.new self
   end
+  
+  attr_reader :download_count
+  
+  private
+  
+  def mark_download!
+    @download_count ||= 0
+    @download_count += 1
+    if properties.warn_on_multiple_downloads and download_count > 1
+      $stderr.puts "[remote_table] Warning: #{url} has been downloaded #{download_count} times."
+    end
+  end 
+  
+  def fully_cached!
+    @fully_cached = true
+  end
+  
+  def fully_cached?
+    !!@fully_cached
+  end
+  
+  def cache
+    @cache ||= []
+  end
 end

data/lib/remote_table/format.rb

@@ -1,6 +1,7 @@
 if ::RUBY_VERSION >= '1.9'
   require 'ensure/encoding'
 else
+  $KCODE = 'U'
   require 'iconv'
 end
 
@@ -12,9 +13,11 @@ class RemoteTable
     autoload :OpenOffice, 'remote_table/format/open_office'
     autoload :FixedWidth, 'remote_table/format/fixed_width'
     autoload :HTML, 'remote_table/format/html'
+    autoload :XML, 'remote_table/format/xml'
     
     autoload :Textual, 'remote_table/format/mixins/textual'
-    autoload :Rooable, 'remote_table/format/mixins/rooable'
+    autoload :ProcessedByRoo, 'remote_table/format/mixins/processed_by_roo'
+    autoload :ProcessedByNokogiri, 'remote_table/format/mixins/processed_by_nokogiri'
     
     attr_reader :t
 
@@ -26,7 +29,13 @@ class RemoteTable
       if ::RUBY_VERSION >= '1.9'
         str.ensure_encoding 'UTF-8', :external_encoding => t.properties.encoding, :invalid_characters => :transcode
       else
-        ::Iconv.conv('UTF-8//TRANSLIT', t.properties.encoding[0], str + ' ')[0..-2]
+        return str if t.properties.encoding[0] =~ /utf.?8/i
+        begin
+          ::Iconv.conv('UTF-8//TRANSLIT', t.properties.encoding[0], str + ' ')[0..-2]
+        rescue ::Iconv::IllegalSequence
+          $stderr.puts "[remote_table] Unable to transliterate #{str} into UTF-8 given #{t.properties.encoding[0]}"
+          str
+        end
       end
     end
     

data/lib/remote_table/format/delimited.rb

@@ -6,7 +6,7 @@ else
     require 'fastercsv'
     ::RemoteTable::CSV = ::FasterCSV
   rescue ::LoadError
-    $stderr.puts "[remote_table gem] You probably need to manually install the fastercsv gem and/or require it in your Gemfile."
+    $stderr.puts "[remote_table] You probably need to manually install the fastercsv gem and/or require it in your Gemfile."
     raise $!
   end
 end
@@ -19,26 +19,18 @@ class RemoteTable
         remove_useless_characters!
         skip_rows!
         CSV.foreach(t.local_file.path, fastercsv_options) do |row|
-          ordered_hash = ::ActiveSupport::OrderedHash.new
-          filled_values = 0
-          case row
-          when CSV::Row
-            row.each do |header, value|
-              next if header.blank?
-              value = '' if value.nil?
-              ordered_hash[header] = utf8 value
-              filled_values += 1 if value.present?
-            end
-          when ::Array
-            index = 0
-            row.each do |value|
-              value = '' if value.nil?
-              ordered_hash[index] = utf8 value
-              filled_values += 1 if value.present?
-              index += 1
+          if row.is_a?(CSV::Row)
+            output = row.inject(::ActiveSupport::OrderedHash.new) do |memo, (key, value)|
+              if key.present?
+                value = '' if value.nil?
+                memo[key] = utf8 value
+              end
+              memo
             end
+            yield output if t.properties.keep_blank_rows or output.any? { |k, v| v.present? }
+          else
+            yield row if t.properties.keep_blank_rows or row.any? { |v| v.present? }
           end
-          yield ordered_hash if t.properties.keep_blank_rows or filled_values > 0
         end
       ensure
         t.local_file.delete
@@ -62,7 +54,7 @@ class RemoteTable
       def fastercsv_options
         hsh = t.options.slice *FASTERCSV_OPTIONS
         hsh.merge! 'skip_blanks' => !t.properties.keep_blank_rows
-        hsh.reverse_merge! 'headers' => :first_row
+        hsh.reverse_merge! 'headers' => t.properties.headers
         hsh.reverse_merge! 'col_sep' => t.properties.delimiter
         hsh.symbolize_keys
       end

data/lib/remote_table/format/excel.rb

@@ -1,7 +1,7 @@
 class RemoteTable
   class Format
     class Excel < Format
-      include Rooable
+      include ProcessedByRoo
       def roo_class
         ::Excel
       end

data/lib/remote_table/format/excelx.rb

@@ -1,7 +1,7 @@
 class RemoteTable
   class Format
     class Excelx < Format
-      include Rooable
+      include ProcessedByRoo
       def roo_class
         ::Excelx
       end

data/lib/remote_table/format/fixed_width.rb

@@ -8,20 +8,23 @@ class RemoteTable
         crop_rows!
         skip_rows!
         cut_columns!
-        parser.parse[:rows].each do |hash|
-          hash.reject! { |k, v| k.blank? }
-          hash.each do |k, v|
-            hash[k] = utf8 v
+        parser.parse[:rows].each do |row|
+          row.reject! { |k, v| k.blank? }
+          row.each do |k, v|
+            row[k] = utf8 v
           end
-          yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
+          yield row if t.properties.keep_blank_rows or row.any? { |k, v| v.present? }
         end
       ensure
         t.local_file.delete
       end
+      
       private
+      
       def parser
         @parser ||= ::Slither::Parser.new definition, t.local_file.path
       end
+      
       def definition
         @definition ||= if t.properties.schema_name.is_a?(::String) or t.properties.schema_name.is_a?(::Symbol)
           ::Slither.send :definition, t.properties.schema_name

data/lib/remote_table/format/html.rb

@@ -4,37 +4,10 @@ class RemoteTable
   class Format
     class HTML < Format
       include Textual
-      def each(&blk)
-        remove_useless_characters!
-        html_headers = (t.properties.headers.is_a?(::Array)) ? t.properties.headers : nil
-        ::Nokogiri::HTML(unescaped_html_without_soft_hyphens, nil, 'UTF-8').xpath(t.properties.row_xpath).each do |row|
-          values = row.xpath(t.properties.column_xpath).map { |td| td.content.gsub(/\s+/, ' ').strip }
-          if html_headers.nil?
-            html_headers = values
-            next
-          end
-          hash = zip html_headers, values
-          yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
-        end
-      ensure
-        t.local_file.delete
-      end
-
-      private
-
-      # http://snippets.dzone.com/posts/show/406
-      def zip(keys, values)
-        hash = ::Hash.new
-        keys.zip(values) { |k,v| hash[k]=v }
-        hash
-      end
-
-      # should we be doing this in ruby?
-      def unescaped_html_without_soft_hyphens
-        str = ::CGI.unescapeHTML utf8(::IO.read(t.local_file.path))
-        # get rid of MS Office baddies
-        str.gsub! '&shy;', ''
-        str
+      include ProcessedByNokogiri
+      
+      def nokogiri_class
+        ::Nokogiri::HTML::Document
       end
     end
   end

data/lib/remote_table/format/mixins/processed_by_nokogiri.rb

@@ -0,0 +1,67 @@
+require 'nokogiri'
+require 'cgi'
+class RemoteTable
+  class Format
+    module ProcessedByNokogiri
+      def each
+        remove_useless_characters!
+        first_row = true
+        keys = t.properties.headers if t.properties.headers.is_a?(::Array)
+        xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, 'UTF-8')
+        (row_css? ? xml.css(t.properties.row_css) : xml.xpath(t.properties.row_xpath)).each do |row|
+          values = if column_css?
+            row.css(t.properties.column_css)
+          elsif column_xpath?
+            row.xpath(t.properties.column_xpath)
+          else
+            [row]
+          end.map { |cell| cell.content.gsub(/\s+/, ' ').strip }
+          if first_row and t.properties.use_first_row_as_header?
+            keys = values
+            first_row = false
+            next
+          end
+          output = if t.properties.output_class == ::Array
+            values
+          else
+            zip keys, values
+          end
+          if t.properties.keep_blank_rows or values.any?
+            yield output
+          end
+        end
+      ensure
+        t.local_file.delete
+      end
+
+      private
+
+      def row_css?
+        !!t.properties.row_css
+      end
+      
+      def column_css?
+        !!t.properties.column_css
+      end
+      
+      def column_xpath?
+        !!t.properties.column_xpath
+      end
+      
+      # http://snippets.dzone.com/posts/show/406
+      def zip(keys, values)
+        hash = ::ActiveSupport::OrderedHash.new
+        keys.zip(values) { |k,v| hash[k]=v }
+        hash
+      end
+
+      # should we be doing this in ruby?
+      def unescaped_xml_without_soft_hyphens
+        str = ::CGI.unescapeHTML utf8(::IO.read(t.local_file.path))
+        # get rid of MS Office baddies
+        str.gsub! '­', ''
+        str
+      end
+    end
+  end
+end

data/lib/remote_table/format/mixins/processed_by_roo.rb

@@ -0,0 +1,52 @@
+require 'roo'
+class RemoteTable
+  class Format
+    module ProcessedByRoo
+      def each(&blk)
+        spreadsheet = roo_class.new t.local_file.path, nil, :ignore
+        spreadsheet.default_sheet = t.properties.sheet.is_a?(::Numeric) ? spreadsheet.sheets[t.properties.sheet] : t.properties.sheet
+        if t.properties.output_class == ::Array
+          (first_data_row..spreadsheet.last_row).each do |y|
+            output = (1..spreadsheet.last_column).map do |x|
+              spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
+            end
+            yield output if t.properties.keep_blank_rows or output.any? { |v| v.present? }
+          end
+        else
+          keys = {}
+          if t.properties.use_first_row_as_header?
+            (1..spreadsheet.last_column).each do |x|
+              keys[x] = spreadsheet.cell(header_row, x)
+              keys[x] = spreadsheet.cell(header_row - 1, x) if keys[x].blank? # look up
+            end
+          else
+            (1..spreadsheet.last_column).each do |x|
+              keys[x] = t.properties.headers[x - 1]
+            end
+          end
+          (first_data_row..spreadsheet.last_row).each do |y|
+            output = (1..spreadsheet.last_column).inject(::ActiveSupport::OrderedHash.new) do |memo, x|
+              if keys[x].present?
+                memo[keys[x]] = spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
+              end
+              memo
+            end
+            yield output if t.properties.keep_blank_rows or output.any? { |k, v| v.present? }
+          end
+        end
+      ensure
+        t.local_file.delete
+      end
+
+      private
+
+      def header_row
+        1 + t.properties.skip
+      end
+
+      def first_data_row
+        1 + header_row
+      end
+    end
+  end
+end

data/lib/remote_table/format/open_office.rb

@@ -1,7 +1,7 @@
 class RemoteTable
   class Format
     class OpenOffice < Format
-      include Rooable
+      include ProcessedByRoo
       def roo_class
         ::Openoffice
       end

data/lib/remote_table/format/xml.rb

@@ -0,0 +1,14 @@
+require 'nokogiri'
+require 'cgi'
+class RemoteTable
+  class Format
+    class XML < Format
+      include Textual
+      include ProcessedByNokogiri
+      
+      def nokogiri_class
+        ::Nokogiri::XML::Document
+      end
+    end
+  end
+end

data/lib/remote_table/hasher.rb

@@ -12,10 +12,16 @@ class RemoteTable
   class Hasher
     include ::Singleton
     def hash(row)
-      row = row.dup
-      row.stringify_keys!
-      str = row.keys.sort.map do |k|
-        row[k].to_query k
+      str = if row.is_a?(::Array)
+        tmp_ary = []
+        row.each_with_index do |v, i|
+          tmp_ary.push v.to_query(i.to_s)
+        end
+        tmp_ary
+      else
+        row.stringify_keys.keys.sort.map do |k|
+          row[k].to_query k
+        end
       end.join('&')
       ::Digest::MD5.hexdigest str
     end

data/lib/remote_table/properties.rb

@@ -17,9 +17,32 @@ class RemoteTable
       @uri
     end
     
+    # Whether to stream the rows without caching them. Saves memory, but you have to re-download the file every time you...
+    # * call []
+    # * call each
+    # Defaults to false.
+    def streaming
+      t.options['streaming'] || false
+    end
+
+    # Defaults to true.
+    def warn_on_multiple_downloads
+      t.options['warn_on_multiple_downloads'] != false
+    end
+    
     # The headers specified by the user
+    #
+    # Default: :first_row
     def headers
-      t.options['headers']
+      t.options['headers'].nil? ? :first_row : t.options['headers']
+    end
+    
+    def use_first_row_as_header?
+      headers == :first_row
+    end
+    
+    def output_class
+      headers == false ? ::Array : ::ActiveSupport::OrderedHash
     end
     
     # The sheet specified by the user as a number or a string
@@ -52,7 +75,7 @@ class RemoteTable
     #
     # Default: "UTF-8"
     def encoding
-      @encoding ||= ::Array.wrap(t.options['encoding'] || [ 'UTF-8', 'US-ASCII', 'ASCII-8BIT' ])
+      @encoding ||= ::Array.wrap(t.options['encoding'] || [ 'ISO-8859-1', 'US-ASCII', 'WINDOWS-1252', 'ASCII-8BIT', 'UTF-8' ])
     end
     
     # The delimiter
@@ -71,6 +94,16 @@ class RemoteTable
     def column_xpath
       t.options['column_xpath']
     end
+
+    # The CSS selector used to find rows
+    def row_css
+      t.options['row_css']
+    end
+    
+    # The CSS selector used to find columns
+    def column_css
+      t.options['column_css']
+    end
     
     # The compression type.
     #
@@ -205,6 +238,8 @@ class RemoteTable
         Format::FixedWidth
       when /htm/
         Format::HTML
+      when /xml/
+        Format::XML
       else
         Format::Delimited
       end

data/lib/remote_table/transformer.rb

@@ -9,7 +9,7 @@ class RemoteTable
       if legacy_transformer
         legacy_transformer.apply row
       else
-        row
+        [row]
       end
     end
     def legacy_transformer

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = "1.1.10"
+  VERSION = "1.2.0"
 end

data/test/test_old_syntax.rb

@@ -106,16 +106,10 @@ class TestOldSyntax < Test::Unit::TestCase
     assert_equal '2', t.rows.first['dup_header']
   end
   
-  should "respect field order in CSVs without headers" do
+  should "return an Array when instructed not to use headers" do
     t = RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA', :skip => 1, :headers => false)
     t.rows.each do |row|
-      last_column_number = -1
-      row.each do |column_number, v|
-        next if column_number == 'row_hash'
-        assert column_number.is_a?(Numeric)
-        assert(column_number > last_column_number)
-        last_column_number = column_number
-      end
+      assert row.is_a?(::Array)
     end
   end
   

data/test/test_remote_table.rb

@@ -9,7 +9,7 @@ class TestRemoteTable < Test::Unit::TestCase
   
   should "add a row hash to every row" do
     t = RemoteTable.new(:url => 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx')
-    assert_equal "06d8a738551c17735e2731e25c8d0461", t[5]['row_hash']
+    assert_equal "06d8a738551c17735e2731e25c8d0461", t[5].row_hash
   end
   
   should "open a google doc" do
@@ -56,19 +56,40 @@ class TestRemoteTable < Test::Unit::TestCase
   should 'allow itself to be cleared for save memory' do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
     t.to_a
-    assert_equal Array, t.instance_variable_get(:@to_a).class
+    assert t.send(:cache).length > 0
     t.free
-    assert_equal NilClass, t.instance_variable_get(:@to_a).class
+    assert t.send(:cache).length == 0
   end
-  
+    
   # fixes ArgumentError: invalid byte sequence in UTF-8
-  should %{safely strip soft hyphens and read non-utf8 html} do
-    t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :encoding => 'windows-1252', :row_xpath => '//table/tr[2]/td/table/tr', :column_xpath => 'td'
+  should %{safely strip soft hyphens and read windows-1252 html} do
+    t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table/tr[2]/td/table/tr', :column_xpath => 'td'
     assert t.rows.detect { |row| row['Model'] == 'A300B4600' }
   end
   
-  should %{transliterate characters into UTF-8 as long as the user provides the right encoding} do
-    t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
+  should %{transliterate characters from ISO-8859-1} do
+    t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv'
     assert t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }
   end
+  
+  should %{read xml with css selectors} do
+    t = RemoteTable.new 'http://www.nanonull.com/TimeService/TimeService.asmx/getCityTime?city=Chicago', :format => :xml, :row_css => 'string', :headers => false
+    assert /(AM|PM)/.match(t[0][0])
+  end
+  
+  should %{optionally stream rows instead of caching them} do
+    t = RemoteTable.new 'http://www.earthtools.org/timezone/40.71417/-74.00639', :format => :xml, :row_xpath => '//timezone/isotime', :headers => false, :streaming => true
+    time1 = t[0][0]
+    assert /\d\d\d\d-\d\d-\d\d/.match(time1)
+    sleep 1
+    time2 = t[0][0]
+    assert(time1 != time2)
+  end
+  
+  should %{not die when it reads Åland Islands} do
+    t = RemoteTable.new :url => 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';'
+    assert_nothing_raised do
+      t[1][0]
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: remote_table
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 31
   prerelease: 
   segments: 
   - 1
-  - 1
-  - 10
-  version: 1.1.10
+  - 2
+  - 0
+  version: 1.2.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-03 00:00:00 Z
+date: 2011-05-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
@@ -293,9 +293,11 @@ files:
 - lib/remote_table/format/excelx.rb
 - lib/remote_table/format/fixed_width.rb
 - lib/remote_table/format/html.rb
-- lib/remote_table/format/mixins/rooable.rb
+- lib/remote_table/format/mixins/processed_by_nokogiri.rb
+- lib/remote_table/format/mixins/processed_by_roo.rb
 - lib/remote_table/format/mixins/textual.rb
 - lib/remote_table/format/open_office.rb
+- lib/remote_table/format/xml.rb
 - lib/remote_table/hasher.rb
 - lib/remote_table/local_file.rb
 - lib/remote_table/properties.rb

data/lib/remote_table/format/mixins/rooable.rb

@@ -1,49 +0,0 @@
-require 'roo'
-class RemoteTable
-  class Format
-    module Rooable
-      def each(&blk)
-        spreadsheet = roo_class.new t.local_file.path, nil, :ignore
-        spreadsheet.default_sheet = t.properties.sheet.is_a?(::Numeric) ? spreadsheet.sheets[t.properties.sheet] : t.properties.sheet
-        column_references = ::Hash.new
-        if t.properties.headers == false
-          # zero-based numeric keys
-          for col in (1..spreadsheet.last_column)
-            column_references[col] = col - 1
-          end
-        elsif t.properties.headers.is_a? ::Array
-          # names
-          for col in (1..spreadsheet.last_column)
-            column_references[col] = t.properties.headers[col - 1]
-          end
-        else
-          # read t.properties.headers from the file itself
-          for col in (1..spreadsheet.last_column)
-            column_references[col] = spreadsheet.cell(header_row, col)
-            column_references[col] = spreadsheet.cell(header_row - 1, col) if column_references[col].blank? # lspreadsheetk up
-          end
-        end
-        first_data_row.upto(spreadsheet.last_row) do |raw_row|
-          ordered_hash = ::ActiveSupport::OrderedHash.new
-          for col in (1..spreadsheet.last_column)
-            next if column_references[col].blank?
-            ordered_hash[column_references[col]] = spreadsheet.cell(raw_row, col).to_s.gsub(/<[^>]+>/, '').strip
-          end
-          yield ordered_hash if t.properties.keep_blank_rows or ordered_hash.any? { |k, v| v.present? }
-        end
-      ensure
-        t.local_file.delete
-      end
-
-      private
-
-      def header_row
-        1 + t.properties.skip
-      end
-
-      def first_data_row
-        1 + header_row
-      end
-    end
-  end
-end