remote_table 1.4.0 → 2.0.0

data/lib/remote_table/transformer.rb

@@ -3,6 +3,7 @@ class RemoteTable
     attr_reader :t
     def initialize(t)
       @t = t
+      @legacy_transformer_mutex = ::Mutex.new
     end
     # eventually this will support a different way of specifying a transformer
     def transform(row)
@@ -14,12 +15,15 @@ class RemoteTable
     end
     def legacy_transformer
       return @legacy_transformer[0] if @legacy_transformer.is_a?(::Array)
-      memo = if (transform_options = t.config.user_specified_options[:transform])
-        transform_options = transform_options.symbolize_keys
-        transform_options[:class].new transform_options.except(:class)
+      @legacy_transformer_mutex.synchronize do
+        return @legacy_transformer[0] if @legacy_transformer.is_a?(::Array)
+        memo = if (transform_settings = t.transform_settings)
+          transform_settings = transform_settings.symbolize_keys
+          transform_settings[:class].new transform_settings.except(:class)
+        end
+        @legacy_transformer = [memo]
+        memo
       end
-      @legacy_transformer = [memo]
-      memo
     end
   end
 end

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = "1.4.0"
+  VERSION = "2.0.0"
 end

data/lib/remote_table/xls.rb

@@ -0,0 +1,11 @@
+class RemoteTable
+  # Parses XLS files using Roo's Excel class.
+  module Xls
+    def self.extended(base)
+      base.extend ProcessedByRoo
+    end
+    def roo_class
+      ::Excel
+    end
+  end
+end

data/lib/remote_table/xlsx.rb

@@ -0,0 +1,11 @@
+class RemoteTable
+  # Parses XLSX files using Roo's Excelx class.
+  module Xlsx
+    def self.extended(base)
+      base.extend ProcessedByRoo
+    end
+    def roo_class
+      ::Excelx
+    end
+  end
+end

data/lib/remote_table/xml.rb

@@ -0,0 +1,13 @@
+class RemoteTable
+  # Parses XML files using Nokogiri's Nokogiri::XML::Document class.
+  module Xml
+    def self.extended(base)
+      base.extend Plaintext
+      base.extend ProcessedByNokogiri
+    end
+    
+    def nokogiri_class
+      ::Nokogiri::XML::Document
+    end
+  end
+end

data/lib/remote_table/yaml.rb

@@ -0,0 +1,14 @@
+class RemoteTable
+  module Yaml
+    def _each
+      require 'yaml'
+      
+      data = ::YAML.load_file local_copy.path
+      data.each do |row|
+        yield row
+      end
+    ensure
+      local_copy.cleanup
+    end
+  end
+end

data/remote_table.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |s|
   s.authors     = ["Seamus Abshere", "Andy Rossmeissl"]
   s.email       = ["seamus@abshere.net"]
   s.homepage    = "https://github.com/seamusabshere/remote_table"
-  s.summary     = "Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV, TSV (tab separated), other delimited, fixed-width files."
-  s.description = %q{Gives you a standard way to parse various formats and treat them as an array of hashes.}
+  s.summary     = %{Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma separated), TSV (tab separated), other delimited, fixed-width files.}
+  s.description = %{Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma separated), TSV (tab separated), other delimited, fixed-width files. Returns an Array of Arrays or Hashes, depending on whether there are headers.}
 
   s.rubyforge_project = "remotetable"
 

data/test/test_big.rb

@@ -31,7 +31,7 @@ describe RemoteTable do
                           :format => :fixed_width,
                           :crop => 21..26, # inclusive
                           :cut => '2-',
-                          :select => lambda { |row| /\A[A-Z]/.match row['code'] },
+                          :select => proc { |row| /\A[A-Z]/.match row['code'] },
                           :schema => [[ 'code',   2, { :type => :string }  ],
                                       [ 'spacer', 2 ],
                                       [ 'name',   52, { :type => :string } ]])

data/test/test_remote_table.rb

@@ -4,20 +4,25 @@ require 'tempfile'
 
 describe RemoteTable do
   it "open an XLSX" do
+    t = RemoteTable.new 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
+    t[5]["Requirements"].must_equal "Secure encryption of all data"
+  end
+
+  it "does its best to download urls without http://" do
     t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
     t[5]["Requirements"].must_equal "Secure encryption of all data"
   end
-  
+
   it "add a row hash to every row" do
-    t = RemoteTable.new(:url => 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx')
+    t = RemoteTable.new(:url => 'http://www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx')
     t[5].row_hash.must_equal "06d8a738551c17735e2731e25c8d0461"
   end
-  
+
   it "open a google doc" do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
     t[0]['name'].must_equal 'Seamus Abshere'
   end
-  
+
   it "open a csv with custom headers" do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :headers => %w{ col1 col2 col3 }
     t[0]['col2'].must_equal 'name'
@@ -31,29 +36,29 @@ describe RemoteTable do
     t[1]['name'].must_equal 'Derek Kastner'
     t[1]['city'].must_equal 'Lansing'
   end
-  
+
   it "return an ordered hash" do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
     t[0].class.must_equal ::ActiveSupport::OrderedHash
   end
-  
+
   it "pass through fastercsv options" do
     f = Tempfile.new 'pass-through-fastercsv-options'
     f.write %{3,Title example,Body example with a <a href="">link</a>,test category}
     f.flush
-    t = RemoteTable.new "file://#{f.path}", :quote_char => %{'}, :headers => nil
+    t = RemoteTable.new "file://#{f.path}", :quote_char => %{'}, :headers => nil # this should really be "headers => false"
     t[0][2].must_equal %{Body example with a <a href="">link</a>}
     f.close
   end
-  
+
   it "open a csv inside a zip file" do
     t = RemoteTable.new  'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
                          :filename => 'Annex Tables/Annex 3/Table A-93.csv',
                          :skip => 1,
-                         :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] }
+                         :select => proc { |row| row['Vehicle Age'].strip =~ /^\d+$/ }
     t[0]['LDGV'].must_equal '9.09%'
   end
-  
+
   it 'not blow up if each is called twice' do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
     count = 0
@@ -62,7 +67,7 @@ describe RemoteTable do
     t.each { |row| count += 1}
     count.must_equal first_run*2
   end
-  
+
   it 'allow itself to be cleared for save memory' do
     t = RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
     t.to_a
@@ -70,23 +75,23 @@ describe RemoteTable do
     t.free
     t.send(:cache).length.must_equal 0
   end
-    
+
   # fixes ArgumentError: invalid byte sequence in UTF-8
   it %{safely strip soft hyphens and read windows-1252 html} do
     t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table[2]//table[1]//tr[3]//tr', :column_xpath => 'td', :encoding => 'windows-1252'
     t.rows.detect { |row| row['Model'] == 'A300B4600' }.wont_equal nil
   end
-  
+
   it %{transliterate characters from ISO-8859-1} do
     t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
     t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }.wont_equal nil
   end
-  
+
   it %{read xml with css selectors} do
     t = RemoteTable.new 'http://www.nanonull.com/TimeService/TimeService.asmx/getCityTime?city=Chicago', :format => :xml, :row_css => 'string', :headers => false
     /(AM|PM)/.match(t[0][0]).wont_equal nil
   end
-  
+
   it %{optionally stream rows instead of caching them} do
     t = RemoteTable.new 'http://www.earthtools.org/timezone/40.71417/-74.00639', :format => :xml, :row_xpath => '//timezone/isotime', :headers => false, :streaming => true
     time1 = t[0][0]
@@ -95,7 +100,7 @@ describe RemoteTable do
     time2 = t[0][0]
     time1.wont_equal time2
   end
-  
+
   {
   # IMPOSSIBLE "../support/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls" => {:format=>"xls",         :encoding=>"binary"},
   "../support/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx"         => {:format=>"xlsx"},
@@ -106,7 +111,7 @@ describe RemoteTable do
   # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html"          => {:format=>"html" },
   # TODO "../support/list-en1-semic-3.office-2011-for-mac-sp1.iso-8859-1.html"     => {:format=>"html", :encoding=>"iso-8859-1"},
   # TODO "../support/list-en1-semic-3.neooffice.utf-8.html"                        => {:format=>"html" },
-  "../support/list-en1-semic-3.neooffice.utf-8.xml"                         => {:format=>"xml", :row_css=>'Row', :column_css => 'Data', :select => lambda { |row| row[1].to_s =~ /[A-Z]{2}/ }},
+  "../support/list-en1-semic-3.neooffice.utf-8.xml"                         => {:format=>"xml", :row_css=>'Row', :column_css => 'Data', :select => proc { |row| row[1].to_s =~ /[A-Z]{2}/ }},
   "../support/list-en1-semic-3.neooffice.iso-8859-1.csv"                    => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
   "../support/list-en1-semic-3.original.iso-8859-1.csv"                     => {:format=>"csv", :encoding=>"iso-8859-1", :delimiter => ';'},
   "../support/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma"       => {:format=>"csv", :encoding=>"MACROMAN"}, # comma because no option in excel
@@ -124,22 +129,22 @@ describe RemoteTable do
       b.must_equal a
     end
   end
-  
+
   it %{recode as UTF-8 even ISO-8859-1 (or any other encoding)} do
     t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';', :encoding => 'ISO-8859-1'
     t[1][0].must_equal %{ÅLAND ISLANDS}
   end
-  
+
   it %{parse a big CSV that is not UTF-8} do
     t = RemoteTable.new 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat', :headers => false#, :encoding => 'UTF-8'
     t[0][1].must_equal 'Goroka'
   end
-  
+
   it "read only certain rows of an XLSX" do
     t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => false
     t[0][0].must_equal "Permissioning and access groups for all content"
     t[4][0].must_equal "Manage Multiple Incentive Programs for Participants"
-    
+
     t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :crop => 11..16, :headers => %w{ col1 }
     t[0]['col1'].must_equal "Permissioning and access groups for all content"
     t[4]['col1'].must_equal "Manage Multiple Incentive Programs for Participants"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: remote_table
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-12 00:00:00.000000000 Z
+date: 2012-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -124,8 +124,9 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Gives you a standard way to parse various formats and treat them as an
-  array of hashes.
+description: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma
+  separated), TSV (tab separated), other delimited, fixed-width files. Returns an
+  Array of Arrays or Hashes, depending on whether there are headers.
 email:
 - seamus@abshere.net
 executables: []
@@ -140,22 +141,20 @@ files:
 - README.markdown
 - Rakefile
 - lib/remote_table.rb
-- lib/remote_table/config.rb
-- lib/remote_table/format.rb
-- lib/remote_table/format/delimited.rb
-- lib/remote_table/format/excel.rb
-- lib/remote_table/format/excelx.rb
-- lib/remote_table/format/fixed_width.rb
-- lib/remote_table/format/html.rb
-- lib/remote_table/format/mixins/processed_by_nokogiri.rb
-- lib/remote_table/format/mixins/processed_by_roo.rb
-- lib/remote_table/format/mixins/textual.rb
-- lib/remote_table/format/open_office.rb
-- lib/remote_table/format/xml.rb
-- lib/remote_table/format/yaml.rb
-- lib/remote_table/local_file.rb
+- lib/remote_table/delimited.rb
+- lib/remote_table/fixed_width.rb
+- lib/remote_table/html.rb
+- lib/remote_table/local_copy.rb
+- lib/remote_table/ods.rb
+- lib/remote_table/plaintext.rb
+- lib/remote_table/processed_by_nokogiri.rb
+- lib/remote_table/processed_by_roo.rb
 - lib/remote_table/transformer.rb
 - lib/remote_table/version.rb
+- lib/remote_table/xls.rb
+- lib/remote_table/xlsx.rb
+- lib/remote_table/xml.rb
+- lib/remote_table/yaml.rb
 - remote_table.gemspec
 - test/fixtures/data.yml
 - test/helper.rb
@@ -201,8 +200,8 @@ rubyforge_project: remotetable
 rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
-summary: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV, TSV (tab
-  separated), other delimited, fixed-width files.
+summary: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma
+  separated), TSV (tab separated), other delimited, fixed-width files.
 test_files:
 - test/fixtures/data.yml
 - test/helper.rb

data/lib/remote_table/config.rb

@@ -1,251 +0,0 @@
-require 'uri'
-class RemoteTable
-  # Represents the config of a RemoteTable, whether they are explicitly set by the user or inferred automatically.
-  class Config
-    attr_reader :t
-    attr_reader :user_specified_options
-    
-    def initialize(t, user_specified_options)
-      @t = t
-      @user_specified_options = user_specified_options
-    end
-            
-    # The parsed URI of the file to get.
-    def uri
-      return @uri if @uri.is_a?(::URI)
-      @uri = ::URI.parse t.url
-      if @uri.host == 'spreadsheets.google.com' or @uri.host == 'docs.google.com'
-        @uri.query = 'output=csv&' + @uri.query.sub(/\&?output=.*?(\&|\z)/, '\1')
-      end
-      @uri
-    end
-    
-    # Whether to stream the rows without caching them. Saves memory, but you have to re-download the file every time you...
-    # * call []
-    # * call each
-    # Defaults to false.
-    def streaming
-      user_specified_options.fetch :streaming, false
-    end
-
-    # Defaults to true.
-    def warn_on_multiple_downloads
-      user_specified_options[:warn_on_multiple_downloads] != false
-    end
-    
-    # The headers specified by the user
-    #
-    # Default: :first_row
-    def headers
-      user_specified_options[:headers].nil? ? :first_row : user_specified_options[:headers]
-    end
-    
-    def use_first_row_as_header?
-      headers == :first_row
-    end
-    
-    def output_class
-      headers == false ? ::Array : ::ActiveSupport::OrderedHash
-    end
-    
-    # The sheet specified by the user as a number or a string
-    def sheet
-      user_specified_options[:sheet]
-    end
-    
-    # Whether to keep blank rows
-    #
-    # Default: false
-    def keep_blank_rows
-      user_specified_options.fetch :keep_blank_rows, false
-    end
-    
-    # Form data to send in with the download request
-    def form_data
-      user_specified_options[:form_data]
-    end
-    
-    # How many rows to skip
-    #
-    # Default: 0
-    def skip
-      user_specified_options.fetch :skip, 0
-    end
-    
-    def internal_encoding
-      user_specified_options.fetch(:encoding, 'UTF-8').upcase
-    end
-    
-    def external_encoding
-      'UTF-8'
-    end
-    
-    def external_encoding_iconv
-      'UTF-8//TRANSLIT'
-    end
-    
-    # The delimiter
-    #
-    # Default: ","
-    def delimiter
-      user_specified_options.fetch :delimiter, ','
-    end
-    
-    # The XPath used to find rows
-    def row_xpath
-      user_specified_options[:row_xpath]
-    end
-    
-    # The XPath used to find columns
-    def column_xpath
-      user_specified_options[:column_xpath]
-    end
-
-    # The CSS selector used to find rows
-    def row_css
-      user_specified_options[:row_css]
-    end
-    
-    # The CSS selector used to find columns
-    def column_css
-      user_specified_options[:column_css]
-    end
-    
-    # The compression type.
-    #
-    # Default: guessed from URI.
-    #
-    # Can be specified as: :gz, :zip, :bz2, :exe (treated as :zip)
-    def compression
-      if user_specified_options.has_key?(:compression)
-        return user_specified_options[:compression]
-      end
-      case ::File.extname(uri.path).downcase
-      when /gz/, /gunzip/
-        :gz
-      when /zip/
-        :zip
-      when /bz2/, /bunzip2/
-        :bz2
-      when /exe/
-        :exe
-      end
-    end
-    
-    # The packing type.
-    #
-    # Default: guessed from URI.
-    #
-    # Can be specified as: :tar
-    def packing
-      if user_specified_options.has_key?(:packing)
-        return user_specified_options[:packing]
-      end
-      if uri.path =~ %r{\.tar(?:\.|$)}i
-        :tar
-      end
-    end
-    
-    # The glob used to pick a file out of an archive.
-    #
-    # Example:
-    #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :glob => '/*.csv'
-    def glob
-      user_specified_options[:glob]
-    end
-    
-    # The filename, which can be used to pick a file out of an archive.
-    #
-    # Example:
-    #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv'
-    def filename
-      user_specified_options[:filename]
-    end
-    
-    # Cut columns up to this character
-    def cut
-      user_specified_options[:cut]
-    end
-    
-    # Crop rows after this line
-    def crop
-      user_specified_options[:crop]
-    end
-    
-    # The fixed-width schema, given as an array
-    #
-    # Example:
-    #     RemoteTable.new('http://cloud.github.com/downloads/seamusabshere/remote_table/test2.fixed_width.txt',
-    #                      :format => :fixed_width,
-    #                      :skip => 1,
-    #                      :schema => [[ 'header4', 10, { :type => :string }  ],
-    #                                  [  'spacer',  1 ],
-    #                                  [  'header5', 10, { :type => :string } ],
-    #                                  [  'spacer',  12 ],
-    #                                  [  'header6', 10, { :type => :string } ]])
-    def schema
-      user_specified_options[:schema]
-    end
-    
-    # The name of the fixed-width schema according to FixedWidth
-    def schema_name
-      user_specified_options[:schema_name]
-    end
-    
-    # A proc to call to decide whether to return a row.
-    def select
-      user_specified_options[:select]
-    end
-    
-    # A proc to call to decide whether to return a row.
-    def reject
-      user_specified_options[:reject]
-    end
-    
-    # A hash of options to create a new Errata instance (see the Errata gem at http://github.com/seamusabshere/errata) to be used on every row.
-    def errata
-      return unless user_specified_options.has_key? :errata
-      @errata ||= if user_specified_options[:errata].is_a? ::Hash
-        ::Errata.new user_specified_options[:errata]
-      else
-        user_specified_options[:errata]
-      end
-    end
-    
-    # Get the format in the form of RemoteTable::Format::Excel, etc.
-    #
-    # Note: treats all spreadsheets.google.com URLs as Format::Delimited (i.e., CSV)
-    #
-    # Default: guessed from file extension (which is usually the same as the URI, but sometimes not if you pick out a specific file from an archive)
-    #
-    # Can be specified as: :xlsx, :xls, :delimited (aka :csv and :tsv), :ods, :fixed_width, :html
-    def format
-      return Format::Delimited if uri.host == 'spreadsheets.google.com' or @uri.host == 'docs.google.com'
-      clue = if user_specified_options.has_key?(:format)
-        user_specified_options[:format]
-      else
-        t.local_file.path
-      end
-      case clue.to_s.downcase
-      when /xlsx/, /excelx/
-        Format::Excelx
-      when /xls/, /excel/
-        Format::Excel
-      when /csv/, /tsv/, /delimited/
-        Format::Delimited
-      when /ods/, /open_?office/
-        Format::OpenOffice
-      when /fixed_?width/
-        Format::FixedWidth
-      when /htm/
-        Format::HTML
-      when /xml/
-        Format::XML
-      when /yaml/, /yml/
-        Format::Yaml
-      else
-        Format::Delimited
-      end
-    end
-  end
-end