remote_table 3.0.0.alpha → 3.0.0.beta

data/CHANGELOG

@@ -1,3 +1,15 @@
+3.0.0.beta / 2013-07-30
+
+* Breaking changes
+
+  * Strip whitespace from headers unless you provide them as an array
+  * Not passing anything options on to CSV... use :delimiter instead of :col_sep
+  * Include columns with blank headers as "empty_N"
+
+* Enhancements
+
+  * Support relative paths with spaces
+
 3.0.0.alpha / 2013-07-25
 
 * Breaking changes

data/lib/remote_table.rb

@@ -53,7 +53,7 @@ class RemoteTable
     # Guess compression based on URL. Used internally.
     # @return [Symbol,nil]
     def guess_compression(url)
-      extname = ::File.extname(::URI.parse(url).path).downcase
+      extname = extname(url).downcase
       case extname
       when /gz/, /gunzip/
         :gz
@@ -69,7 +69,7 @@ class RemoteTable
     # Guess packing from URL. Used internally.
     # @return [Symbol,nil]
     def guess_packing(url)
-      basename = ::File.basename(::URI.parse(url).path).downcase
+      basename = basename(url).downcase
       if basename.include?('.tar') or basename.include?('.tgz')
         :tar
       end
@@ -109,6 +109,24 @@ class RemoteTable
       uri.query = params.join('&')
       uri.to_s
     end
+
+    private
+
+    def basename(url)
+      ::File.basename path(url)
+    end
+
+    def extname(url)
+      ::File.extname path(url)
+    end
+
+    def path(url)
+      if url.include?('://')
+        ::URI.parse(url).path
+      else
+        File.expand_path url
+      end
+    end
   end
 
   EXTERNAL_ENCODING = 'UTF-8'
@@ -129,11 +147,13 @@ class RemoteTable
     :keep_blank_rows => false,
     :skip => 0,
     :encoding => 'UTF-8',
-    :delimiter => ','
+    :delimiter => ',',
+    :quote_char => '"',
   }
   OLD_SETTING_NAMES = {
     :pre_select => [:select],
     :pre_reject => [:reject],
+    :delimiter  => [:col_sep],
   }
 
   include ::Enumerable
@@ -178,6 +198,13 @@ class RemoteTable
   # Headers specified by the user: +:first_row+ (the default), +false+, or a list of headers.
   # @return [:first_row,false,Array<String>]
   attr_reader :headers
+
+  # Quote character for delimited files.
+  #
+  # Defaults to double quotes.
+  #
+  # @return [String]
+  attr_reader :quote_char
     
   # The sheet specified by the user as a number or a string.
   # @return[String,Integer]
@@ -199,7 +226,7 @@ class RemoteTable
   # @return [String]
   attr_reader :encoding
   
-  # The delimiter, a.k.a. column separator. Passed to Ruby CSV as +:col_sep+. Default is :delimited.
+  # The delimiter, a.k.a. column separator. Passed to Ruby CSV as +:col_sep+. Default is ','.
   # @return [String]
   attr_reader :delimiter
   
@@ -377,6 +404,7 @@ class RemoteTable
     if headers.is_a?(::Array) and headers.any?(&:blank?)
       raise ::ArgumentError, "[remote_table] If you specify headers, none of them can be blank"
     end
+    @quote_char = grab settings, :quote_char
 
     @compression = grab(settings, :compression) || RemoteTable.guess_compression(url)
     @packing = grab(settings, :packing) || RemoteTable.guess_packing(url)
@@ -422,6 +450,7 @@ class RemoteTable
       end
     else
       mark_download!
+      preprocess!
       memo = _each do |row|
         parser.parse(row).each do |virtual_row|
           virtual_row.row_hash = ::HashDigest.hexdigest row
@@ -481,6 +510,10 @@ class RemoteTable
   end
 
   private
+
+  def preprocess!
+    # noop, overridden sometimes
+  end
   
   def mark_download!
     @download_count_mutex.synchronize do

data/lib/remote_table/delimited.rb

@@ -14,24 +14,15 @@ class RemoteTable
       Engine = ::FasterCSV
     end
 
-    PASSTHROUGH_CSV_SETTINGS = [
-      :unconverted_fields,
-      :col_sep,
-      :row_sep,
-      :return_headers,
-      :header_converters,
-      :quote_char,
-      :converters,
-      :force_quotes,
-    ]
-
-    # Yield each row using Ruby's CSV parser (FasterCSV on Ruby 1.8).
-    def _each
+    def preprocess!
       delete_harmful!
       convert_eol_to_unix!
       transliterate_whole_file_to_utf8!
       skip_rows!
+    end
 
+    # Yield each row using Ruby's CSV parser (FasterCSV on Ruby 1.8).
+    def _each
       Engine.new(local_copy.encoded_io, csv_options).each do |row|
 
         some_value_present = false
@@ -55,7 +46,6 @@ class RemoteTable
           # represent the row as a hash
           hash = ::ActiveSupport::OrderedHash.new
           row.each do |k, v|
-            next unless k.present?
             v = v.to_s
             if not some_value_present and not keep_blank_rows and v.present?
               some_value_present = true
@@ -81,11 +71,31 @@ class RemoteTable
     #
     # @return [Hash]
     def csv_options
-      memo = other_options.slice(*PASSTHROUGH_CSV_SETTINGS)
-      memo[:skip_blanks] = !keep_blank_rows
-      memo[:headers] ||= headers
-      memo[:col_sep] ||= delimiter
-      memo
+      {
+        skip_blanks: !keep_blank_rows,
+        headers:     headers,
+        col_sep:     delimiter,
+        quote_char:  quote_char,
+      }
+    end
+
+    def headers
+      return @_headers if defined?(@_headers)
+      @_headers = case @headers
+      when FalseClass, NilClass
+        false
+      when :first_row, TrueClass
+        i = 0
+        line = local_copy.encoded_io.gets
+        Engine.parse_line(line).map do |v|
+          header = v.to_s.gsub(/\s+/, ' ').strip
+          header.present? ? header : "empty_#{i+=1}"
+        end
+      when Array
+        @headers
+      else
+        raise "Invalid headers: #{headers.inspect}"
+      end
     end
   end
 end

data/lib/remote_table/fixed_width.rb

@@ -13,15 +13,17 @@ class RemoteTable
       @definition_mutex = ::Mutex.new
     end
 
-    def _each
-      require 'fixed_width-multibyte'
-
+    def preprocess!
       delete_harmful!
       convert_eol_to_unix!
       transliterate_whole_file_to_utf8!
       crop_rows!
       skip_rows!
       cut_columns!
+    end
+
+    def _each
+      require 'fixed_width-multibyte'
 
       fixed_width_parser.parse[:rows].each do |row|
         some_value_present = false

data/lib/remote_table/processed_by_nokogiri.rb

@@ -5,6 +5,11 @@ class RemoteTable
     SINGLE_SPACE = ' '
     SOFT_HYPHEN = '­'
 
+    def preprocess!
+      delete_harmful!
+      transliterate_whole_file_to_utf8!
+    end
+
     # Yield each row using Nokogiri.
     def _each
       require 'nokogiri'
@@ -16,9 +21,6 @@ class RemoteTable
       unless row_css or row_xpath
         raise ::ArgumentError, "[remote_table] Need :row_css or :row_xpath in order to process XML or HTML"
       end
-
-      delete_harmful!
-      transliterate_whole_file_to_utf8!
       
       xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, RemoteTable::EXTERNAL_ENCODING)
       (row_css ? xml.css(row_css) : xml.xpath(row_xpath)).each do |row|

data/lib/remote_table/version.rb

@@ -1,3 +1,3 @@
 class RemoteTable
-  VERSION = '3.0.0.alpha'
+  VERSION = '3.0.0.beta'
 end

data/remote_table.gemspec

@@ -31,4 +31,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yard'
   s.add_development_dependency 'pry'
+  s.add_development_dependency 'pry-rescue'
+  s.add_development_dependency 'pry-stack_explorer'
 end

data/test/data/lots of spaces.csv

@@ -0,0 +1,2 @@
+a one, b two, c three ,
+a1,b2,c3

data/test/data/table.html

@@ -0,0 +1,17 @@
+<table>
+  <tr>
+    <td>h1</td>
+    <td>h2</td>
+    <td>h3</td>
+  </tr>
+  <tr>
+    <td>a</td>
+    <td>b</td>
+    <td>c</td>
+  </tr>
+  <tr>
+    <td>d</td>
+    <td>e</td>
+    <td>f</td>
+  </tr>
+</table>

data/test/helper.rb

@@ -1,7 +1,9 @@
 require 'bundler/setup'
+
 require 'minitest/spec'
-require 'minitest/autorun'
 require 'minitest/reporters'
+require 'minitest/autorun'
+# require 'pry-rescue/minitest'
 #MiniTest::Unit.runner = MiniTest::SuiteRunner.new
 #MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 require 'remote_table'

data/test/test_remote.rb

@@ -36,11 +36,12 @@ describe RemoteTable do
     end
 
     it "open a csv inside a zip file" do
-      t = RemoteTable.new  'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-                           :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-                           :skip => 1,
-                           :select => proc { |row| row['Vehicle Age'].strip =~ /^\d+$/ }
-      t[0]['LDGV'].must_equal '9.09%'
+      t = RemoteTable.new('http://www.epa.gov/climatechange/Downloads/ghgemissions/2011-Annex-Tables.zip',
+            :filename => 'Annex Tables/Table A-93.csv',
+            :skip => 1,
+            :headers => %w{ age LDGV LDGT HDGV LDDV LDDT HDDT MC },
+            :select => proc { |row| row['age'].to_i.to_s == row['age'] })
+      t[0]['LDGV'].must_equal '5.20%'
     end
 
     it 'not blow up if each is called twice' do
@@ -60,12 +61,6 @@ describe RemoteTable do
       t.send(:cache).length.must_equal 0
     end
 
-    # fixes ArgumentError: invalid byte sequence in UTF-8
-    it %{safely strip soft hyphens and read windows-1252 html} do
-      t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-A.htm", :row_xpath => '//table[2]//table[1]//tr[3]//tr', :column_xpath => 'td', :encoding => 'windows-1252'
-      t.rows.detect { |row| row['Model'] == 'A300B4600' }.wont_equal nil
-    end
-
     it %{transliterate characters from ISO-8859-1} do
       t = RemoteTable.new :url => 'http://static.brighterplanet.com/science/data/consumables/pets/breed_genders.csv', :encoding => 'ISO-8859-1'
       t.rows.detect { |row| row['name'] == 'Briquet Griffon Vendéen' }.wont_equal nil
@@ -85,8 +80,8 @@ describe RemoteTable do
       time1.wont_equal time2
     end
 
-    it %{recode as UTF-8 even ISO-8859-1 (or any other encoding)} do
-      t = RemoteTable.new 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';', :encoding => 'ISO-8859-1'
+    it %{reads country names} do
+      t = RemoteTable.new 'http://www.iso.org/iso/country_names_and_code_elements_txt', :skip => 1, :headers => false, :delimiter => ';'
       t[1][0].must_equal %{ÅLAND ISLANDS}
     end
 

data/test/test_remote_table.rb

@@ -14,6 +14,13 @@ describe RemoteTable do
     by_path.rows.must_equal by_url.rows
   end
 
+  it "strips whitespace from headers" do
+    t = RemoteTable.new 'test/data/lots of spaces.csv'
+    t[0]['a one'].must_equal 'a1'
+    t[0]['b two'].must_equal 'b2'
+    t[0]['c three'].must_equal 'c3'
+  end
+
   {
   # IMPOSSIBLE "../data/list-en1-semic-3.office-2011-for-mac-sp1-excel-95.binary.xls" => {:format=>"xls",         :encoding=>"binary"},
   "../data/list-en1-semic-3.office-2011-for-mac-sp1.binary.xlsx"         => {:format=>"xlsx"},
@@ -59,4 +66,18 @@ describe RemoteTable do
     t[1]['name'].must_equal 'Derek Kastner'
     t[1]['city'].must_equal 'Lansing'
   end
+
+  it "reads html with xpath" do
+    t = RemoteTable.new 'test/data/table.html', row_xpath: '//tr', column_xpath: 'td'
+    t[0]['h1'].must_equal 'a'
+    t[1]['h3'].must_equal 'f'
+  end
+
+  # fixes ArgumentError: invalid byte sequence in UTF-8
+  # disabled because xpath not be somehow broken - works in chrome
+  it %{safely strip soft hyphens and read windows-1252 html} do
+    row_xpath = '/html/body/table[2]/tbody/tr/td/center/table/tbody/tr[3]/td/table/tbody/tr[2]/td[1]'
+    t = RemoteTable.new 'test/data/faa-aircraft.html', :row_xpath => row_xpath, :column_xpath => 'td', :encoding => 'windows-1252'
+    t.rows.detect { |row| row['Model'] == 'A300B4600' }.wont_equal nil
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: remote_table
 version: !ruby/object:Gem::Version
-  version: 3.0.0.alpha
+  version: 3.0.0.beta
   prerelease: 6
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-26 00:00:00.000000000 Z
+date: 2013-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -220,6 +220,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma
   separated), TSV (tab separated), other delimited, fixed-width files, and shapefiles.
   Returns an Array of Arrays or Hashes, depending on whether there are headers.
@@ -269,7 +301,9 @@ files:
 - test/data/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma
 - test/data/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html
 - test/data/list-en1-semic-3.original.iso-8859-1.csv
+- test/data/lots of spaces.csv
 - test/data/ranges.csv
+- test/data/table.html
 - test/helper.rb
 - test/test_big.rb
 - test/test_errata.rb
@@ -322,7 +356,9 @@ test_files:
 - test/data/list-en1-semic-3.office-2011-for-mac-sp1.mac.csv-comma
 - test/data/list-en1-semic-3.office-2011-for-mac-sp1.utf-8.html
 - test/data/list-en1-semic-3.original.iso-8859-1.csv
+- test/data/lots of spaces.csv
 - test/data/ranges.csv
+- test/data/table.html
 - test/helper.rb
 - test/test_big.rb
 - test/test_errata.rb