remote_table 2.1.2 → 3.0.0.alpha

data/CHANGELOG

@@ -1,8 +1,17 @@
-2.1.2 / 2013-10-08
+3.0.0.alpha / 2013-07-25
 
-* Bug fixes
+* Breaking changes
+
+  * Just use :encoding to specify a file's internal/initial/original encoding. No more :internal_encoding, this was jargony.
+  * No more shapefile support - hard to imagine a shapefile as a table
+  * No more :transform, legacy or not
+  * For :errata, now you pass an object that responds to #rejects?(row) and #correct!(row)
+  * No more :errata_settings
+
+* Enhancements
 
-  * Correctly detect format from filename - don't get confused if a filename has "xls" in it (thanks @activefx #10)
+  * :parser option takes an object that responds to #parse(row) and returns an array of one or more rows.
+  * RemoteTable.transpose(url, key_key, value_key) helper
 
 2.1.1 / 2013-03-25
 

data/README.markdown

@@ -1,6 +1,6 @@
 # remote_table
 
-Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma separated), TSV (tab separated), SHP (ESRI shapefiles), other delimited, fixed-width files.
+Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma separated), TSV (tab separated), other delimited, fixed-width files.
 
 Tested on MRI 1.8, MRI 1.9, and JRuby 1.6.7+. Thread-safe.
 
@@ -441,12 +441,6 @@ Everything is forced into UTF-8. You can improve the quality of the conversion b
                                 [ 'spacer',  1 ],
                                 [ 'header1', 10, { :type => :string } ]]
 
-    # ESRI Shapefile from NREL
-    require 'geo_ruby'
-    require 'dbf'
-    RemoteTable.new 'http://www.nrel.gov/gis/cfm/data/GIS_Data_Technology_Specific/United_States/Solar/High_Resolution/Lower_48_DNI_High_Resolution.zip',
-      :format => :shp
-
 ## Requirements
 
 * Unix tools like curl, iconv, perl, cat, cut, tail, etc. accessible from your `$PATH`
@@ -455,7 +449,6 @@ Everything is forced into UTF-8. You can improve the quality of the conversion b
 ## Wishlist
 
 * Win32 compat
-* The new "custom parser" syntax (aka transformer) hasn't been defined yet... only the old-style syntax is available
 
 ## Authors
 

data/lib/remote_table.rb

@@ -14,7 +14,6 @@ end
 require 'hash_digest'
 
 require 'remote_table/local_copy'
-require 'remote_table/transformer'
 
 require 'remote_table/plaintext'
 require 'remote_table/processed_by_roo'
@@ -27,7 +26,6 @@ require 'remote_table/fixed_width'
 require 'remote_table/html'
 require 'remote_table/xml'
 require 'remote_table/yaml'
-require 'remote_table/shp'
 
 class Hash
   # Added by remote_table to store a hash (think checksum) of the data with which a particular Hash is initialized.
@@ -44,6 +42,14 @@ end
 # Open Google Docs spreadsheets, local or remote XLSX, XLS, ODS, CSV (comma separated), TSV (tab separated), other delimited, fixed-width files.
 class RemoteTable
   class << self
+    # Transpose two columns into a mapping from one to the other.
+    def transpose(url, key_key, value_key, options = {})
+      new(url, options).inject({}) do |memo, row|
+        memo[row[key_key]] = row[value_key]
+        memo
+      end
+    end
+
     # Guess compression based on URL. Used internally.
     # @return [Symbol,nil]
     def guess_compression(url)
@@ -72,26 +78,24 @@ class RemoteTable
     # Guess file format from the basename. Since a file might be decompressed and/or pulled out of an archive with a glob, this usually can't be called until a file is downloaded.
     # @return [Symbol,nil]
     def guess_format(basename)
-      case basename.to_s.downcase.strip
-      when /ods\z/, /open_?office\z/
+      case basename.to_s.downcase
+      when /ods/, /open_?office/
         :ods
-      when /xlsx\z/, /excelx\z/
+      when /xlsx/, /excelx/
         :xlsx
-      when /xls\z/, /excel\z/
+      when /xls/, /excel/
         :xls
-      when /csv\z/, /tsv\z/, /delimited\z/
+      when /csv/, /tsv/, /delimited/
         # note that there is no RemoteTable::Csv class - it's normalized to :delimited
         :delimited
-      when /fixed_?width\z/
+      when /fixed_?width/
         :fixed_width
-      when /html?\z/
+      when /htm/
         :html
-      when /xml\z/
+      when /xml/
         :xml
-      when /yaml\z/, /yml\z/
+      when /yaml/, /yml/
         :yaml
-      when /shp\z/
-        :shp
       end
     end
 
@@ -107,14 +111,6 @@ class RemoteTable
     end
   end
 
-  # @private
-  # Here to support legacy code.
-  class Transform
-    def self.row_hash(row)
-      ::HashDigest.hexdigest row
-    end
-  end
-
   EXTERNAL_ENCODING = 'UTF-8'
   EXTERNAL_ENCODING_ICONV = 'UTF-8//TRANSLIT'
   GOOGLE_DOCS_SPREADSHEET = [
@@ -124,7 +120,7 @@ class RemoteTable
   VALID = {
     :compression => [:gz, :zip, :bz2, :exe],
     :packing => [:tar],
-    :format => [:xlsx, :xls, :delimited, :ods, :fixed_width, :html, :xml, :yaml, :csv, :shp],
+    :format => [:xlsx, :xls, :delimited, :ods, :fixed_width, :html, :xml, :yaml, :csv],
   }
   DEFAULT = {
     :streaming => false,
@@ -132,19 +128,16 @@ class RemoteTable
     :headers => :first_row,
     :keep_blank_rows => false,
     :skip => 0,
-    :internal_encoding => 'UTF-8',
+    :encoding => 'UTF-8',
     :delimiter => ','
   }
   OLD_SETTING_NAMES = {
-    :internal_encoding => [:encoding],
-    :transform_settings => [:transform],
     :pre_select => [:select],
     :pre_reject => [:reject],
-    :errata_settings => [:errata],
   }
 
   include ::Enumerable
-
+  
   # The URL of the local or remote file.
   #
   # @example Local
@@ -169,10 +162,6 @@ class RemoteTable
   # @return [Integer]
   attr_reader :download_count
 
-  # @private
-  # Used internally to access the transformer (aka parser).
-  attr_reader :transformer
-
   # @private
   # Used internally to access to a downloaded copy of the file.
   # @return [RemoteTable::LocalCopy]
@@ -185,39 +174,39 @@ class RemoteTable
   # Whether to warn the user on multiple downloads. Defaults to true.
   # @return [true,false]
   attr_reader :warn_on_multiple_downloads
-
+  
   # Headers specified by the user: +:first_row+ (the default), +false+, or a list of headers.
   # @return [:first_row,false,Array<String>]
   attr_reader :headers
-
+    
   # The sheet specified by the user as a number or a string.
   # @return[String,Integer]
   attr_reader :sheet
-
+  
   # Whether to keep blank rows. Default is false.
   # @return [true,false]
   attr_reader :keep_blank_rows
-
+  
   # Form data to POST in the download request. It should probably be in +application/x-www-form-urlencoded+.
   # @return [String]
   attr_reader :form_data
-
+  
   # How many rows to skip at the beginning of the file or table. Default is 0.
   # @return [Integer]
   attr_reader :skip
 
-  # The original encoding of the source file. Default is UTF-8. Previously passed as +:encoding+.
+  # The original encoding of the source file. Default is UTF-8.
   # @return [String]
-  attr_reader :internal_encoding
-
+  attr_reader :encoding
+  
   # The delimiter, a.k.a. column separator. Passed to Ruby CSV as +:col_sep+. Default is :delimited.
   # @return [String]
   attr_reader :delimiter
-
+  
   # The XPath used to find rows in HTML or XML.
   # @return [String]
   attr_reader :row_xpath
-
+  
   # The XPath used to find columns in HTML or XML.
   # @return [String]
   attr_reader :column_xpath
@@ -225,11 +214,11 @@ class RemoteTable
   # The CSS selector used to find rows in HTML or XML.
   # @return [String]
   attr_reader :row_css
-
+  
   # The CSS selector used to find columns in HTML or XML.
   # @return [String]
   attr_reader :column_css
-
+  
   # The format of the source file. Can be +:xlsx+, +:xls+, +:delimited+, +:ods+, +:fixed_width+, +:html+, +:xml+, +:yaml+.
   # @return [Symbol]
   attr_reader :format
@@ -241,7 +230,7 @@ class RemoteTable
   # The packing type. Guessed from URL if not provided. Only +:tar+ is supported.
   # @return [Symbol]
   attr_reader :packing
-
+  
   # The glob used to pick a file out of an archive.
   #
   # @return [String]
@@ -249,7 +238,7 @@ class RemoteTable
   # @example Pick out the only CSV in a ZIP file
   #   RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :glob => '/*.csv'
   attr_reader :glob
-
+  
   # The filename, which can be used to pick a file out of an archive.
   #
   # @return [String]
@@ -267,7 +256,7 @@ class RemoteTable
   #   # ALMOST
   #   RemoteTable.new 'file:///atoz.txt', :cut => '1,12,13,15,19,20'
   attr_reader :cut
-
+  
   # Use a range of rows in a plaintext file.
   #
   # @return [Range]
@@ -278,7 +267,7 @@ class RemoteTable
   #                   :select => proc { |row| CbecsEnergyIntensity::NAICS_CODE_SYNTHESIZER.call(row) },
   #                   :crop => (21..37))
   attr_reader :crop
-
+  
   # The fixed-width schema, given as a multi-dimensional array.
   #
   # @return [Array<Array{String,Integer,Hash}>]
@@ -293,30 +282,29 @@ class RemoteTable
   #                                [  'spacer',  12 ],
   #                                [  'header6', 10, { :type => :string } ]])
   attr_reader :schema
-
+  
   # If you somehow already defined a fixed-width schema (so you can re-use it?), specify it here.
   # @return [String,Symbol]
   attr_reader :schema_name
-
+  
   # A proc that decides whether to include a row. Previously passed as +:select+.
   # @return [Proc]
   attr_reader :pre_select
-
+  
   # A proc that decides whether to include a row. Previously passed as +:reject+.
   # @return [Proc]
   attr_reader :pre_reject
 
-  # Settings to create a transformer.
-  # @return [Hash]
-  attr_reader :transform_settings
-
-  # A hash to initialize an Errata instance to be used on every row. Applied after creating +row_hash+ and before passing to +:synthesize+ procs, etc. Previously passed as +:errata+.
+  # An object that responds to #rejects?(row) and #correct!(row). Applied after creating +row_hash+.
+  #
+  # * #rejects?(row) - if row should be treated like it doesn't exist
+  # * #correct!(row) - destructively update a row to fix something
   #
-  # See the Errata library at https://github.com/seamusabshere/errata
+  # See the Errata library at https://github.com/seamusabshere/errata for an example implementation.
   #
   # @return [Hash]
-  attr_reader :errata_settings
-
+  attr_reader :errata
+  
   # The format of the source file. Can be specified as: :xlsx, :xls, :delimited (aka :csv), :ods, :fixed_width, :html, :xml, :yaml
   #
   # Note: treats all +docs.google.com+ and +spreadsheets.google.com+ URLs as +:delimited+.
@@ -326,6 +314,20 @@ class RemoteTable
   # @return [Hash]
   attr_reader :format
 
+  # @private
+  class NullParser
+    def parse(row)
+      [row]
+    end
+  end
+
+  # An object that responds to #parse(row) and returns an array of one or more rows.
+  #
+  # @return [#parse]
+  def parser
+    @final_parser ||= (@parser || NullParser.new)
+  end
+
   # Options passed by the user that may be passed through to the underlying parsing library.
   # @return [Hash]
   attr_reader :other_options
@@ -354,7 +356,6 @@ class RemoteTable
   def initialize(*args)
     @download_count_mutex = ::Mutex.new
     @extend_bang_mutex = ::Mutex.new
-    @errata_mutex = ::Mutex.new
 
     @cache = []
     @download_count = 0
@@ -387,25 +388,24 @@ class RemoteTable
     @keep_blank_rows = grab settings, :keep_blank_rows
     @form_data = grab settings, :form_data
     @skip = grab settings, :skip
-    @internal_encoding = grab settings, :internal_encoding
+    @encoding = grab settings, :encoding
     @row_xpath = grab settings, :row_xpath
     @column_xpath = grab settings, :column_xpath
     @row_css = grab settings, :row_css
     @column_css = grab settings, :column_css
     @glob = grab settings, :glob
     @filename = grab settings, :filename
-    @transform_settings = grab settings, :transform_settings
     @cut = grab settings, :cut
     @crop = grab settings, :crop
     @schema = grab settings, :schema
     @schema_name = grab settings, :schema_name
     @pre_select = grab settings, :pre_select
     @pre_reject = grab settings, :pre_reject
-    @errata_settings = grab settings, :errata_settings
+    @errata = grab settings, :errata
+    @parser = grab settings, :parser
 
     @other_options = settings
-
-    @transformer = Transformer.new self
+    
     @local_copy = LocalCopy.new self
   end
 
@@ -423,7 +423,7 @@ class RemoteTable
     else
       mark_download!
       memo = _each do |row|
-        transformer.transform(row).each do |virtual_row|
+        parser.parse(row).each do |virtual_row|
           virtual_row.row_hash = ::HashDigest.hexdigest row
           if errata
             next if errata.rejects? virtual_row
@@ -447,7 +447,7 @@ class RemoteTable
 
   # @deprecated
   alias :each_row :each
-
+  
   # @return [Array<Hash,Array>] All rows.
   def to_a
     if fully_cached?
@@ -459,7 +459,7 @@ class RemoteTable
 
   # @deprecated
   alias :rows :to_a
-
+  
   # Get a row by row number. Zero-based.
   #
   # @return [Hash,Array]
@@ -470,33 +470,18 @@ class RemoteTable
       to_a[row_number]
     end
   end
-
+  
   # Clear the row cache in case it helps your GC.
   #
   # @return [nil]
   def free
     @fully_cached = false
-    @errata = nil
     cache.clear
     nil
   end
 
-  # @private
-  def errata
-    @errata || @errata_mutex.synchronize do
-      @errata ||= begin
-        if defined?(::Errata) and errata_settings.is_a?(::Errata)
-          ::Kernel.warn %{[remote_table] Passing :errata_settings as an Errata object is deprecated. Please pass a Hash of settings instead.}
-          errata_settings
-        elsif errata_settings.is_a?(::Hash)
-          ::Errata.new errata_settings
-        end
-      end
-    end
-  end
-
   private
-
+  
   def mark_download!
     @download_count_mutex.synchronize do
       @download_count += 1
@@ -504,12 +489,12 @@ class RemoteTable
     if warn_on_multiple_downloads and download_count > 1
       ::Kernel.warn "[remote_table] #{url} has been downloaded #{download_count} times."
     end
-  end
-
+  end 
+  
   def fully_cached!
     @fully_cached = true
   end
-
+  
   def fully_cached?
     !!@fully_cached
   end

data/lib/remote_table/fixed_width.rb

@@ -9,7 +9,7 @@ class RemoteTable
 
     # @private
     def after_extend
-      @parser_mutex = ::Mutex.new
+      @fixed_width_parser_mutex = ::Mutex.new
       @definition_mutex = ::Mutex.new
     end
 
@@ -23,7 +23,7 @@ class RemoteTable
       skip_rows!
       cut_columns!
 
-      parser.parse[:rows].each do |row|
+      fixed_width_parser.parse[:rows].each do |row|
         some_value_present = false
         hash = ::ActiveSupport::OrderedHash.new
         row.each do |k, v|
@@ -43,9 +43,9 @@ class RemoteTable
 
     private
     
-    def parser
-      @parser || @parser_mutex.synchronize do
-        @parser ||= begin
+    def fixed_width_parser
+      @fixed_width_parser || @fixed_width_parser_mutex.synchronize do
+        @fixed_width_parser ||= begin
           if ::FixedWidth::Section.private_instance_methods.map(&:to_sym).include?(:unpacker)
             raise ::RuntimeError, "[remote_table] You need to use exclusively the fixed_width-multibyte library https://github.com/seamusabshere/fixed_width"
           end