remote_table 1.3.0 → 1.4.0

data/Rakefile

@@ -1,25 +1,17 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
 
 require 'rake'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
+  test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
 end
 
-begin
-  require 'rake/rdoctask'
-  Rake::RDocTask.new do |rdoc|
-    rdoc.rdoc_dir = 'rdoc'
-    rdoc.title = 'remote_table'
-    rdoc.options << '--line-numbers' << '--inline-source'
-    rdoc.rdoc_files.include('README*')
-    rdoc.rdoc_files.include('lib/**/*.rb')
-  end
-rescue LoadError
-   puts "Rdoc is not available"
+require 'yard'
+YARD::Rake::YardocTask.new do |y|
+  y.options << '--no-private'
 end
 
 task :default => :test

data/lib/remote_table.rb

@@ -1,20 +1,20 @@
 if ::RUBY_VERSION < '1.9' and $KCODE != 'UTF8'
-  $stderr.puts "[remote_table] Ruby 1.8 detected, setting $KCODE to UTF8 so that ActiveSupport::Multibyte works properly."
+  ::Kernel.warn "[remote_table] Ruby 1.8 detected, setting $KCODE to UTF8 so that ActiveSupport::Multibyte works properly."
   $KCODE = 'UTF8'
 end
 
 require 'active_support'
 require 'active_support/version'
-%w{
-  active_support/core_ext/hash
-  active_support/core_ext/string
-  active_support/core_ext/module
-  active_support/core_ext/array
-}.each do |active_support_3_requirement|
-  require active_support_3_requirement
-end if ::ActiveSupport::VERSION::MAJOR == 3
+if ::ActiveSupport::VERSION::MAJOR >= 3
+  require 'active_support/core_ext'
+end
 require 'hash_digest'
 
+require 'remote_table/format'
+require 'remote_table/config'
+require 'remote_table/local_file'
+require 'remote_table/transformer'
+
 class Hash
   attr_accessor :row_hash
 end
@@ -23,13 +23,7 @@ class Array
   attr_accessor :row_hash
 end
 
-class RemoteTable
-  autoload :Format, 'remote_table/format'
-  autoload :Properties, 'remote_table/properties'
-  autoload :LocalFile, 'remote_table/local_file'
-  autoload :Transformer, 'remote_table/transformer'
-  autoload :Utils, 'remote_table/utils'
-  
+class RemoteTable  
   # Legacy
   class Transform
     def self.row_hash(row)
@@ -40,7 +34,7 @@ class RemoteTable
   include ::Enumerable
   
   attr_reader :url
-  attr_reader :options
+  attr_reader :config
   
   # Create a new RemoteTable.
   #
@@ -51,16 +45,16 @@ class RemoteTable
   # Old syntax:
   #     RemoteTable.new(:url => 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :foo => 'bar')
   #
-  # See the <tt>Properties</tt> object for the sorts of options you can pass.
+  # See the <tt>Config</tt> object for the sorts of options you can pass.
   def initialize(*args)
-    @options = args.last.is_a?(::Hash) ? args.last.symbolize_keys : {}
+    options = args.last.is_a?(::Hash) ? args.last.symbolize_keys : {}
+    
     @url = if args.first.is_a? ::String
       args.first.dup
     else
-      @options[:url].dup
+      options[:url].dup
     end
-    @url.freeze
-    @options.freeze
+    @config = Config.new self, options
   end
   
   # not thread safe
@@ -72,17 +66,17 @@ class RemoteTable
       retval = format.each do |row|
         transformer.transform(row).each do |virtual_row|
           virtual_row.row_hash = ::HashDigest.hexdigest row
-          if properties.errata
-            next if properties.errata.rejects? virtual_row
-            properties.errata.correct! virtual_row
+          if config.errata
+            next if config.errata.rejects? virtual_row
+            config.errata.correct! virtual_row
           end
-          next if properties.select and !properties.select.call(virtual_row)
-          next if properties.reject and properties.reject.call(virtual_row)
-          cache.push virtual_row unless properties.streaming
+          next if config.select and !config.select.call(virtual_row)
+          next if config.reject and config.reject.call(virtual_row)
+          cache.push virtual_row unless config.streaming
           yield virtual_row
         end
       end
-      fully_cached! unless properties.streaming
+      fully_cached! unless config.streaming
       retval
     end
   end
@@ -117,17 +111,12 @@ class RemoteTable
     @local_file ||= LocalFile.new self
   end
   
-  # Used internally to access to the properties of the table, either set by the user or implied
-  def properties
-    @properties ||= Properties.new self
-  end
-  
   # Used internally to access to the driver that reads the format
   def format
-    @format ||= properties.format.new self
+    @format ||= config.format.new self
   end
   
-  # Used internally to acess the transformer (aka parser).
+  # Used internally to access the transformer (aka parser).
   def transformer
     @transformer ||= Transformer.new self
   end
@@ -139,8 +128,8 @@ class RemoteTable
   def mark_download!
     @download_count ||= 0
     @download_count += 1
-    if properties.warn_on_multiple_downloads and download_count > 1
-      $stderr.puts "[remote_table] Warning: #{url} has been downloaded #{download_count} times."
+    if config.warn_on_multiple_downloads and download_count > 1
+      ::Kernel.warn "[remote_table] #{url} has been downloaded #{download_count} times."
     end
   end 
   

data/lib/remote_table/{properties.rb → config.rb}

@@ -1,19 +1,15 @@
 require 'uri'
 class RemoteTable
-  # Represents the properties of a RemoteTable, whether they are explicitly set by the user or inferred automatically.
-  class Properties
+  # Represents the config of a RemoteTable, whether they are explicitly set by the user or inferred automatically.
+  class Config
     attr_reader :t
-    attr_reader :current_options
+    attr_reader :user_specified_options
     
-    def initialize(t)
+    def initialize(t, user_specified_options)
       @t = t
-      @current_options = t.options.symbolize_keys
+      @user_specified_options = user_specified_options
     end
-    
-    def update(options)
-      current_options.update options
-    end
-        
+            
     # The parsed URI of the file to get.
     def uri
       return @uri if @uri.is_a?(::URI)
@@ -29,19 +25,19 @@ class RemoteTable
     # * call each
     # Defaults to false.
     def streaming
-      current_options[:streaming] || false
+      user_specified_options.fetch :streaming, false
     end
 
     # Defaults to true.
     def warn_on_multiple_downloads
-      current_options[:warn_on_multiple_downloads] != false
+      user_specified_options[:warn_on_multiple_downloads] != false
     end
     
     # The headers specified by the user
     #
     # Default: :first_row
     def headers
-      current_options[:headers].nil? ? :first_row : current_options[:headers]
+      user_specified_options[:headers].nil? ? :first_row : user_specified_options[:headers]
     end
     
     def use_first_row_as_header?
@@ -53,33 +49,31 @@ class RemoteTable
     end
     
     # The sheet specified by the user as a number or a string
-    #
-    # Default: 0
     def sheet
-      current_options[:sheet] || 0
+      user_specified_options[:sheet]
     end
     
     # Whether to keep blank rows
     #
     # Default: false
     def keep_blank_rows
-      current_options[:keep_blank_rows] || false
+      user_specified_options.fetch :keep_blank_rows, false
     end
     
     # Form data to send in with the download request
     def form_data
-      current_options[:form_data]
+      user_specified_options[:form_data]
     end
     
     # How many rows to skip
     #
     # Default: 0
     def skip
-      current_options[:skip] || 0
+      user_specified_options.fetch :skip, 0
     end
     
     def internal_encoding
-      (current_options[:encoding] || 'UTF-8').upcase
+      user_specified_options.fetch(:encoding, 'UTF-8').upcase
     end
     
     def external_encoding
@@ -94,27 +88,27 @@ class RemoteTable
     #
     # Default: ","
     def delimiter
-      current_options[:delimiter] || ','
+      user_specified_options.fetch :delimiter, ','
     end
     
     # The XPath used to find rows
     def row_xpath
-      current_options[:row_xpath]
+      user_specified_options[:row_xpath]
     end
     
     # The XPath used to find columns
     def column_xpath
-      current_options[:column_xpath]
+      user_specified_options[:column_xpath]
     end
 
     # The CSS selector used to find rows
     def row_css
-      current_options[:row_css]
+      user_specified_options[:row_css]
     end
     
     # The CSS selector used to find columns
     def column_css
-      current_options[:column_css]
+      user_specified_options[:column_css]
     end
     
     # The compression type.
@@ -123,8 +117,8 @@ class RemoteTable
     #
     # Can be specified as: :gz, :zip, :bz2, :exe (treated as :zip)
     def compression
-      if current_options.has_key?(:compression)
-        return current_options[:compression]
+      if user_specified_options.has_key?(:compression)
+        return user_specified_options[:compression]
       end
       case ::File.extname(uri.path).downcase
       when /gz/, /gunzip/
@@ -144,8 +138,8 @@ class RemoteTable
     #
     # Can be specified as: :tar
     def packing
-      if current_options.has_key?(:packing)
-        return current_options[:packing]
+      if user_specified_options.has_key?(:packing)
+        return user_specified_options[:packing]
       end
       if uri.path =~ %r{\.tar(?:\.|$)}i
         :tar
@@ -157,7 +151,7 @@ class RemoteTable
     # Example:
     #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :glob => '/*.csv'
     def glob
-      current_options[:glob]
+      user_specified_options[:glob]
     end
     
     # The filename, which can be used to pick a file out of an archive.
@@ -165,17 +159,17 @@ class RemoteTable
     # Example:
     #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv'
     def filename
-      current_options[:filename]
+      user_specified_options[:filename]
     end
     
     # Cut columns up to this character
     def cut
-      current_options[:cut]
+      user_specified_options[:cut]
     end
     
     # Crop rows after this line
     def crop
-      current_options[:crop]
+      user_specified_options[:crop]
     end
     
     # The fixed-width schema, given as an array
@@ -190,31 +184,31 @@ class RemoteTable
     #                                  [  'spacer',  12 ],
     #                                  [  'header6', 10, { :type => :string } ]])
     def schema
-      current_options[:schema]
+      user_specified_options[:schema]
     end
     
     # The name of the fixed-width schema according to FixedWidth
     def schema_name
-      current_options[:schema_name]
+      user_specified_options[:schema_name]
     end
     
     # A proc to call to decide whether to return a row.
     def select
-      current_options[:select]
+      user_specified_options[:select]
     end
     
     # A proc to call to decide whether to return a row.
     def reject
-      current_options[:reject]
+      user_specified_options[:reject]
     end
     
     # A hash of options to create a new Errata instance (see the Errata gem at http://github.com/seamusabshere/errata) to be used on every row.
     def errata
-      return unless current_options.has_key? :errata
-      @errata ||= if current_options[:errata].is_a? ::Hash
-        ::Errata.new current_options[:errata]
+      return unless user_specified_options.has_key? :errata
+      @errata ||= if user_specified_options[:errata].is_a? ::Hash
+        ::Errata.new user_specified_options[:errata]
       else
-        current_options[:errata]
+        user_specified_options[:errata]
       end
     end
     
@@ -227,8 +221,8 @@ class RemoteTable
     # Can be specified as: :xlsx, :xls, :delimited (aka :csv and :tsv), :ods, :fixed_width, :html
     def format
       return Format::Delimited if uri.host == 'spreadsheets.google.com' or @uri.host == 'docs.google.com'
-      clue = if current_options.has_key?(:format)
-        current_options[:format]
+      clue = if user_specified_options.has_key?(:format)
+        user_specified_options[:format]
       else
         t.local_file.path
       end
@@ -247,6 +241,8 @@ class RemoteTable
         Format::HTML
       when /xml/
         Format::XML
+      when /yaml/, /yml/
+        Format::Yaml
       else
         Format::Delimited
       end

data/lib/remote_table/format.rb

@@ -1,22 +1,22 @@
-if ::RUBY_VERSION >= '1.9'
-  require 'ensure/encoding'
-else
-  require 'iconv'
+require 'iconv'
+if RUBY_VERSION >= '1.9'
+  # for an excellent explanation see http://blog.segment7.net/2010/12/17/from-iconv-iconv-to-string-encode
+  Kernel.warn "[remote_table] Apologies - using iconv because Ruby 1.9.x's String#encode doesn't have transliteration tables (yet)"
 end
 
+require 'remote_table/format/mixins/textual'
+require 'remote_table/format/mixins/processed_by_roo'
+require 'remote_table/format/mixins/processed_by_nokogiri'
+require 'remote_table/format/excel'
+require 'remote_table/format/excelx'
+require 'remote_table/format/delimited'
+require 'remote_table/format/open_office'
+require 'remote_table/format/fixed_width'
+require 'remote_table/format/html'
+require 'remote_table/format/xml'
+require 'remote_table/format/yaml'
 class RemoteTable  
   class Format
-    autoload :Excel, 'remote_table/format/excel'
-    autoload :Excelx, 'remote_table/format/excelx'
-    autoload :Delimited, 'remote_table/format/delimited'
-    autoload :OpenOffice, 'remote_table/format/open_office'
-    autoload :FixedWidth, 'remote_table/format/fixed_width'
-    autoload :HTML, 'remote_table/format/html'
-    autoload :XML, 'remote_table/format/xml'
-    
-    autoload :Textual, 'remote_table/format/mixins/textual'
-    autoload :ProcessedByRoo, 'remote_table/format/mixins/processed_by_roo'
-    autoload :ProcessedByNokogiri, 'remote_table/format/mixins/processed_by_nokogiri'
     
     attr_reader :t
 
@@ -25,28 +25,25 @@ class RemoteTable
     end
     
     def transliterate_to_utf8(str)
-      return if str.nil?
-      $stderr.puts "[remote_table translit] Before:  #{str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
-      transliterated_str = if ::RUBY_VERSION >= '1.9'
-        str.ensure_encoding t.properties.external_encoding, :external_encoding => t.properties.internal_encoding, :invalid_characters => :transcode
-      else
-        ::Iconv.conv(t.properties.external_encoding_iconv, t.properties.internal_encoding, str.to_s + ' ')[0..-2]
+      if str.is_a?(::String)
+        [ iconv.iconv(str), iconv.iconv(nil) ].join
       end
-      $stderr.puts "[remote_table translit] After:   #{transliterated_str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
-      transliterated_str
     end
 
     def assume_utf8(str)
       if str.is_a?(::String) and ::RUBY_VERSION >= '1.9'
-        str.encode! t.properties.external_encoding
+        str.encode! t.config.external_encoding
       else
         str
       end
     end
     
-    include ::Enumerable
-    def each
-      raise "must be defined by format"
+    private
+    
+    def iconv
+      @iconv ||= ::Iconv.new(t.config.external_encoding_iconv, t.config.internal_encoding)
     end
+    
+    include ::Enumerable
   end
 end