seamusabshere-remote_table 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Seamus Abshere
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,18 @@
+= remote_table
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+   bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Seamus Abshere. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,66 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "remote_table"
+    gem.summary = %Q{Remotely open and parse XLS, ODS, CSV and fixed-width tables.}
+    gem.description = %Q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
+    gem.email = "seamus@abshere.net"
+    gem.homepage = "http://github.com/seamusabshere/remote_table"
+    gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
+    %w{ activesupport roo fastercsv ryanwood-slither }.each { |name| gem.add_dependency name } # TODO: do I need to include activesupport, etc.?
+    gem.require_path = "lib"
+    gem.files.include %w(lib/remote_table/**/*) unless gem.files.empty? # seems to fail once it's in the wild
+    gem.rdoc_options << '--line-numbers' << '--inline-source'
+    gem.requirements << 'curl'
+    # gem.rubyforge_project = "remotetable"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+  Jeweler::RubyforgeTasks.new do |rubyforge|
+    rubyforge.doc_task = "rdoc"
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "remote_table #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/remote_table.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'activesupport'
+require 'activerecord'
+require 'tempfile'
+require 'fastercsv'
+require 'slither' # config.gem 'ryanwood-slither', :lib => 'slither', :source => 'http://gems.github.com'
+require 'roo'
+require 'remote_table/transform'
+require 'remote_table/request'
+require 'remote_table/package'
+require 'remote_table/file'
+require 'remote_table/file/csv'
+require 'remote_table/file/fixed_width'
+require 'remote_table/file/roo_spreadsheet'
+require 'remote_table/file/ods'
+require 'remote_table/file/xls'
+
+class RemoteTable
+  attr_accessor :request, :package, :file, :transform
+  attr_accessor :table
+  
+  def initialize(bus)
+    @transform = Transform.new(bus)
+    @package = Package.new(bus)
+    @request = Request.new(bus)
+    @file = File.new(bus)
+  end
+  
+  def each_row
+    finish_table! unless table
+    table.each_row { |row| yield row }
+  end
+  
+  def rows
+    cache_rows! if @_row_cache.nil?
+    @_row_cache
+  end
+  
+  private
+  
+  def finish_table!
+    package_path = request.download
+    file_path = package.stage(package_path)
+    raw_table = file.tabulate(file_path)
+    self.table = transform.apply(raw_table) # must return something that responds to each_row
+  end
+  
+  def cache_rows!
+    @_row_cache = []
+    each_row { |row| @_row_cache << row }
+  end
+end

data/lib/remote_table/file.rb

@@ -0,0 +1,56 @@
+class RemoteTable
+  class File
+    attr_accessor :filename, :format, :delimiter, :skip, :cut, :crop, :sheet, :headers, :schema, :schema_name, :trap
+    attr_accessor :path
+    
+    def initialize(bus)
+      @filename = bus[:filename]
+      @format = bus[:format] || format_from_filename
+      @delimiter = bus[:delimiter]
+      @sheet = bus[:sheet] || 0
+      @skip = bus[:skip] # rows
+      @crop = bus[:crop] # rows
+      @cut = bus[:cut]   # columns
+      @headers = bus[:headers]
+      @schema = bus[:schema]
+      @schema_name = bus[:schema_name]
+      @trap = bus[:trap]
+      extend "RemoteTable::#{format.to_s.camelcase}".constantize
+    end
+    
+    def tabulate(path)
+      define_fixed_width_schema! if format == :fixed_width and schema.is_a?(Array) # TODO move to generic subclass callback
+      self.path = path
+      self
+    end
+    
+    private
+    
+    # doesn't support trap or spacer
+    def define_fixed_width_schema!
+      raise "can't define both schema_name and schema" if !schema_name.blank?
+      self.schema_name = "autogenerated_#{filename.gsub(/[^a-z0-9_]/i, '')}".to_sym
+      self.trap ||= lambda { true }
+      Slither.define schema_name do |d|
+        d.rows do |row|
+          row.trap(&trap)
+          schema.each do |name, width, options|
+            if name == :spacer
+              row.spacer width
+            else
+              row.column name, width, options
+            end
+          end
+        end
+      end
+    end
+    
+    def format_from_filename
+      extname = ::File.extname(filename).gsub('.', '')
+      return :csv if extname.blank?
+      format = [ :xls, :ods ].detect { |i| i == extname.to_sym }
+      format = :csv if format.blank?
+      format
+    end
+  end
+end

data/lib/remote_table/file/csv.rb

@@ -0,0 +1,45 @@
+class RemoteTable
+  module Csv
+    def each_row(&block)
+      skip_rows!
+      FasterCSV.parse(open(path), fastercsv_options) do |row|
+        if row.respond_to?(:fields) # it's a traditional fastercsv row hash
+          next if row.fields.compact.blank?
+          hash = HashWithIndifferentAccess.new(row.to_hash)
+        else                        # it's an array, which i think happens if you're using :headers => nil or :col_sep
+          next if row.compact.blank?
+          index = 0
+          hash = row.inject(ActiveSupport::OrderedHash.new) { |memo, element| memo[index] = element; index += 1; memo }
+        end
+        yield hash
+      end
+    ensure
+      restore_rows!
+    end
+    
+    private
+    
+    def fastercsv_options
+      fastercsv_options = { :skip_blanks => true }              # ...and this will skip []
+      if headers == false
+        fastercsv_options.merge!(:headers => nil)
+      else
+        fastercsv_options.merge!(:headers => :first_row)
+      end
+      fastercsv_options.merge!(:col_sep => delimiter) if delimiter
+      fastercsv_options
+    end
+    
+    def skip_rows!
+      return unless skip
+      original = "#{path}.original"
+      FileUtils.cp(path, original)
+      `cat #{original} | tail -n +#{skip + 1} > #{path}`
+    end
+    
+    def restore_rows!
+      return unless skip
+      FileUtils.mv "#{path}.original", path
+    end
+  end
+end

data/lib/remote_table/file/fixed_width.rb

@@ -0,0 +1,53 @@
+class RemoteTable
+  module FixedWidth
+    def each_row(&block)
+      crop_rows!
+      skip_rows!
+      cut_columns!
+      a = Slither.parse(path, schema_name)
+      a[:rows].each { |row| yield HashWithIndifferentAccess.new(row) }
+    ensure
+      uncut_columns!
+      unskip_rows!
+      uncrop_rows!
+    end
+    
+    private
+    
+    def cut_columns!
+      return unless cut
+      original = "#{path}.uncut"
+      FileUtils.cp(path, original)
+      `cat #{original} | cut -c #{cut} > #{path}`
+    end
+    
+    def uncut_columns!
+      return unless cut
+      FileUtils.mv "#{path}.uncut", path
+    end
+    
+    def skip_rows!
+      return unless skip
+      original = "#{path}.unskipped"
+      FileUtils.cp(path, original)
+      `cat #{original} | tail -n +#{skip + 1} > #{path}`
+    end
+    
+    def unskip_rows!
+      return unless skip
+      FileUtils.mv "#{path}.unskipped", path
+    end
+    
+    def crop_rows!
+      return unless crop
+      original = "#{path}.uncropped"
+      FileUtils.cp(path, original)
+      `cat #{original} | tail -n +#{crop.first} | head -n #{crop.last - crop.first + 1} > #{path}`
+    end
+    
+    def uncrop_rows!
+      return unless crop
+      FileUtils.mv "#{path}.uncropped", path
+    end
+  end
+end

data/lib/remote_table/file/ods.rb

@@ -0,0 +1,11 @@
+class RemoteTable
+  module Ods
+    def self.extended(base)
+      base.send :extend, RooSpreadsheet
+    end
+    
+    def roo_klass
+      Openoffice
+    end
+  end
+end

data/lib/remote_table/file/roo_spreadsheet.rb

@@ -0,0 +1,30 @@
+class RemoteTable
+  module RooSpreadsheet
+    def each_row(&block)
+      headers = {}
+      oo = roo_klass.new(path, nil, :ignore)
+      oo.default_sheet = sheet.is_a?(Numeric) ? oo.sheets[sheet] : sheet
+      for col in (1..oo.last_column)
+        headers[col] = oo.cell(header_row, col)
+        headers[col] = oo.cell(header_row - 1, col) if headers[col].blank? # look up
+      end
+      first_data_row.upto(oo.last_row) do |row|
+        values = {}
+        for col in (1..oo.last_column)
+          values[headers[col]] = oo.cell(row, col).to_s.gsub(/<[^>]+>/, '').strip
+        end
+        yield HashWithIndifferentAccess.new(values)
+      end
+    end
+
+    private
+
+    def header_row
+      1 + skip.to_i
+    end
+
+    def first_data_row
+      1 + header_row
+    end
+  end
+end

data/lib/remote_table/file/xls.rb

@@ -0,0 +1,11 @@
+class RemoteTable
+  module Xls
+    def self.extended(base)
+      base.send :extend, RooSpreadsheet
+    end
+    
+    def roo_klass
+      Excel
+    end
+  end
+end

data/lib/remote_table/package.rb

@@ -0,0 +1,84 @@
+class RemoteTable
+  class Package
+    attr_accessor :url, :compression, :packing, :filename
+    
+    def initialize(bus)
+      @url = bus[:url] or raise "need url"
+      @compression = bus[:compression] || compression_from_basename
+      @packing = bus[:packing] || packing_from_basename_and_compression
+      @filename = bus[:filename] || filename_from_basename_and_compression_and_packing
+      add_hints!(bus)
+    end
+    
+    def add_hints!(hash)
+      hash[:filename] = filename unless hash.has_key?(:filename)
+    end
+        
+    def stage(path)
+      decompress(path)
+      unpack(path)
+      identify(path)
+      file_path(path)
+    end
+    
+    private
+
+    def decompress(path)
+      return unless compression
+      cmd, args = case compression
+      when :zip, :exe
+        ["unzip", "-d #{::File.dirname(path)}"]
+      when :bz2
+        'bunzip2'
+      when :gz
+        'gunzip'
+      end
+      move_and_process path, compression, cmd, args
+    end
+    
+    def unpack(path)
+      return unless packing
+      cmd, args = case packing
+      when :tar
+        ['tar -xf', "-C #{::File.dirname(path)}"]
+      end
+      move_and_process path, packing, cmd, args
+    end
+    
+    def move_and_process(path, extname, cmd, args)
+      `mv #{path} #{path}.#{extname} && #{cmd} #{path}.#{extname} #{args}`
+    end
+
+    # ex. A: 2007-01.csv.gz  (compression not capable of storing multiple files)
+    # ex. B: 2007-01.tar.gz  (packing)
+    # ex. C: 2007-01.zip     (compression capable of storing multiple files)    
+    # in C but not in the others, we can default to the basename of the package
+    # in order to do this we'll need to mv the uncompressed file on top of the original file
+    def identify(path)
+      ::File.mv(path, file_path(path)) if !packing and [ nil, :bz2, :gz ].include?(compression)
+    end
+    
+    def file_path(path)
+      ::File.join(::File.dirname(path), filename)
+    end
+
+    def basename_parts
+      ::File.basename(URI.parse(url).path).split('.').map(&:to_sym)
+    end
+    
+    def compression_from_basename
+      [ :zip, :exe, :bz2, :gz ].detect { |i| i == basename_parts.last }
+    end
+
+    def packing_from_basename_and_compression
+      [ :tar ].detect { |i| i == ((basename_parts.last == compression) ? basename_parts[-2] : basename_parts.last) }
+    end
+    
+    def filename_from_basename_and_compression_and_packing
+      ary = basename_parts
+      ary.pop if ary.last == compression
+      ary.pop if ary.last == packing
+      ary.join('.')
+    end
+  end
+end

data/lib/remote_table/request.rb

@@ -0,0 +1,36 @@
+class RemoteTable
+  class Request
+    attr_accessor :url, :post_data, :username, :password
+    
+    # TODO: support post_data
+    # TODO: support HTTP basic auth
+    def initialize(bus)
+      @url = bus[:url] or raise "need url"
+    end
+    
+    def download
+      path = ::File.join(staging_dir_path, 'REMOTE_TABLE_PACKAGE')
+      `curl --silent \"#{url_with_google_docs_handling}\" > #{path}`
+      path
+    end
+    
+    private
+    
+    def staging_dir_path
+      path = Tempfile.open(url.gsub(/[^a-z0-9]+/i, '_')).path
+      FileUtils.rm_f(path)
+      FileUtils.mkdir(path)
+      at_exit { FileUtils.rm_rf(path) }
+      path
+    end
+    
+    def url_with_google_docs_handling
+      url = self.url
+      if url.include?('spreadsheets.google.com')
+        url = url.gsub(/\&output=.*(\&|\z)/, '')
+        url << "&output=csv"
+      end
+      url
+    end
+  end
+end

data/lib/remote_table/transform.rb

@@ -0,0 +1,32 @@
+class RemoteTable
+  class Transform
+    attr_accessor :select, :reject, :transform_class, :transform_options, :transform, :raw_table
+    
+    def initialize(bus)
+      if transform_params = bus.delete(:transform)
+        @transform_class = transform_params.delete(:class)
+        @transform_options = transform_params
+        @transform = @transform_class.new(@transform_options)
+        @transform.add_hints!(bus)
+      end
+      @select = bus[:select]
+      @reject = bus[:reject]
+    end
+    
+    def apply(raw_table)
+      self.raw_table = raw_table
+      self
+    end
+    
+    def each_row(&block)
+      raw_table.each_row do |row|
+        virtual_rows = transform ? transform.apply(row) : row # allow transform.apply(row) to return multiple rows
+        Array.wrap(virtual_rows).each do |virtual_row|
+          next if select and !select.call(virtual_row)
+          next if reject and reject.call(virtual_row)
+          yield virtual_row
+        end
+      end
+    end
+  end
+end

data/remote_table.gemspec

@@ -0,0 +1,81 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{remote_table}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
+  s.date = %q{2009-08-18}
+  s.description = %q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
+  s.email = %q{seamus@abshere.net}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/remote_table.rb",
+     "lib/remote_table/file.rb",
+     "lib/remote_table/file.rb",
+     "lib/remote_table/file/csv.rb",
+     "lib/remote_table/file/csv.rb",
+     "lib/remote_table/file/fixed_width.rb",
+     "lib/remote_table/file/fixed_width.rb",
+     "lib/remote_table/file/ods.rb",
+     "lib/remote_table/file/ods.rb",
+     "lib/remote_table/file/roo_spreadsheet.rb",
+     "lib/remote_table/file/roo_spreadsheet.rb",
+     "lib/remote_table/file/xls.rb",
+     "lib/remote_table/file/xls.rb",
+     "lib/remote_table/package.rb",
+     "lib/remote_table/package.rb",
+     "lib/remote_table/request.rb",
+     "lib/remote_table/request.rb",
+     "lib/remote_table/transform.rb",
+     "lib/remote_table/transform.rb",
+     "remote_table.gemspec",
+     "test/remote_table_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/seamusabshere/remote_table}
+  s.rdoc_options = ["--charset=UTF-8", "--line-numbers", "--inline-source"]
+  s.require_paths = ["lib"]
+  s.requirements = ["curl"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Remotely open and parse XLS, ODS, CSV and fixed-width tables.}
+  s.test_files = [
+    "test/remote_table_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<roo>, [">= 0"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+      s.add_runtime_dependency(%q<ryanwood-slither>, [">= 0"])
+    else
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<roo>, [">= 0"])
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+      s.add_dependency(%q<ryanwood-slither>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<roo>, [">= 0"])
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+    s.add_dependency(%q<ryanwood-slither>, [">= 0"])
+  end
+end

data/test/remote_table_test.rb

@@ -0,0 +1,101 @@
+require 'test_helper'
+
+class FuelOilParser
+  def initialize(options = {})
+    # nothing
+  end
+  def add_hints!(bus)
+    bus[:sheet] = 'Data 1'
+    bus[:skip] = 2
+    bus[:select] = lambda { |row| row[:year] > 1989 }
+  end
+  def apply(row)
+    virtual_rows = []
+    row.keys.grep(/(.*) Residual Fuel Oil/) do |location_column_name|
+      next if (cost = row[location_column_name]).blank? or (date = row['Date']).blank?
+      if $1.starts_with?('U.S.')
+        locatable = "united_states (Country)"
+      elsif $1.include?('PADD')
+        /\(PADD (.*)\)/.match($1)
+        next if $1 == '1' # skip PADD 1 because we always prefer subdistricts
+        locatable = "#{$1} (PetroleumAdministrationForDefenseDistrict)"
+      else
+        locatable = "#{$1} (State)"
+      end
+      date = Time.parse(date)
+      virtual_rows << HashWithIndifferentAccess.new(
+        :locatable => locatable,
+        :cost => cost,
+        :year => date.year,
+        :month => date.month
+      )
+    end
+    virtual_rows
+  end
+end
+
+class RemoteTableTest < Test::Unit::TestCase
+  should "open an XLS inside a zip file" do
+    t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls')
+    assert_equal 'ACURA',      t.rows.first['Manufacturer']
+    assert_equal 'NSX',        t.rows.first['carline name']
+    assert_equal 'VOLVO',      t.rows.last['Manufacturer']
+    assert_equal 'V70 XC AWD', t.rows.last['carline name']
+  end
+  
+  should "have indifferent hash access" do
+    t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls')
+    assert_equal 'ACURA',      t.rows.first['Manufacturer'.to_sym]
+    assert_equal 'NSX',        t.rows.first['carline name'.to_sym]
+    assert_equal 'VOLVO',      t.rows.last['Manufacturer'.to_sym]
+    assert_equal 'V70 XC AWD', t.rows.last['carline name'.to_sym]
+  end
+  
+  should "open a Google Docs url" do
+    t = RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA')
+    assert_equal 'Gulf Coast',     t.rows.first['PAD district name']
+    assert_equal 'AL',             t.rows.first['State']
+    assert_equal 'Rocky Mountain', t.rows.last['PAD district name']
+    assert_equal 'WY',             t.rows.last['State']
+  end
+  
+  should "open an ODS" do
+    t = RemoteTable.new(:url => 'http://static.brighterplanet.com/science/profiler/footprint_model.ods', :sheet => 'Export')
+    assert_equal 'automobiles', t.rows.first['component']
+    assert_equal 2005.0,        t.rows.first['period'].to_f
+  end
+    
+  should "open a CSV inside a zip file" do
+    t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
+    assert_equal 'ACURA',             t.rows.first['Manufacturer']
+    assert_equal 'NSX',               t.rows.first['carline name']
+    assert_equal 'TOYOTA',            t.rows.last['Manufacturer']
+    assert_equal 'RAV4 SOFT TOP 4WD', t.rows.last['carline name']
+  end
+  
+  should "open a fixed-width file with an inline schema inside a zip file" do
+    t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
+                        :filename => 'Gd6-dsc.txt',
+                        :format => :fixed_width,
+                        :crop => 21..26, # inclusive
+                        :cut => '2-',
+                        :select => lambda { |row| /\A[A-Z]/.match row[:code] },
+                        :schema => [[ :code,   2, { :type => :string }  ],
+                                    [ :spacer, 2 ],
+                                    [ :name,   52, { :type => :string } ]])
+    assert_equal 'regular grade gasoline (octane number of 87)', t.rows.first['name']
+    assert_equal 'R',                                            t.rows.first['code']
+    assert_equal 'electricity',                                  t.rows.last['name']
+    assert_equal 'El',                                           t.rows.last['code']
+  end
+  
+  should "open an XLS with a parser" do
+    ma_1990_01 = {"month"=>1, "cost"=>"54.0", "locatable"=>"Massachusetts (State)", "year"=>1990}
+    ga_1990_01 = {"month"=>1, "cost"=>"50.7", "locatable"=>"Georgia (State)", "year"=>1990}
+
+    t = RemoteTable.new(:url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls',
+                        :transform => { :class => FuelOilParser })
+    assert_equal ma_1990_01, t.rows[0]
+    assert_equal ga_1990_01, t.rows[1]
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'remote_table'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification 
+name: seamusabshere-remote_table
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Seamus Abshere
+- Andy Rossmeissl
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-18 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: roo
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ryanwood-slither
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.
+email: seamus@abshere.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/remote_table.rb
+- lib/remote_table/file.rb
+- lib/remote_table/file/csv.rb
+- lib/remote_table/file/fixed_width.rb
+- lib/remote_table/file/ods.rb
+- lib/remote_table/file/roo_spreadsheet.rb
+- lib/remote_table/file/xls.rb
+- lib/remote_table/package.rb
+- lib/remote_table/request.rb
+- lib/remote_table/transform.rb
+- remote_table.gemspec
+- test/remote_table_test.rb
+- test/test_helper.rb
+has_rdoc: false
+homepage: http://github.com/seamusabshere/remote_table
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+- --line-numbers
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: 
+- curl
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Remotely open and parse XLS, ODS, CSV and fixed-width tables.
+test_files: 
+- test/remote_table_test.rb
+- test/test_helper.rb