remote_table 0.2.32 → 1.0.0

data/lib/remote_table/local_file.rb

@@ -0,0 +1,92 @@
+require 'fileutils'
+require 'escape'
+require 'tmpdir'
+class RemoteTable
+  class LocalFile
+    
+    attr_reader :t
+    
+    def initialize(t)
+      @t = t
+    end
+    
+    def path
+      save_locally
+      @path
+    end
+    
+    private
+    
+    def save_locally
+      return if @path.is_a?(::String)
+      @path = ::File.join(staging_dir_path, ::File.basename(t.properties.uri.path))
+      download
+      decompress
+      unpack
+      pick
+      @path
+    end
+    
+    def download
+      if t.properties.uri.scheme == 'file'
+        ::FileUtils.cp t.properties.uri.path, @path
+      else
+        # sabshere 1/20/11 FIXME: ::RemoteTable.config.curl_bin_path or smth
+        ::RemoteTable.executor.backtick_with_reporting %{
+          curl
+          --header "Expect: "
+          --location
+          #{"--data #{::Escape.shell_single_word t.properties.form_data}" if t.properties.form_data.present?}
+          --output #{::Escape.shell_single_word @path}
+          #{::Escape.shell_single_word t.properties.uri.to_s}
+          2>&1
+        }
+      end
+    end
+    
+    def decompress
+      return unless t.properties.compression
+      new_path = @path.chomp ".#{t.properties.compression}"
+      cmd = case t.properties.compression
+      when 'zip', 'exe'
+        "unzip #{::Escape.shell_single_word @path} -d #{::File.dirname(@path)}"
+        # can't set path yet because there may be multiple files
+      when 'bz2'
+        "bunzip2 --stdout #{::Escape.shell_single_word @path} > #{::Escape.shell_single_word new_path}"
+        @path = new_path
+      when 'gz'
+        "gunzip --stdout #{::Escape.shell_single_word @path} > #{::Escape.shell_single_word new_path}"
+        @path = new_path
+      end
+      ::RemoteTable.executor.backtick_with_reporting cmd
+    end
+    
+    def unpack
+      return unless t.properties.packing
+      cmd = case t.properties.packing
+      when 'tar'
+        "tar -xf #{::Escape.shell_single_word @path} -C #{::File.dirname(@path)}"
+      end
+      ::RemoteTable.executor.backtick_with_reporting cmd
+    end
+    
+    # ex. A: 2007-01.csv.gz  (compression not capable of storing multiple files)
+    # ex. B: 2007-01.tar.gz  (packing)
+    # ex. C: 2007-01.zip     (compression capable of storing multiple files)
+    def pick
+      if t.properties.filename.present?
+        @path = ::File.join ::File.dirname(@path), t.properties.filename
+      elsif t.properties.glob.present?
+        @path = ::Dir[::File.dirname(@path)+t.properties.glob].first
+      end
+    end
+    
+    def staging_dir_path    
+      return @staging_dir_path if @staging_dir_path.is_a?(::String)
+      @staging_dir_path = ::File.join ::Dir.tmpdir, 'remote_table_gem', rand.to_s
+      ::FileUtils.mkdir_p @staging_dir_path
+      ::RemoteTable.cleaner.remove_at_exit @staging_dir_path
+      @staging_dir_path
+    end
+  end
+end

data/lib/remote_table/properties.rb

@@ -0,0 +1,209 @@
+require 'uri'
+class RemoteTable
+  # Represents the properties of a RemoteTable, whether they are explicitly set by the user or inferred automatically.
+  class Properties
+    attr_reader :t
+    def initialize(t)
+      @t = t
+    end
+    
+    # The parsed URI of the file to get.
+    def uri
+      return @uri if @uri.is_a?(::URI)
+      @uri = ::URI.parse t.url
+      if @uri.host == 'spreadsheets.google.com'
+        @uri.query = 'output=csv&' + @uri.query.sub(/\&?output=.*?(\&|\z)/, '\1')
+      end
+      @uri
+    end
+    
+    # The headers specified by the user
+    def headers
+      t.options['headers']
+    end
+    
+    # The sheet specified by the user as a number or a string
+    #
+    # Default: 0
+    def sheet
+      t.options['sheet'] || 0
+    end
+    
+    # Whether to keep blank rows
+    #
+    # Default: false
+    def keep_blank_rows
+      t.options['keep_blank_rows'] || false
+    end
+    
+    # Form data to send in with the download request
+    def form_data
+      t.options['form_data']
+    end
+    
+    # How many rows to skip
+    #
+    # Default: 0
+    def skip
+      t.options['skip'].to_i
+    end
+    
+    # The encoding
+    #
+    # Default: "UTF-8"
+    def encoding
+      t.options['encoding'] || 'UTF-8'
+    end
+    
+    # The delimiter
+    #
+    # Default: ","
+    def delimiter
+      t.options['delimiter'] || ','
+    end
+    
+    # The XPath used to find rows
+    def row_xpath
+      t.options['row_xpath']
+    end
+    
+    # The XPath used to find columns
+    def column_xpath
+      t.options['column_xpath']
+    end
+    
+    # The compression type.
+    #
+    # Default: guessed from URI.
+    #
+    # Can be specified as: "gz", "zip", "bz2", "exe" (treated as "zip")
+    def compression
+      clue = if t.options['compression']
+        t.options['compression'].to_s
+      else
+        ::File.extname uri.path
+      end
+      case clue.downcase
+      when /gz/, /gunzip/
+        'gz'
+      when /zip/
+        'zip'
+      when /bz2/, /bunzip2/
+        'bz2'
+      when /exe/
+        'exe'
+      end
+    end
+    
+    # The packing type.
+    #
+    # Default: guessed from URI.
+    #
+    # Can be specified as: "tar"
+    def packing
+      clue = if t.options['packing']
+        t.options['packing'].to_s
+      else
+        ::File.extname(uri.path.sub(/\.#{compression}\z/, ''))
+      end
+      case clue.downcase
+      when /tar/
+        'tar'
+      end
+    end
+    
+    # The glob used to pick a file out of an archive.
+    #
+    # Example:
+    #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', 'glob' => '/*.csv'
+    def glob
+      t.options['glob']
+    end
+    
+    # The filename, which can be used to pick a file out of an archive.
+    #
+    # Example:
+    #     RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', 'filename' => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv'
+    def filename
+      t.options['filename']
+    end
+    
+    # Cut columns up to this character
+    def cut
+      t.options['cut']
+    end
+    
+    # Crop rows after this line
+    def crop
+      t.options['crop']
+    end
+    
+    # The fixed-width schema, given as an array
+    #
+    # Example:
+    #     RemoteTable.new('http://cloud.github.com/downloads/seamusabshere/remote_table/test2.fixed_width.txt',
+    #                      'format' => 'fixed_width',
+    #                      'skip' => 1,
+    #                      'schema' => [[ 'header4', 10, { :type => :string }  ],
+    #                                  [  'spacer',  1 ],
+    #                                  [  'header5', 10, { :type => :string } ],
+    #                                  [  'spacer',  12 ],
+    #                                  [  'header6', 10, { :type => :string } ]])
+    def schema
+      t.options['schema']
+    end
+    
+    # The name of the fixed-width schema according to Slither
+    def schema_name
+      t.options['schema_name']
+    end
+    
+    # A proc to call to decide whether to return a row.
+    def select
+      t.options['select']
+    end
+    
+    # A proc to call to decide whether to return a row.
+    def reject
+      t.options['reject']
+    end
+    
+    # An Errata object (see the Errata gem at http://github.com/seamusabshere/errata) to be used on every row.
+    def errata
+      t.options['errata']
+    end
+    
+    # Get the format in the form of RemoteTable::Format::Excel, etc.
+    #
+    # Note: treats all spreadsheets.google.com URLs as Format::Delimited (i.e., CSV)
+    #
+    # Default: guessed from file extension (which is usually the same as the URI, but sometimes not if you pick out a specific file from an archive)
+    #
+    # Can be specified as: "xlsx", "xls", "csv", "ods", "fixed_width", "html"
+    def format
+      return Format::Delimited if uri.host == 'spreadsheets.google.com'
+      clue = if t.options['format']
+        t.options['format'].to_s
+      else
+        ::File.extname t.local_file.path
+      end
+      return Format::Delimited if clue.blank?
+      case clue.downcase
+      when /xlsx/, /excelx/
+        Format::Excelx
+      when /xls/, /excel/
+        Format::Excel
+      when /csv/, /tsv/, /delimited/
+        Format::Delimited
+      when /ods/, /open_?office/
+        Format::OpenOffice
+      when /fixed_?width/
+        Format::FixedWidth
+      when /htm/
+        Format::HTML
+      else
+        raise Format::Unknown, clue
+      end
+    end
+  end
+end

data/lib/remote_table/transformer.rb

@@ -0,0 +1,17 @@
+class RemoteTable
+  class Transformer
+    attr_reader :t
+    attr_accessor :legacy_transformer
+    def initialize(t)
+      @t = t
+    end
+    # eventually this will support a different way of specifying a transformer
+    def transform(row)
+      if legacy_transformer
+        legacy_transformer.apply row
+      else
+        [ row ]
+      end
+    end
+  end
+end

data/lib/remote_table/version.rb

@@ -0,0 +1,3 @@
+class RemoteTable
+  VERSION = "1.0.0"
+end

data/lib/remote_table.rb

@@ -1,130 +1,122 @@
-require 'digest/md5'
-require 'uri'
-require 'tmpdir'
-require 'kconv'
 require 'active_support'
 require 'active_support/version'
 %w{
-  active_support/core_ext/string/conversions
-  active_support/core_ext/object/blank
-  active_support/core_ext/string/inflections
-  active_support/core_ext/array/wrap
-  active_support/core_ext/hash/except
-  active_support/core_ext/class/attribute_accessors
+  active_support/core_ext/hash
+  active_support/core_ext/string
+  active_support/core_ext/module
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
 end if ActiveSupport::VERSION::MAJOR == 3
 
-if RUBY_VERSION >= '1.9'
-  require 'csv'
-  FasterCSV = CSV
-else
-  begin
-    require 'fastercsv'
-  rescue LoadError
-    $stderr.puts "[remote_table gem] You probably need to manually install the fastercsv gem."
-    raise $!
-  end
-end
+class RemoteTable
+  autoload :Format, 'remote_table/format'
+  autoload :Properties, 'remote_table/properties'
+  autoload :LocalFile, 'remote_table/local_file'
+  autoload :Transformer, 'remote_table/transformer'
 
-require 'slither'
-require 'escape'
-require 'roo'
-I_KNOW_I_AM_USING_AN_OLD_AND_BUGGY_VERSION_OF_LIBXML2 = true
-require 'nokogiri'
-require 'remote_table/transform'
-require 'remote_table/request'
-require 'remote_table/package'
-require 'remote_table/file'
-require 'remote_table/file/csv'
-require 'remote_table/file/fixed_width'
-require 'remote_table/file/roo_spreadsheet'
-require 'remote_table/file/ods'
-require 'remote_table/file/xls'
-require 'remote_table/file/xlsx'
-require 'remote_table/file/html'
+  # singletons
+  autoload :Cleaner, 'remote_table/cleaner'
+  autoload :Executor, 'remote_table/executor'
+  autoload :Hasher, 'remote_table/hasher'
+  
+  # Legacy
+  class Transform
+    def self.row_hash(row)
+      ::RemoteTable.hasher.hash row
+    end
+  end
 
-class RemoteTable
-  cattr_accessor :paths_for_removal
-  class << self
-    def cleanup
-      paths_for_removal.each do |path|
-        FileUtils.rm_rf path
-        paths_for_removal.delete path
-      end if paths_for_removal.is_a?(Array)
+  include ::Enumerable
+  
+  attr_reader :url
+  attr_reader :options
+  
+  # Create a new RemoteTable.
+  #
+  #     RemoteTable.new(url, options = {})
+  #
+  # New syntax:
+  #     RemoteTable.new('www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', 'foo' => 'bar')
+  # Old syntax:
+  #     RemoteTable.new(:url => 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :foo => 'bar')
+  #
+  # See the <tt>Properties</tt> object for the sorts of options you can pass.
+  def initialize(*args)
+    @options = args.last.is_a?(::Hash) ? args.last.dup : {}
+    if args.first.is_a? ::String
+      @url = args.first.dup
+    else
+      @url = @options['url'] || @options[:url]
     end
-    
-    def remove_at_exit(path)
-      self.paths_for_removal ||= Array.new
-      paths_for_removal.push path
+    # deprecated
+    if options[:transform]
+      transformer.legacy_transformer = options[:transform][:class].new options[:transform].except(:class)
+      transformer.legacy_transformer.add_hints! @options
     end
+    @options.stringify_keys!
+    @url.freeze
+    @options.freeze
+    at_exit { ::RemoteTable.cleaner.cleanup }
   end
-
-  attr_accessor :request, :package, :file, :transform
-  attr_accessor :table
   
-  include Enumerable
+  def each(&blk)
+    format.each do |row|
+      row['row_hash'] = ::RemoteTable.hasher.hash row
+      # allow the transformer to return multiple "virtual rows" for every real row
+      transformer.transform(row).each do |virtual_row|
+        if properties.errata
+          next if properties.errata.rejects? virtual_row
+          properties.errata.correct! virtual_row
+        end
+        next if properties.select and !properties.select.call(virtual_row)
+        next if properties.reject and properties.reject.call(virtual_row)
+        yield virtual_row
+      end
+    end
+  end
   
-  def initialize(bus)
-    @transform = Transform.new(bus)
-    @package = Package.new(bus)
-    @request = Request.new(bus)
-    @file = File.new(bus)
-    at_exit { RemoteTable.cleanup }
+  # Get a row by row number
+  def [](row_number)
+    to_a[row_number]
   end
   
-  def each
-    finish_table! unless table
-    table.each_row { |row| yield row }
+  # Get the whole row array back
+  def rows
+    to_a
   end
-  alias :each_row :each
   
-  def to_a
-    cache_rows! if @_row_cache.nil?
-    @_row_cache
+  # Used internally as a sort of garbage collector.
+  def self.cleaner
+    Cleaner.instance
   end
-  alias :rows :to_a
   
-  def <=>(other)
-    raise "Not implemented"
+  # Used internally to execute stuff in shells.
+  def self.executor
+    Executor.instance
   end
-
-  protected
   
-  def self.bang(path, cmd)
-    tmp_path = "#{path}.tmp"
-    RemoteTable.backtick_with_reporting "cat #{Escape.shell_single_word path} | #{cmd} > #{Escape.shell_single_word tmp_path}"
-    FileUtils.mv tmp_path, path
+  # Used internally to create unique hashes of rows.
+  def self.hasher
+    Hasher.instance
   end
   
-  # TODO this should probably live somewhere else
-  def self.backtick_with_reporting(cmd)
-    cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
-    output = `#{cmd}`
-    if not $?.success?
-      raise %{
-From the remote_table gem...
-
-Command failed:
-#{cmd}
-
-Output:
-#{output}
-}
-    end
+  # Used internally to access to a downloaded copy of the file
+  def local_file
+    @local_file ||= LocalFile.new self
   end
   
-  private
+  # Used internally to access to the properties of the table, either set by the user or implied
+  def properties
+    @properties ||= Properties.new self
+  end
   
-  def finish_table!
-    package_path = request.download
-    file_path = package.stage(package_path)
-    raw_table = file.tabulate(file_path)
-    self.table = transform.apply(raw_table) # must return something that responds to each_row
+  # Used internally to access to the driver that reads the format
+  def format
+    @format ||= properties.format.new self
   end
   
-  def cache_rows!
-    @_row_cache = []
-    each_row { |row| @_row_cache << row }
+  # Used internally to acess the transformer (aka parser).
+  def transformer
+    @transformer ||= Transformer.new self
   end
 end

data/remote_table.gemspec

@@ -1,83 +1,38 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "remote_table/version"
 
 Gem::Specification.new do |s|
-  s.name = %q{remote_table}
-  s.version = "0.2.32"
+  s.name        = "remote_table"
+  s.version     = RemoteTable::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Seamus Abshere", "Andy Rossmeissl"]
+  s.email       = ["seamus@abshere.net"]
+  s.homepage    = "https://github.com/seamusabshere/remote_table"
+  s.summary     = "Open local or remote XLSX, XLS, ODS, CSV and fixed-width files."
+  s.description = %q{Gives you a standard way to parse various formats and treat them as an array of hashes.}
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-10-07}
-  s.description = %q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
-  s.email = %q{seamus@abshere.net}
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-     ".gitignore",
-     "CHANGELOG",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "lib/remote_table.rb",
-     "lib/remote_table/file.rb",
-     "lib/remote_table/file/csv.rb",
-     "lib/remote_table/file/fixed_width.rb",
-     "lib/remote_table/file/html.rb",
-     "lib/remote_table/file/ods.rb",
-     "lib/remote_table/file/roo_spreadsheet.rb",
-     "lib/remote_table/file/xls.rb",
-     "lib/remote_table/file/xlsx.rb",
-     "lib/remote_table/package.rb",
-     "lib/remote_table/request.rb",
-     "lib/remote_table/transform.rb",
-     "remote_table.gemspec",
-     "test/remote_table_test.rb",
-     "test/test_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/seamusabshere/remote_table}
-  s.rdoc_options = ["--charset=UTF-8", "--line-numbers", "--inline-source"]
-  s.require_paths = ["lib"]
-  s.requirements = ["curl"]
-  s.rubyforge_project = %q{remotetable}
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Remotely open and parse XLS, ODS, CSV and fixed-width tables.}
-  s.test_files = [
-    "test/remote_table_test.rb",
-     "test/test_helper.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.rubyforge_project = "remotetable"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<roo>, ["= 1.3.11"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_runtime_dependency(%q<slither>, [">= 0.99.4"])
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
-      s.add_runtime_dependency(%q<escape>, [">= 0.0.4"])
-      s.add_development_dependency(%q<errata>, [">= 0.2.0"])
-    else
-      s.add_dependency(%q<roo>, ["= 1.3.11"])
-      s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_dependency(%q<slither>, [">= 0.99.4"])
-      s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
-      s.add_dependency(%q<escape>, [">= 0.0.4"])
-      s.add_dependency(%q<errata>, [">= 0.2.0"])
-    end
-  else
-    s.add_dependency(%q<roo>, ["= 1.3.11"])
-    s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-    s.add_dependency(%q<slither>, [">= 0.99.4"])
-    s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
-    s.add_dependency(%q<escape>, [">= 0.0.4"])
-    s.add_dependency(%q<errata>, [">= 0.2.0"])
-  end
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency 'activesupport', '>=2.3.4'
+  # sabshere 9/30/10 depending on fastercsv when using ruby 1.9.2 results in exiting with error
+  # s.add_dependency 'fastercsv', '>=1.5.0'
+  s.add_dependency 'roo', '~>1.9'
+  s.add_dependency 'slither', '>=0.99.4'
+  s.add_dependency 'i18n' # activesupport?
+  s.add_dependency 'builder' # roo?
+  s.add_dependency 'zip' # roo
+  s.add_dependency 'nokogiri', '>=1.4.1' # roo
+  s.add_dependency 'spreadsheet' #roo
+  s.add_dependency 'google-spreadsheet-ruby' #roo
+  s.add_dependency 'escape', '>=0.0.4'
+  s.add_development_dependency 'errata', '>=0.2.0'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'shoulda'
+  s.add_development_dependency 'ruby-debug'
 end
-

data/test/{test_helper.rb → helper.rb}

@@ -1,9 +1,16 @@
 require 'rubygems'
+require 'bundler'
+unless RUBY_VERSION >= '1.9'
+  gem 'fastercsv'
+  require 'fastercsv'
+end
+Bundler.setup
 require 'test/unit'
 require 'shoulda'
+require 'ruby-debug'
 # require 'ruby-debug'
-require 'errata'
-require 'active_support/all'
+# require 'errata'
+# require 'active_support/all'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))