remote_table-ruby19 0.2.30

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/CHANGELOG

@@ -0,0 +1,3 @@
+0.1.6
+* For CSVs, force convert headers using String#toutf8. :encoding => 'N'|'U' didn't work.
+* Fix handling of long urls when passing off to Tempfile.

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Brighter Planet
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,27 @@
+=remote_table
+
+Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.
+
+==Real-life usage
+
+Used by data_miner (http://github.com/seamusabshere/data_miner)
+
+==Example
+
+Taken from <tt>#{GEMDIR}/test/remote_table_test.rb</tt>:
+
+  >> t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
+  => #<RemoteTable:0x359da50 @transform=#<RemoteTable::Transform:0x359d154 @select=nil, @reject=nil>, @file=#<RemoteTable::File:0x35970c4 @delimiter=nil, @headers=nil, @cut=nil, @filename="98guide6.csv", @skip=nil, @schema_name=nil, @crop=nil, @format=:csv, @trap=nil, @sheet=0, @schema=nil>, @package=#<RemoteTable::Package:0x359c538 @url="http://www.fueleconomy.gov/FEG/epadata/98guide6.zip", @filename="98guide6.csv", @compression=:zip, @packing=nil>, @request=#<RemoteTable::Request:0x3596bec @url="http://www.fueleconomy.gov/FEG/epadata/98guide6.zip">>
+  >> t.rows.first
+  => {"cyl"=>"6", "eng dscr"=>"DOHC VTEC", "trans dscr"=>"2MODE CLKUP", "trans"=>"Auto(L4)", "cmb"=>"20", "2pv"=>nil, "carline name"=>"NSX", "displ"=>"3.0", "ucmb"=>"23.5311", "hpv"=>nil, "4pv"=>nil, "Class"=>"TWO SEATERS", "Manufacturer"=>"ACURA", "fl"=>"P", "2lv"=>nil, "G"=>nil, "hlv"=>nil, "drv"=>"R", "cty"=>"18", "ucty"=>"19.8733", "S"=>nil, "4lv"=>nil, "fcost"=>"1050", "T"=>nil, "hwy"=>"24", "uhwy"=>"30.3612"}
+
+See the test file and also data_miner examples of custom parsers.
+
+==Authors
+
+* Seamus Abshere <seamus@abshere.net>
+* Andy Rossmeissl <andy@rossmeissl.net>
+
+== Copyright
+
+Copyright (c) 2010 Brighter Planet. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,70 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "remote_table"
+    gem.summary = %Q{Remotely open and parse XLS, ODS, CSV and fixed-width tables.}
+    gem.description = %Q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
+    gem.email = "seamus@abshere.net"
+    gem.homepage = "http://github.com/seamusabshere/remote_table"
+    gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
+    gem.add_dependency 'roo', '1.3.11' # 1.9.3 breaks
+    gem.add_dependency 'fastercsv', '>=1.5.0'
+    gem.add_dependency 'activesupport', '>=2.3.4'
+    gem.add_dependency 'slither', '>=0.99.3'
+    gem.add_dependency 'nokogiri', '>=1.4.1'
+    gem.add_dependency 'escape', '>=0.0.4'
+    gem.add_development_dependency 'errata', '>=0.2.0'
+    gem.require_path = "lib"
+    gem.rdoc_options << '--line-numbers' << '--inline-source'
+    gem.requirements << 'curl'
+    gem.rubyforge_project = "remotetable"
+  end
+  Jeweler::GemcutterTasks.new
+  Jeweler::RubyforgeTasks.new do |rubyforge|
+    rubyforge.doc_task = "rdoc"
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "remote_table #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.30

data/lib/remote_table.rb

@@ -0,0 +1,117 @@
+require 'digest/md5'
+require 'uri'
+require 'tmpdir'
+require 'active_support'
+require 'active_support/version'
+%w{
+  active_support/core_ext/string/conversions
+  active_support/core_ext/object/blank
+  active_support/core_ext/string/inflections
+  active_support/core_ext/array/wrap
+  active_support/core_ext/hash/except
+  active_support/core_ext/class/attribute_accessors
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ActiveSupport::VERSION::MAJOR == 3
+
+require 'escape'
+require 'slither'
+require 'roo'
+I_KNOW_I_AM_USING_AN_OLD_AND_BUGGY_VERSION_OF_LIBXML2 = true
+require 'nokogiri'
+require 'remote_table/transform'
+require 'remote_table/request'
+require 'remote_table/package'
+require 'remote_table/file'
+require 'remote_table/file/csv'
+require 'remote_table/file/fixed_width'
+require 'remote_table/file/roo_spreadsheet'
+require 'remote_table/file/ods'
+require 'remote_table/file/xls'
+require 'remote_table/file/xlsx'
+require 'remote_table/file/html'
+
+class RemoteTable
+  cattr_accessor :paths_for_removal
+  class << self
+    def cleanup
+      paths_for_removal.each do |path|
+        FileUtils.rm_rf path
+        paths_for_removal.delete path
+      end if paths_for_removal.is_a?(Array)
+    end
+    
+    def remove_at_exit(path)
+      self.paths_for_removal ||= Array.new
+      paths_for_removal.push path
+    end
+  end
+
+  attr_accessor :request, :package, :file, :transform
+  attr_accessor :table
+  
+  include Enumerable
+  
+  def initialize(bus)
+    @transform = Transform.new(bus)
+    @package = Package.new(bus)
+    @request = Request.new(bus)
+    @file = File.new(bus)
+    at_exit { RemoteTable.cleanup }
+  end
+  
+  def each
+    finish_table! unless table
+    table.each_row { |row| yield row }
+  end
+  alias :each_row :each
+  
+  def to_a
+    cache_rows! if @_row_cache.nil?
+    @_row_cache
+  end
+  alias :rows :to_a
+  
+  def <=>(other)
+    raise "Not implemented"
+  end
+
+  protected
+  
+  def self.bang(path, cmd)
+    tmp_path = "#{path}.tmp"
+    RemoteTable.backtick_with_reporting "cat #{Escape.shell_single_word path} | #{cmd} > #{Escape.shell_single_word tmp_path}"
+    FileUtils.mv tmp_path, path
+  end
+  
+  # TODO this should probably live somewhere else
+  def self.backtick_with_reporting(cmd)
+    cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
+    output = `#{cmd}`
+    if not $?.success?
+      raise %{
+From the remote_table gem...
+
+Command failed:
+#{cmd}
+
+Output:
+#{output}
+}
+    end
+  end
+  
+  private
+  
+  def finish_table!
+    package_path = request.download
+    file_path = package.stage(package_path)
+    raw_table = file.tabulate(file_path)
+    self.table = transform.apply(raw_table) # must return something that responds to each_row
+  end
+  
+  def cache_rows!
+    @_row_cache = []
+    each_row { |row| @_row_cache << row }
+  end
+end

data/lib/remote_table/file.rb

@@ -0,0 +1,100 @@
+class RemoteTable
+  class File
+    attr_accessor :filename, :format, :delimiter, :skip, :cut, :crop, :sheet, :headers, :schema, :schema_name, :trap
+    attr_accessor :encoding
+    attr_accessor :path
+    attr_accessor :keep_blank_rows
+    attr_accessor :row_xpath
+    attr_accessor :column_xpath
+    
+    def initialize(bus)
+      @filename = bus[:filename]
+      @format = bus[:format] || format_from_filename
+      @delimiter = bus[:delimiter]
+      @sheet = bus[:sheet] || 0
+      @skip = bus[:skip] # rows
+      @keep_blank_rows = bus[:keep_blank_rows] || false
+      @crop = bus[:crop] # rows
+      @cut = bus[:cut]   # columns
+      @headers = bus[:headers]
+      @schema = bus[:schema]
+      @schema_name = bus[:schema_name]
+      @trap = bus[:trap]
+      @encoding = bus[:encoding] || 'UTF-8'
+      @row_xpath = bus[:row_xpath]
+      @column_xpath = bus[:column_xpath]
+      extend "RemoteTable::#{format.to_s.camelcase}".constantize
+    end
+    
+    def tabulate(path)
+      define_fixed_width_schema! if format == :fixed_width and schema.is_a?(Array) # TODO move to generic subclass callback
+      self.path = path
+      self
+    end
+    
+    private
+    
+    # doesn't support trap
+    def define_fixed_width_schema!
+      raise "can't define both schema_name and schema" if !schema_name.blank?
+      self.schema_name = "autogenerated_#{filename.gsub(/[^a-z0-9_]/i, '')}".to_sym
+      self.trap ||= lambda { true }
+      Slither.define schema_name do |d|
+        d.rows do |row|
+          row.trap(&trap)
+          schema.each do |name, width, options|
+            if name == 'spacer'
+              row.spacer width
+            else
+              row.column name, width, options
+            end
+          end
+        end
+      end
+    end
+    
+    def backup_file!
+      FileUtils.cp path, "#{path}.backup"
+    end
+    
+    def skip_rows!
+      return unless skip
+      RemoteTable.bang path, "tail -n +#{skip + 1}"
+    end
+    
+    USELESS_CHARACTERS = [
+      '\xef\xbb\xbf',   # UTF-8 byte order mark
+      '\xc2\xad'        # soft hyphen, often inserted by MS Office (html: ­)
+    ]
+    def remove_useless_characters!
+      RemoteTable.bang path, "perl -pe 's/#{USELESS_CHARACTERS.join '//g; s/'}//g'"
+    end
+    
+    def convert_file_to_utf8!
+      RemoteTable.bang path, "iconv -c -f #{Escape.shell_single_word encoding} -t UTF-8"
+    end
+    
+    def restore_file!
+      FileUtils.mv "#{path}.backup", path if ::File.readable? "#{path}.backup"
+    end
+    
+    def cut_columns!
+      return unless cut
+      RemoteTable.bang path, "cut -c #{Escape.shell_single_word cut.to_s}"
+    end
+    
+    def crop_rows!
+      return unless crop
+      RemoteTable.bang path, "tail -n +#{Escape.shell_single_word crop.first.to_s} | head -n #{crop.last - crop.first + 1}"
+    end
+    
+    def format_from_filename
+      extname = ::File.extname(filename).gsub('.', '')
+      return :csv if extname.blank?
+      format = [ :xls, :ods, :xlsx ].detect { |i| i == extname.to_sym }
+      format = :html if extname =~ /\Ahtm/
+      format = :csv if format.blank?
+      format
+    end
+  end
+end

data/lib/remote_table/file/csv.rb

@@ -0,0 +1,57 @@
+if RUBY_VERSION >= "1.9"
+  require 'csv'
+  ::RemoteTable::MyCSV = ::CSV
+else
+  require 'fastercsv'
+  ::RemoteTable::MyCSV = ::FasterCSV
+end
+
+class RemoteTable
+  module Csv
+    def each_row(&block)
+      backup_file!
+      convert_file_to_utf8!
+      remove_useless_characters!
+      skip_rows!
+      ::RemoteTable::MyCSV.foreach(path, fastercsv_options) do |row|
+        ordered_hash = ActiveSupport::OrderedHash.new
+        filled_values = 0
+        case row
+        when ::RemoteTable::MyCSV::Row
+          row.each do |header, value|
+            next if header.blank?
+            value = '' if value.nil?
+            ordered_hash[header] = value
+            filled_values += 1 if value.present?
+          end
+        when Array
+          index = 0
+          row.each do |value|
+            value = '' if value.nil?
+            ordered_hash[index] = value
+            filled_values += 1 if value.present?
+            index += 1
+          end
+        else
+          raise "Unexpected #{row.inspect}"
+        end
+        yield ordered_hash if keep_blank_rows or filled_values.nonzero?
+      end
+    ensure
+      restore_file!
+    end
+    
+    private
+    
+    def fastercsv_options
+      fastercsv_options = { :skip_blanks => !keep_blank_rows }
+      if headers == false
+        fastercsv_options.merge!(:headers => nil)
+      else
+        fastercsv_options.merge!(:headers => :first_row)
+      end
+      fastercsv_options.merge!(:col_sep => delimiter) if delimiter
+      fastercsv_options
+    end
+  end
+end

data/lib/remote_table/file/fixed_width.rb

@@ -0,0 +1,19 @@
+class RemoteTable
+  module FixedWidth
+    def each_row(&block)
+      backup_file!
+      convert_file_to_utf8!
+      remove_useless_characters!
+      crop_rows!
+      skip_rows!
+      cut_columns!
+      a = Slither.parse(path, schema_name)
+      a[:rows].each do |hash|
+        hash.reject! { |k, v| k.blank? }
+        yield hash if keep_blank_rows or hash.any? { |k, v| v.present? }
+      end
+    ensure
+      restore_file!
+    end
+  end
+end

data/lib/remote_table/file/html.rb

@@ -0,0 +1,37 @@
+class RemoteTable
+  module Html
+    def each_row(&block)
+      backup_file!
+      convert_file_to_utf8!
+      remove_useless_characters!
+      html_headers = (headers.is_a?(Array)) ? headers : nil
+      Nokogiri::HTML(unescaped_html_without_soft_hyphens, nil, 'UTF-8').xpath(row_xpath).each do |row|
+        values = row.xpath(column_xpath).map { |td| td.content.gsub(/\s+/, ' ').strip }
+        if html_headers.nil?
+          html_headers = values
+          next
+        end
+        hash = zip html_headers, values
+        yield hash if keep_blank_rows or hash.any? { |k, v| v.present? }
+      end
+    ensure
+      restore_file!
+    end
+
+    private
+    
+    # http://snippets.dzone.com/posts/show/406
+    def zip(keys, values)
+      hash = Hash.new
+      keys.zip(values) { |k,v| hash[k]=v }
+      hash
+    end
+    
+    # should we be doing this in ruby?
+    def unescaped_html_without_soft_hyphens
+      str = CGI.unescapeHTML IO.read(path)
+      str.gsub! /­/, ''
+      str
+    end
+  end
+end