remote_table 1.3.0 → 1.4.0

data/README.rdoc

@@ -1,167 +0,0 @@
-=remote_table
-
-Open local or remote XLSX, XLS, ODS, CSV and fixed-width files.
-
-==Real-life usage
-
-Used by http://data.brighterplanet.com and the data_miner gem (http://github.com/seamusabshere/data_miner)
-
-==Requirements
-
-* POSIX operating system (not windows)
-* curl, iconv, perl, cat, cut, tail, etc. accessible from /usr/local/bin:/usr/bin:/bin
-
-As this library matures, those should go away.
-
-==Example
-
-    ?> t = RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv'
-    => #<RemoteTable:0x359da50 [...]>
-    ?> t[0]
-    => {"cyl"=>"6", "eng dscr"=>"DOHC VTEC", "trans dscr"=>"2MODE CLKUP", "trans"=>"Auto(L4)", "cmb"=>"20", "2pv"=>nil, "carline name"=>"NSX", "displ"=>"3.0", "ucmb"=>"23.5311", "hpv"=>nil, "4pv"=>nil, "Class"=>"TWO SEATERS", "Manufacturer"=>"ACURA", "fl"=>"P", "2lv"=>nil, "G"=>nil, "hlv"=>nil, "drv"=>"R", "cty"=>"18", "ucty"=>"19.8733", "S"=>nil, "4lv"=>nil, "fcost"=>"1050", "T"=>nil, "hwy"=>"24", "uhwy"=>"30.3612"}
-
-More examples:
-
-    RemoteTable.new "file://#{f.path}", :quote_char => %{'}, :headers => nil
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.alternate_order.csv'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.alternate_order.ods'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.alternate_order.xls'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.csv'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.ods'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.xls'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/test2.#{format}'
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/test2.#{format}', :keep_blank_rows => true
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA&single=true&gid=0'
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA'
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA', :skip => 1, :headers => false
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw'
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :headers => %w{ col1 col2 col3 }
-
-    RemoteTable.new 'http://spreadsheets.google.com/pub?key=tujrgUOwDSLWb-P4KCt1qBg'
-
-    RemoteTable.new 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls', :transform => { :class => FuelOilParser }
-
-    RemoteTable.new 'http://www.freebase.com/type/exporttypeinstances/base/horses/horse_breed?page=0&filter_mode=type&filter_view=table&show%01p%3D%2Ftype%2Fobject%2Fname%01index=0&show%01p%3D%2Fcommon%2Ftopic%2Fimage%01index=1&show%01p%3D%2Fcommon%2Ftopic%2Farticle%01index=2&sort%01p%3D%2Ftype%2Fobject%2Ftype%01p%3Dlink%01p%3D%2Ftype%2Flink%2Ftimestamp%01index=false&=&exporttype=csv-8'
-
-    RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls'
-
-    RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv'
-
-    RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :glob => '/*.csv'
-
-    RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv'
-
-    RemoteTable.new 'http://www.worldmapper.org/data/opendoc/2_worldmapper_data.ods', :sheet => 'Data', :keep_blank_rows => true
-
-    RemoteTable.new 'https://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA'
-
-    RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
-
-    RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :headers => %w{foo bar baz}
-
-    RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx', :headers => false
-
-    RemoteTable.new 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0', :form_data => 'UserTableName=T_100_Segment__All_Carriers&[...]', :compression => :zip, :glob => '/*.csv'
-
-    RemoteTable.new "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-E.htm",
-                    :encoding => 'US-ASCII',
-                    :row_xpath => '//table/tr[2]/td/table/tr',
-                    :column_xpath => 'td'
-
-    RemoteTable.new "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-G.htm",
-                    :encoding => 'windows-1252',
-                    :row_xpath => '//table/tr[2]/td/table/tr',
-                    :column_xpath => 'td',
-                    :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
-                                          :responder => AircraftGuru.new)
-
-    RemoteTable.new "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-G.htm",
-                    :encoding => 'windows-1252',
-                    :row_xpath => '//table/tr[2]/td/table/tr',
-                    :column_xpath => 'td',
-                    :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
-                                 :responder => AircraftGuru.new }
-
-    RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
-                    :filename => 'Gd6-dsc.txt',
-                    :format => :fixed_width,
-                    :crop => 21..26, # inclusive
-                    :cut => '2-',
-                    :select => lambda { |row| /\A[A-Z]/.match row['code'] },
-                    :schema => [[ 'code',   2, { :type => :string }  ],
-                                [ 'spacer', 2 ],
-                                [ 'name',   52, { :type => :string } ]]
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/test2.fixed_width.txt',
-                    :format => :fixed_width,
-                    :skip => 1,
-                    :schema => [[ 'header4', 10, { :type => :string }  ],
-                                [ 'spacer',  1 ],
-                                [ 'header5', 10, { :type => :string } ],
-                                [ 'spacer',  12 ],
-                                [ 'header6', 10, { :type => :string } ]]
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/test2.fixed_width.txt',
-                    :format => :fixed_width,
-                    :keep_blank_rows => true,
-                    :skip => 1,
-                    :schema => [[ 'header4', 10, { :type => :string }  ],
-                                [ 'spacer',  1 ],
-                                [ 'header5', 10, { :type => :string } ],
-                                [ 'spacer',  12 ],
-                                [ 'header6', 10, { :type => :string } ]]
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.fixed_width.txt',
-                    :format => :fixed_width,
-                    :skip => 1,
-                    :schema => [[ 'header1', 10, { :type => :string }  ],
-                                [ 'spacer',  1 ],
-                                [ 'header2', 10, { :type => :string } ],
-                                [ 'spacer',  12 ],
-                                [ 'header3', 10, { :type => :string } ]]
-
-    RemoteTable.new 'http://cloud.github.com/downloads/seamusabshere/remote_table/remote_table_row_hash_test.alternate_order.fixed_width.txt',
-                    :format => :fixed_width,
-                    :skip => 1,
-                    :schema => [[ 'spacer',  11 ],
-                                [ 'header2', 10, { :type => :string }  ],
-                                [ 'spacer',  1 ],
-                                [ 'header3', 10, { :type => :string } ],
-                                [ 'spacer',  1 ],
-                                [ 'header1', 10, { :type => :string } ]]
-
-==Helpful hints
-
-* ASCII-8BIT is the same as BINARY
-* ISO-8859-1 is the same as Latin1
-
-==Custom parsers
-
-See the test file and also data_miner examples of custom parsers.
-
-==Wishlist
-
-* The new parser syntax (aka transformer) hasn't been defined yet... only the old-style syntax is available
-* We currently call curl (and a lot of other utilities) using a shell. Is there a safer way to do this?
-
-==Authors
-
-* Seamus Abshere <seamus@abshere.net>
-* Andy Rossmeissl <andy@rossmeissl.net>
-
-== Copyright
-
-Copyright (c) 2011 Brighter Planet. See LICENSE for details.

data/lib/remote_table/utils.rb

@@ -1,157 +0,0 @@
-require 'fileutils'
-require 'posix/spawn'
-require 'tmpdir'
-
-class RemoteTable
-  class SpawnError < ::RuntimeError; end
-  
-  module Utils
-    def self.tmp_path(ancestor)
-      basename = ::File.basename(ancestor).sub(/remote_table-[0-9]+-/, '')
-      ::Kernel.srand
-      ::File.join ::Dir.tmpdir, "remote_table-#{::Kernel.rand(1e11)}-#{basename}"
-    end
-
-    def self.spawn(*argv)
-      options = argv.extract_options!
-      if options[:in] or options[:out]
-        # capture these now because posix/spawn is known to bork them
-        in_out = options.slice(:in, :out).map { |k, v| ":#{k} => #{v.path}" }.join(', ')
-        # --
-        pid = ::POSIX::Spawn.spawn *argv, options
-        ::Process.waitpid pid
-        raise SpawnError, "[remote_table] spawn #{argv.join(' ')} (#{in_out}) failed with exit status #{$?.exitstatus}" unless $?.success?
-      else
-        child = ::POSIX::Spawn::Child.new *argv
-        raise SpawnError, "[remote_table] spawn #{argv.join(' ')}) failed with #{child.err}" unless child.success?
-      end
-      nil
-    end
-
-    def self.in_place(*args)
-      options = args.extract_options!
-      input = args.shift
-      argv = args
-      output = tmp_path input
-      ::File.open(input, 'r') do |f0|
-        ::File.open(output, 'wb') do |f1|
-          spawn *argv, :in => f0, :out => f1
-        end
-      end
-      ::FileUtils.mv output, input
-      nil
-    rescue SpawnError => e
-      if options[:ignore_error]
-        $stderr.puts "#{e.inspect} (ignoring error...)"
-        ::FileUtils.mv output, input
-      else
-        raise e
-      end
-    end
-
-    def self.download(uri, form_data = nil)
-      output = tmp_path uri.path
-
-      if uri.scheme == 'file'
-        $stderr.puts "[remote_table] Getting #{uri.path} from the local file system" if ::ENV['REMOTE_TABLE_VERBOSE'] == 'true'
-        ::FileUtils.cp uri.path, output
-        return output
-      end
-      
-      argv = [ 'curl', '--location', '--show-error', '--silent', '--compressed', '--header', 'Expect: ' ]
-      if form_data
-        argv += [ '--data', form_data ]
-      end
-      argv += [ uri.to_s, '--output', output ]
-
-      # sabshere 7/20/11 make web requests move more slowly so you don't get accused of DOS
-      if ::ENV.has_key?('REMOTE_TABLE_DELAY_BETWEEN_REQUESTS')
-        ::Kernel.sleep ::ENV['REMOTE_TABLE_DELAY_BETWEEN_REQUESTS'].to_i
-      end
-
-      $stderr.puts "[remote_table] Downloading #{uri.to_s}" if ::ENV['REMOTE_TABLE_VERBOSE'] == 'true'
-      spawn *argv
-      output
-    end
-    
-    def self.decompress(input, compression)
-      case compression
-      when :zip, :exe
-        Utils.unzip input
-      when :bz2
-        Utils.bunzip2 input
-      when :gz
-        Utils.gunzip input
-      else
-        raise ::ArgumentError, "[remote_table] Unrecognized compression #{compression}"
-      end
-    end
-    
-    def self.unpack(input, packing)
-      case packing
-      when :tar
-        Utils.untar input
-      else
-        raise ::ArgumentError, "[remote_table] Unrecognized packing #{packing}"
-      end
-    end
-    
-    def self.pick(input, options = {})
-      options = options.symbolize_keys
-      if (options[:filename] or options[:glob]) and not ::File.directory?(input)
-        raise ::RuntimeError, "[remote_table] Expecting #{input} to be a directory"
-      end
-      if filename = options[:filename]
-        src = ::File.join input, filename
-        raise(::RuntimeError, "[remote_table] Expecting #{src} to be a file") unless ::File.file?(src)
-        output = tmp_path src
-        ::FileUtils.mv src, output
-        ::FileUtils.rm_rf input if ::File.dirname(input).start_with?(::Dir.tmpdir)
-      elsif glob = options[:glob]
-        src = ::Dir[input+glob].first
-        raise(::RuntimeError, "[remote_table] Expecting #{glob} to find a file in #{input}") unless src and ::File.file?(src)
-        output = tmp_path src
-        ::FileUtils.mv src, output
-        ::FileUtils.rm_rf input if ::File.dirname(input).start_with?(::Dir.tmpdir)
-      else
-        output = tmp_path input
-        ::FileUtils.mv input, output
-      end
-      output
-    end
-
-    def self.gunzip(input)
-      output = tmp_path input
-      ::File.open(output, 'wb') do |f|
-        spawn 'gunzip', '--stdout', input, :out => f
-      end
-      ::FileUtils.rm_f input
-      output
-    end
-
-    def self.bunzip2(input)
-      output = tmp_path input
-      ::File.open(output, 'wb') do |f|
-        spawn 'bunzip2', '--stdout', input, :out => f
-      end
-      ::FileUtils.rm_f input
-      output
-    end
-
-    def self.untar(input)
-      dest_dir = tmp_path input
-      ::FileUtils.mkdir dest_dir
-      spawn 'tar', '-xf', input, '-C', dest_dir
-      ::FileUtils.rm_f input
-      dest_dir
-    end
-
-    def self.unzip(input)
-      dest_dir = tmp_path input
-      ::FileUtils.mkdir dest_dir
-      spawn 'unzip', '-qq', '-n', input, '-d', dest_dir
-      ::FileUtils.rm_f input
-      dest_dir
-    end
-  end
-end