remote_table 0.2.32 → 1.0.0

data/CHANGELOG

@@ -1,3 +1,8 @@
+1.0.0
+* Refactored to follow more Ruby conventions
+* Suggesting new syntax that looks more like an Enumerable... t[5] instead of t.rows[5]
+* Switching to string option keys (but old syntax is supported)
+[...no changelog for 0.1.6--1.0.0...sorry]
 0.1.6
 * For CSVs, force convert headers using String#toutf8. :encoding => 'N'|'U' didn't work.
 * Fix handling of long urls when passing off to Tempfile.

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in remote_table.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: .
+  specs:
+    remote_table (1.0.0)
+      activesupport (>= 2.3.4)
+      builder
+      escape (>= 0.0.4)
+      google-spreadsheet-ruby
+      i18n
+      nokogiri (>= 1.4.1)
+      roo (~> 1.9)
+      slither (>= 0.99.4)
+      spreadsheet
+      zip
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.0.3)
+    builder (3.0.0)
+    columnize (0.3.2)
+    errata (0.2.4)
+      activesupport (>= 2.3.4)
+      remote_table (>= 0.2.31)
+    escape (0.0.4)
+    google-spreadsheet-ruby (0.1.2)
+      nokogiri (>= 1.4.3.1)
+      oauth (>= 0.3.6)
+    i18n (0.5.0)
+    linecache (0.43)
+    nokogiri (1.4.3.1)
+    oauth (0.4.4)
+    roo (1.9.3)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    ruby-ole (1.2.10.1)
+    shoulda (2.10.3)
+    slither (0.99.4)
+    spreadsheet (0.6.4.1)
+      ruby-ole
+    test-unit (2.1.2)
+    zip (2.0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (>= 2.3.4)
+  builder
+  errata (>= 0.2.0)
+  escape (>= 0.0.4)
+  google-spreadsheet-ruby
+  i18n
+  nokogiri (>= 1.4.1)
+  remote_table!
+  roo (~> 1.9)
+  ruby-debug
+  shoulda
+  slither (>= 0.99.4)
+  spreadsheet
+  test-unit
+  zip

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Brighter Planet
+Copyright (c) 2011 Brighter Planet
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -1,6 +1,6 @@
 =remote_table
 
-Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.
+Open local or remote XLSX, XLS, ODS, CSV and fixed-width files.
 
 ==Real-life usage
 
@@ -8,15 +8,29 @@ Used by data_miner (http://github.com/seamusabshere/data_miner)
 
 ==Example
 
-Taken from <tt>#{GEMDIR}/test/remote_table_test.rb</tt>:
+Taken from <tt>#{GEMDIR}/test/test_remote_table.rb</tt>:
 
-  >> t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
-  => #<RemoteTable:0x359da50 @transform=#<RemoteTable::Transform:0x359d154 @select=nil, @reject=nil>, @file=#<RemoteTable::File:0x35970c4 @delimiter=nil, @headers=nil, @cut=nil, @filename="98guide6.csv", @skip=nil, @schema_name=nil, @crop=nil, @format=:csv, @trap=nil, @sheet=0, @schema=nil>, @package=#<RemoteTable::Package:0x359c538 @url="http://www.fueleconomy.gov/FEG/epadata/98guide6.zip", @filename="98guide6.csv", @compression=:zip, @packing=nil>, @request=#<RemoteTable::Request:0x3596bec @url="http://www.fueleconomy.gov/FEG/epadata/98guide6.zip">>
-  >> t.rows.first
-  => {"cyl"=>"6", "eng dscr"=>"DOHC VTEC", "trans dscr"=>"2MODE CLKUP", "trans"=>"Auto(L4)", "cmb"=>"20", "2pv"=>nil, "carline name"=>"NSX", "displ"=>"3.0", "ucmb"=>"23.5311", "hpv"=>nil, "4pv"=>nil, "Class"=>"TWO SEATERS", "Manufacturer"=>"ACURA", "fl"=>"P", "2lv"=>nil, "G"=>nil, "hlv"=>nil, "drv"=>"R", "cty"=>"18", "ucty"=>"19.8733", "S"=>nil, "4lv"=>nil, "fcost"=>"1050", "T"=>nil, "hwy"=>"24", "uhwy"=>"30.3612"}
+    should "open an XLSX" do
+      t = RemoteTable.new 'www.customerreferenceprogram.org/uploads/CRP_RFP_template.xlsx'
+      assert_equal "Secure encryption of all data", t[5]["Requirements"]
+    end
+
+or on the console
+
+    ?> t = RemoteTable.new 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', 'filename' => '98guide6.csv'
+    => #<RemoteTable:0x359da50 [...]>
+    ?> t[0]
+    => {"cyl"=>"6", "eng dscr"=>"DOHC VTEC", "trans dscr"=>"2MODE CLKUP", "trans"=>"Auto(L4)", "cmb"=>"20", "2pv"=>nil, "carline name"=>"NSX", "displ"=>"3.0", "ucmb"=>"23.5311", "hpv"=>nil, "4pv"=>nil, "Class"=>"TWO SEATERS", "Manufacturer"=>"ACURA", "fl"=>"P", "2lv"=>nil, "G"=>nil, "hlv"=>nil, "drv"=>"R", "cty"=>"18", "ucty"=>"19.8733", "S"=>nil, "4lv"=>nil, "fcost"=>"1050", "T"=>nil, "hwy"=>"24", "uhwy"=>"30.3612"}
 
 See the test file and also data_miner examples of custom parsers.
 
+==Wishlist
+
+* The new parser syntax (aka transformer) hasn't been defined yet... only the old-style syntax is available
+* We currently call curl (and a lot of other utilities) using a shell. Is there a safer way to do this?
+* Row hashes may come out differently for Ruby 1.8 and Ruby 1.9, which ruins the whole purpose.
+* Since <tt>Enumerable</tt> provides <tt>#to_a</tt>, I'm not sure if it's caching the row loading.
+
 ==Authors
 
 * Seamus Abshere <seamus@abshere.net>
@@ -24,4 +38,4 @@ See the test file and also data_miner examples of custom parsers.
 
 == Copyright
 
-Copyright (c) 2010 Brighter Planet. See LICENSE for details.
+Copyright (c) 2011 Brighter Planet. See LICENSE for details.

data/Rakefile

@@ -1,72 +1,23 @@
-require 'rubygems'
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "remote_table"
-    gem.summary = %Q{Remotely open and parse XLS, ODS, CSV and fixed-width tables.}
-    gem.description = %Q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
-    gem.email = "seamus@abshere.net"
-    gem.homepage = "http://github.com/seamusabshere/remote_table"
-    gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    # sabshere [unknown date] roo 1.9.3 doesn't work, so use old 1.3 version
-    gem.add_dependency 'roo', '1.3.11'
-    # sabshere 9/30/10 depending on fastercsv when using ruby 1.9.2 results in exiting with error
-    # gem.add_dependency 'fastercsv', '>=1.5.0'
-    gem.add_dependency 'activesupport', '>=2.3.4'
-    gem.add_dependency 'slither', '>=0.99.4'
-    gem.add_dependency 'nokogiri', '>=1.4.1'
-    gem.add_dependency 'escape', '>=0.0.4'
-    gem.add_development_dependency 'errata', '>=0.2.0'
-    gem.require_path = "lib"
-    gem.rdoc_options << '--line-numbers' << '--inline-source'
-    gem.requirements << 'curl'
-    gem.rubyforge_project = "remotetable"
-  end
-  Jeweler::GemcutterTasks.new
-  # Jeweler::RubyforgeTasks.new do |rubyforge|
-  #   rubyforge.doc_task = "rdoc"
-  # end
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
-end
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
+require 'rake'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/*_test.rb'
+  test.pattern = 'test/**/test_*.rb'
   test.verbose = true
 end
 
 begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/*_test.rb'
-    test.verbose = true
+  require 'rake/rdoctask'
+  Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = 'taps'
+    rdoc.options << '--line-numbers' << '--inline-source'
+    rdoc.rdoc_files.include('README*')
+    rdoc.rdoc_files.include('lib/**/*.rb')
   end
 rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
-
-
-
-task :default => :test
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  if File.exist?('VERSION')
-    version = File.read('VERSION')
-  else
-    version = ""
-  end
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "remote_table #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+   puts "Rdoc is not available"
 end

data/lib/remote_table/cleaner.rb

@@ -0,0 +1,19 @@
+require 'singleton'
+require 'fileutils'
+class RemoteTable
+  class Cleaner
+    include ::Singleton
+    def paths_for_removal
+      @paths_for_removal ||= []
+    end
+    def cleanup
+      paths_for_removal.each do |path|
+        ::FileUtils.rm_rf path
+        paths_for_removal.delete path
+      end
+    end
+    def remove_at_exit(path)
+      paths_for_removal << path
+    end
+  end
+end

data/lib/remote_table/executor.rb

@@ -0,0 +1,29 @@
+require 'singleton'
+require 'escape'
+require 'fileutils'
+class RemoteTable
+  class Executor
+    include ::Singleton
+    def bang(path, cmd)
+      tmp_path = "#{path}.bang.#{rand}"
+      backtick_with_reporting "/bin/cat #{::Escape.shell_single_word path} | #{cmd} > #{::Escape.shell_single_word tmp_path}"
+      ::FileUtils.mv tmp_path, path
+    end
+
+    def backtick_with_reporting(cmd)
+      cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
+      output = `#{cmd}`
+      if not $?.success?
+        raise %{
+From the remote_table gem...
+
+Command failed:
+#{cmd}
+
+Output:
+#{output}
+}
+      end
+    end
+  end
+end

data/lib/remote_table/format/delimited.rb

@@ -0,0 +1,62 @@
+if RUBY_VERSION >= '1.9'
+  require 'csv'
+  ::FasterCSV = ::CSV
+else
+  begin
+    require 'fastercsv'
+  rescue ::LoadError
+    $stderr.puts "[remote_table gem] You probably need to manually install the fastercsv gem and/or require it in your Gemfile."
+    raise $!
+  end
+end
+
+class RemoteTable
+  class Format
+    class Delimited < Format
+      include Textual
+      def each(&blk)
+        backup_file!
+        convert_file_to_utf8!
+        remove_useless_characters!
+        skip_rows!
+        ::FasterCSV.foreach(t.local_file.path, fastercsv_options) do |row|
+          ordered_hash = ::ActiveSupport::OrderedHash.new
+          filled_values = 0
+          case row
+          when ::FasterCSV::Row
+            row.each do |header, value|
+              next if header.blank?
+              value = '' if value.nil?
+              ordered_hash[header] = value
+              filled_values += 1 if value.present?
+            end
+          when ::Array
+            index = 0
+            row.each do |value|
+              value = '' if value.nil?
+              ordered_hash[index] = value
+              filled_values += 1 if value.present?
+              index += 1
+            end
+          end
+          yield ordered_hash if t.properties.keep_blank_rows or filled_values > 0
+        end
+      ensure
+        restore_file!
+      end
+
+      private
+
+      def fastercsv_options
+        fastercsv_options = { :skip_blanks => !t.properties.keep_blank_rows }
+        if t.properties.headers == false
+          fastercsv_options.merge!(:headers => nil)
+        else
+          fastercsv_options.merge!(:headers => :first_row)
+        end
+        fastercsv_options.merge!(:col_sep => t.properties.delimiter) if t.properties.delimiter
+        fastercsv_options
+      end
+    end
+  end
+end

data/lib/remote_table/format/excel.rb

@@ -0,0 +1,10 @@
+class RemoteTable
+  class Format
+    class Excel < Format
+      include Rooable
+      def roo_class
+        ::Excel
+      end
+    end
+  end
+end

data/lib/remote_table/format/excelx.rb

@@ -0,0 +1,10 @@
+class RemoteTable
+  class Format
+    class Excelx < Format
+      include Rooable
+      def roo_class
+        ::Excelx
+      end
+    end
+  end
+end

data/lib/remote_table/format/fixed_width.rb

@@ -0,0 +1,47 @@
+require 'slither'
+class RemoteTable
+  class Format
+    class FixedWidth < Format
+      include Textual
+      def each(&blk)
+        backup_file!
+        convert_file_to_utf8!
+        remove_useless_characters!
+        crop_rows!
+        skip_rows!
+        cut_columns!
+        parser.parse[:rows].each do |hash|
+          hash.reject! { |k, v| k.blank? }
+          yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
+        end
+      ensure
+        restore_file!
+      end
+      private
+      def parser
+        @parser ||= ::Slither::Parser.new definition, t.local_file.path
+      end
+      def definition
+        @definition ||= if t.properties.schema_name.is_a?(::String) or t.properties.schema_name.is_a?(::Symbol)
+          ::Slither.send :definition, t.properties.schema_name
+        elsif t.properties.schema.is_a?(::Array)
+          everything = lambda { |_| true }
+          ::Slither.define(rand.to_s) do |d|
+            d.rows do |row|
+              row.trap(&everything)
+              t.properties.schema.each do |name, width, options|
+                if name == 'spacer'
+                  row.spacer width
+                else
+                  row.column name, width, options
+                end
+              end
+            end
+          end
+        else
+          raise "expecting schema_name to be a String or Symbol, or schema to be an Array"
+        end
+      end
+    end
+  end
+end

data/lib/remote_table/format/html.rb

@@ -0,0 +1,43 @@
+require 'nokogiri'
+require 'cgi'
+class RemoteTable
+  class Format
+    class HTML < Format
+      include Textual
+      def each(&blk)
+        backup_file!
+        convert_file_to_utf8!
+        remove_useless_characters!
+        html_headers = (t.properties.headers.is_a?(::Array)) ? t.properties.headers : nil
+        ::Nokogiri::HTML(unescaped_html_without_soft_hyphens, nil, 'UTF-8').xpath(t.properties.row_xpath).each do |row|
+          values = row.xpath(t.properties.column_xpath).map { |td| td.content.gsub(/\s+/, ' ').strip }
+          if html_headers.nil?
+            html_headers = values
+            next
+          end
+          hash = zip html_headers, values
+          yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
+        end
+      ensure
+        restore_file!
+      end
+
+      private
+
+      # http://snippets.dzone.com/posts/show/406
+      def zip(keys, values)
+        hash = ::Hash.new
+        keys.zip(values) { |k,v| hash[k]=v }
+        hash
+      end
+
+      # should we be doing this in ruby?
+      def unescaped_html_without_soft_hyphens
+        str = ::CGI.unescapeHTML ::IO.read(t.local_file.path)
+        # get rid of MS Office baddies
+        str.gsub! /&shy;/, ''
+        str
+      end
+    end
+  end
+end

data/lib/remote_table/format/mixins/rooable.rb

@@ -0,0 +1,47 @@
+require 'roo'
+class RemoteTable
+  class Format
+    module Rooable
+      def each(&blk)
+        spreadsheet = roo_class.new t.local_file.path, nil, :ignore
+        spreadsheet.default_sheet = t.properties.sheet.is_a?(::Numeric) ? spreadsheet.sheets[t.properties.sheet] : t.properties.sheet
+        column_references = ::Hash.new
+        if t.properties.headers == false
+          # zero-based numeric keys
+          for col in (1..spreadsheet.last_column)
+            column_references[col] = col - 1
+          end
+        elsif t.properties.headers.is_a? ::Array
+          # names
+          for col in (1..spreadsheet.last_column)
+            column_references[col] = t.properties.headers[col - 1]
+          end
+        else
+          # read t.properties.headers from the file itself
+          for col in (1..spreadsheet.last_column)
+            column_references[col] = spreadsheet.cell(header_row, col)
+            column_references[col] = spreadsheet.cell(header_row - 1, col) if column_references[col].blank? # lspreadsheetk up
+          end
+        end
+        first_data_row.upto(spreadsheet.last_row) do |raw_row|
+          ordered_hash = ::ActiveSupport::OrderedHash.new
+          for col in (1..spreadsheet.last_column)
+            next if column_references[col].blank?
+            ordered_hash[column_references[col]] = spreadsheet.cell(raw_row, col).to_s.gsub(/<[^>]+>/, '').strip
+          end
+          yield ordered_hash if t.properties.keep_blank_rows or ordered_hash.any? { |k, v| v.present? }
+        end
+      end
+
+      private
+
+      def header_row
+        1 + t.properties.skip
+      end
+
+      def first_data_row
+        1 + header_row
+      end
+    end
+  end
+end

data/lib/remote_table/format/mixins/textual.rb

@@ -0,0 +1,34 @@
+require 'fileutils'
+require 'escape'
+class RemoteTable
+  class Format
+    module Textual
+      def convert_file_to_utf8!
+        ::RemoteTable.executor.bang t.local_file.path, "iconv -c -f #{::Escape.shell_single_word t.properties.encoding} -t UTF-8"
+      end
+      
+      USELESS_CHARACTERS = [
+        '\xef\xbb\xbf',   # UTF-8 byte order mark
+        '\xc2\xad'        # soft hyphen, often inserted by MS Office (html: ­)
+      ]
+      def remove_useless_characters!
+        ::RemoteTable.executor.bang t.local_file.path, "perl -pe 's/#{USELESS_CHARACTERS.join '//g; s/'}//g'"
+      end
+      
+      def skip_rows!
+        return unless t.properties.skip > 0
+        ::RemoteTable.executor.bang t.local_file.path, "tail -n +#{t.properties.skip + 1}"
+      end
+      
+      def crop_rows!
+        return unless t.properties.crop
+        ::RemoteTable.executor.bang t.local_file.path, "tail -n +#{::Escape.shell_single_word t.properties.crop.first.to_s} | head -n #{t.properties.crop.last - t.properties.crop.first + 1}"
+      end
+      
+      def cut_columns!
+        return unless t.properties.cut
+        ::RemoteTable.executor.bang t.local_file.path, "cut -c #{::Escape.shell_single_word t.properties.cut.to_s}"
+      end
+    end
+  end
+end

data/lib/remote_table/format/open_office.rb

@@ -0,0 +1,10 @@
+class RemoteTable
+  class Format
+    class OpenOffice < Format
+      include Rooable
+      def roo_class
+        ::Openoffice
+      end
+    end
+  end
+end

data/lib/remote_table/format.rb

@@ -0,0 +1,35 @@
+class RemoteTable  
+  class Format
+    class Unknown < StandardError; end
+    
+    autoload :Excel, 'remote_table/format/excel'
+    autoload :Excelx, 'remote_table/format/excelx'
+    autoload :Delimited, 'remote_table/format/delimited'
+    autoload :OpenOffice, 'remote_table/format/open_office'
+    autoload :FixedWidth, 'remote_table/format/fixed_width'
+    autoload :HTML, 'remote_table/format/html'
+    
+    autoload :Textual, 'remote_table/format/mixins/textual'
+    autoload :Rooable, 'remote_table/format/mixins/rooable'
+    
+    attr_reader :t
+
+    def initialize(t)
+      @t = t
+    end
+    
+    include ::Enumerable
+    def each
+      raise "must be defined by format"
+    end
+    
+    def backup_file!
+      ::FileUtils.cp t.local_file.path, "#{t.local_file.path}.backup"
+    end
+    
+    def restore_file!
+      return unless ::File.readable? "#{t.local_file.path}.backup"
+      ::FileUtils.mv "#{t.local_file.path}.backup", t.local_file.path
+    end
+  end
+end

data/lib/remote_table/hasher.rb

@@ -0,0 +1,25 @@
+require 'singleton'
+require 'digest/md5'
+class RemoteTable
+  class Hasher
+    include ::Singleton
+    def hash(row)
+      normalized_hash = if RUBY_VERSION >= '1.9'
+        row.keys.sort.inject(::Hash.new) do |memo, k|
+          normalized_k = k.to_s.toutf8
+          normalized_v = row[k].respond_to?(:to_s) ? row[k].to_s.toutf8 : row[k]
+          memo[normalized_k] = normalized_v
+          memo
+        end
+      else
+        ::Hash.new.replace(row)
+      end
+      # sabshere 1/21/11 may currently break across versions of ruby
+      # ruby-1.8.7-p174 > Marshal.dump({'a' => '1'})
+      #  => "\004\b{\006\"\006a\"\0061" 
+      # ruby-1.9.2-p0 > Marshal.dump({'a' => '1'})
+      # => "\x04\b{\x06I\"\x06a\x06:\x06ETI\"\x061\x06;\x00T"
+      ::Digest::MD5.hexdigest ::Marshal.dump(normalized_hash)
+    end
+  end
+end