remote_table 1.3.0 → 1.4.0

data/lib/remote_table/format/delimited.rb

@@ -1,37 +1,33 @@
-if RUBY_VERSION >= '1.9'
-  require 'csv'
-  ::RemoteTable::MyCSV = ::CSV
-else
-  begin
-    require 'fastercsv'
-    ::RemoteTable::MyCSV = ::FasterCSV
-  rescue ::LoadError
-    $stderr.puts "[remote_table] You probably need to manually install the fastercsv gem and/or require it in your Gemfile."
-    raise $!
-  end
-end
-
 class RemoteTable
   class Format
     class Delimited < Format
+      if ::RUBY_VERSION >= '1.9'
+        require 'csv'
+        Engine = ::CSV
+      else
+        require 'fastercsv'
+        Engine = ::FasterCSV
+      end
+
       include Textual
+
       def each(&blk)
         remove_useless_characters!
         fix_newlines!
         transliterate_whole_file_to_utf8!
         skip_rows!
-        MyCSV.new(t.local_file.encoded_io, fastercsv_options).each do |row|
-          if row.is_a?(MyCSV::Row)
+        Engine.new(t.local_file.encoded_io, fastercsv_options).each do |row|
+          if row.is_a?(Engine::Row)
             hash = row.inject(::ActiveSupport::OrderedHash.new) do |memo, (k, v)|
               if k.present?
                 memo[k] = v.to_s
               end
               memo
             end
-            yield hash if t.properties.keep_blank_rows or hash.any? { |k, v| v.present? }
+            yield hash if t.config.keep_blank_rows or hash.any? { |k, v| v.present? }
           elsif row.is_a?(::Array)
             array = row.map { |v| v.to_s }
-            yield array if t.properties.keep_blank_rows or array.any? { |v| v.present? }
+            yield array if t.config.keep_blank_rows or array.any? { |v| v.present? }
           end
         end
       ensure
@@ -54,10 +50,10 @@ class RemoteTable
       ]
 
       def fastercsv_options
-        hsh = t.options.slice *FASTERCSV_OPTIONS
-        hsh.merge! :skip_blanks => !t.properties.keep_blank_rows
-        hsh.reverse_merge! :headers => t.properties.headers
-        hsh.reverse_merge! :col_sep => t.properties.delimiter
+        hsh = t.config.user_specified_options.slice *FASTERCSV_OPTIONS
+        hsh[:skip_blanks] = !t.config.keep_blank_rows
+        hsh.reverse_merge! :headers => t.config.headers
+        hsh.reverse_merge! :col_sep => t.config.delimiter
       end
     end
   end

data/lib/remote_table/format/fixed_width.rb

@@ -1,10 +1,10 @@
-require 'fixed_width'
-
 class RemoteTable
   class Format
     class FixedWidth < Format
       include Textual
       def each(&blk)
+        require 'fixed_width-multibyte'
+
         remove_useless_characters!
         fix_newlines!
         transliterate_whole_file_to_utf8!
@@ -16,7 +16,7 @@ class RemoteTable
           row.each do |k, v|
             row[k] = v.strip
           end
-          yield row if t.properties.keep_blank_rows or row.any? { |k, v| v.present? }
+          yield row if t.config.keep_blank_rows or row.any? { |k, v| v.present? }
         end
       ensure
         t.local_file.cleanup
@@ -27,21 +27,21 @@ class RemoteTable
       def parser
         return @parser if @parser.is_a?(::FixedWidth::Parser)
         if ::FixedWidth::Section.private_instance_methods.map(&:to_sym).include?(:unpacker)
-          raise "[remote_table] You need a different (newer) version of the FixedWidth gem that supports multibyte encoding, sometime after https://github.com/timonk/fixed_width/pull/1 was incorporated"
+          raise ::RuntimeError, "You need a different (newer) version of the FixedWidth gem that supports multibyte encoding, sometime after https://github.com/timonk/fixed_width/pull/1 was incorporated"
         end
         @parser = ::FixedWidth::Parser.new definition, t.local_file.encoded_io
       end
       
       def definition
-        @definition ||= if t.properties.schema_name.is_a?(::String) or t.properties.schema_name.is_a?(::Symbol)
-          ::FixedWidth.send :definition, t.properties.schema_name
-        elsif t.properties.schema.is_a?(::Array)
+        @definition ||= if t.config.schema_name.is_a?(::String) or t.config.schema_name.is_a?(::Symbol)
+          ::FixedWidth.send :definition, t.config.schema_name
+        elsif t.config.schema.is_a?(::Array)
           everything = lambda { |_| true }
           srand # in case this was forked by resque
           ::FixedWidth.define(rand.to_s) do |d|
             d.rows do |row|
               row.trap(&everything)
-              t.properties.schema.each do |name, width, options|
+              t.config.schema.each do |name, width, options|
                 name = name.to_s
                 if name == 'spacer'
                   row.spacer width
@@ -52,7 +52,7 @@ class RemoteTable
             end
           end
         else
-          raise "expecting schema_name to be a String or Symbol, or schema to be an Array"
+          raise ::ArgumentError, "expecting schema_name to be a String or Symbol, or schema to be an Array"
         end
       end
     end

data/lib/remote_table/format/html.rb

@@ -1,5 +1,3 @@
-require 'nokogiri'
-require 'cgi'
 class RemoteTable
   class Format
     class HTML < Format

data/lib/remote_table/format/mixins/processed_by_nokogiri.rb

@@ -1,21 +1,22 @@
-require 'nokogiri'
-require 'cgi'
 class RemoteTable
   class Format
     module ProcessedByNokogiri
       def each
-        raise "[remote_table] Need :row_css or :row_xpath in order to process XML or HTML" unless t.properties.row_css or t.properties.row_xpath
+        require 'nokogiri'
+        require 'cgi'
+        
+        raise ::ArgumentError, "Need :row_css or :row_xpath in order to process XML or HTML" unless t.config.row_css or t.config.row_xpath
         remove_useless_characters!
         transliterate_whole_file_to_utf8!
         
-        headers = t.properties.headers
+        headers = t.config.headers
 
         xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, 'UTF-8')
-        (row_css? ? xml.css(t.properties.row_css) : xml.xpath(t.properties.row_xpath)).each do |row|
+        (row_css? ? xml.css(t.config.row_css) : xml.xpath(t.config.row_xpath)).each do |row|
           values = if column_css?
-            row.css(t.properties.column_css)
+            row.css(t.config.column_css)
           elsif column_xpath?
-            row.xpath(t.properties.column_xpath)
+            row.xpath(t.config.column_xpath)
           else
             [row]
           end.map { |cell| assume_utf8 cell.content.gsub(/\s+/, ' ').strip }
@@ -23,12 +24,12 @@ class RemoteTable
             headers = values.select(&:present?)
             next
           end
-          output = if t.properties.output_class == ::Array
+          output = if t.config.output_class == ::Array
             values
           else
             zip headers, values
           end
-          if t.properties.keep_blank_rows or values.any?
+          if t.config.keep_blank_rows or values.any?
             yield output
           end
         end
@@ -39,15 +40,15 @@ class RemoteTable
       private
 
       def row_css?
-        !!t.properties.row_css
+        !!t.config.row_css
       end
       
       def column_css?
-        !!t.properties.column_css
+        !!t.config.column_css
       end
       
       def column_xpath?
-        !!t.properties.column_xpath
+        !!t.config.column_xpath
       end
       
       # http://snippets.dzone.com/posts/show/406

data/lib/remote_table/format/mixins/processed_by_roo.rb

@@ -1,33 +1,37 @@
-require 'roo'
 class RemoteTable
   class Format
     module ProcessedByRoo
       def each(&blk)
+        require 'iconv'
+        require 'roo'
+
         spreadsheet = roo_class.new t.local_file.path, nil, :ignore
-        spreadsheet.default_sheet = t.properties.sheet.is_a?(::Numeric) ? spreadsheet.sheets[t.properties.sheet] : t.properties.sheet
+        if t.config.sheet
+          spreadsheet.default_sheet = t.config.sheet
+        end
         
-        first_row = if t.properties.crop
-          t.properties.crop.first + 1
+        first_row = if t.config.crop
+          t.config.crop.first + 1
         else
-          t.properties.skip + 1
+          t.config.skip + 1
         end
           
-        last_row = if t.properties.crop
-          t.properties.crop.last
+        last_row = if t.config.crop
+          t.config.crop.last
         else
           spreadsheet.last_row
         end
         
-        if t.properties.output_class == ::Array
+        if t.config.output_class == ::Array
           (first_row..last_row).each do |y|
             output = (1..spreadsheet.last_column).map do |x|
               assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
             end
-            yield output if t.properties.keep_blank_rows or output.any? { |v| v.present? }
+            yield output if t.config.keep_blank_rows or output.any? { |v| v.present? }
           end
         else
-          headers = {}
-          if t.properties.use_first_row_as_header?
+          headers = ::ActiveSupport::OrderedHash.new
+          if t.config.use_first_row_as_header?
             (1..spreadsheet.last_column).each do |x|
               v = spreadsheet.cell(first_row, x)
               v = spreadsheet.cell(first_row - 1, x) if v.blank? # look up
@@ -39,7 +43,7 @@ class RemoteTable
             # "advance the cursor"
             first_row += 1
           else
-            t.properties.headers.each_with_index do |k, i|
+            t.config.headers.each_with_index do |k, i|
               headers[k] = i + 1
             end
           end
@@ -48,7 +52,7 @@ class RemoteTable
             headers.each do |k, x|
               output[k] = assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
             end
-            yield output if t.properties.keep_blank_rows or output.any? { |k, v| v.present? }
+            yield output if t.config.keep_blank_rows or output.any? { |k, v| v.present? }
           end
         end
       ensure

data/lib/remote_table/format/mixins/textual.rb

@@ -7,36 +7,36 @@ class RemoteTable
         '\xc2\xad',       # soft hyphen, often inserted by MS Office (html: ­)
       ]
       def remove_useless_characters!
-        Utils.in_place t.local_file.path, 'perl', '-pe', "s/#{USELESS_CHARACTERS.join '//g; s/'}//g"
-        if t.properties.internal_encoding =~ /windows.?1252/i
+        t.local_file.in_place :perl, "s/#{USELESS_CHARACTERS.join('//g; s/')}//g"
+        if t.config.internal_encoding =~ /windows.?1252/i
           # soft hyphen again, as I have seen it appear in windows 1252
-          Utils.in_place t.local_file.path, 'perl', '-pe', 's/\xad//g'
+          t.local_file.in_place :perl, 's/\xad//g'
         end
       end
       
       def transliterate_whole_file_to_utf8!
-        Utils.in_place t.local_file.path, 'iconv', '-c', '-f', t.properties.internal_encoding, '-t', t.properties.external_encoding_iconv, :ignore_error => true
-        t.properties.update :encoding => t.properties.external_encoding
+        t.local_file.in_place :iconv, t.config.external_encoding_iconv, t.config.internal_encoding
+        t.config.user_specified_options[:encoding] = t.config.external_encoding
       end
       
       def fix_newlines!
-        Utils.in_place t.local_file.path, 'perl', '-pe', 's/\r\n|\n|\r/\n/g'
+        t.local_file.in_place :perl, 's/\r\n|\n|\r/\n/g'
       end
       
       def skip_rows!
-        return unless t.properties.skip > 0
-        Utils.in_place t.local_file.path, 'tail', '-n', "+#{t.properties.skip + 1}"
+        return unless t.config.skip > 0
+        t.local_file.in_place :tail, "+#{t.config.skip + 1}"
       end
       
       def crop_rows!
-        return unless t.properties.crop
-        Utils.in_place t.local_file.path, 'tail', '-n', "+#{t.properties.crop.first}"
-        Utils.in_place t.local_file.path, 'head', '-n', (t.properties.crop.last - t.properties.crop.first + 1).to_s
+        return unless t.config.crop
+        t.local_file.in_place :tail, "+#{t.config.crop.first}"
+        t.local_file.in_place :head, (t.config.crop.last - t.config.crop.first + 1)
       end
       
       def cut_columns!
-        return unless t.properties.cut
-        Utils.in_place t.local_file.path, 'cut', '-c', t.properties.cut.to_s
+        return unless t.config.cut
+        t.local_file.in_place :cut, t.config.cut
       end
     end
   end

data/lib/remote_table/format/open_office.rb

@@ -3,6 +3,9 @@ class RemoteTable
     class OpenOffice < Format
       include ProcessedByRoo
       def roo_class
+        if ::RUBY_PLATFORM == 'java'
+          ::Kernel.warn "[remote_table] Opening ODS files on JRuby is known to fail because of a flaw in the underlying Roo library"
+        end
         ::Openoffice
       end
     end

data/lib/remote_table/format/xml.rb

@@ -1,5 +1,3 @@
-require 'nokogiri'
-require 'cgi'
 class RemoteTable
   class Format
     class XML < Format

data/lib/remote_table/format/yaml.rb

@@ -0,0 +1,14 @@
+require 'yaml'
+
+class RemoteTable
+  class Format
+    class Yaml < Format
+      def each(&blk)
+        data = YAML.load_file t.local_file.path
+        data.each &blk
+      ensure
+        t.local_file.cleanup
+      end
+    end
+  end
+end

data/lib/remote_table/local_file.rb

@@ -1,13 +1,71 @@
 require 'fileutils'
+require 'unix_utils'
 
 class RemoteTable
   class LocalFile #:nodoc:all
+    class << self
+      def decompress(input, compression)
+        output = case compression
+        when :zip, :exe
+          ::UnixUtils.unzip input
+        when :bz2
+          ::UnixUtils.bunzip2 input
+        when :gz
+          ::UnixUtils.gunzip input
+        else
+          raise ::ArgumentError, "Unrecognized compression #{compression}"
+        end
+        ::FileUtils.rm_f input
+        output
+      end
+      
+      def unpack(input, packing)
+        output = case packing
+        when :tar
+          ::UnixUtils.untar input
+        else
+          raise ::ArgumentError, "Unrecognized packing #{packing}"
+        end
+        ::FileUtils.rm_f input
+        output
+      end
+      
+      def pick(input, options = {})
+        options = options.symbolize_keys
+        if (options[:filename] or options[:glob]) and not ::File.directory?(input)
+          raise ::RuntimeError, "Expecting #{input} to be a directory"
+        end
+        if filename = options[:filename]
+          src = ::File.join input, filename
+          raise(::RuntimeError, "Expecting #{src} to be a file") unless ::File.file?(src)
+          output = ::UnixUtils.tmp_path src
+          ::FileUtils.mv src, output
+          ::FileUtils.rm_rf input if ::File.dirname(input).start_with?(::Dir.tmpdir)
+        elsif glob = options[:glob]
+          src = ::Dir[input+glob].first
+          raise(::RuntimeError, "Expecting #{glob} to find a file in #{input}") unless src and ::File.file?(src)
+          output = ::UnixUtils.tmp_path src
+          ::FileUtils.mv src, output
+          ::FileUtils.rm_rf input if ::File.dirname(input).start_with?(::Dir.tmpdir)
+        else
+          output = ::UnixUtils.tmp_path input
+          ::FileUtils.mv input, output
+        end
+        output
+      end
+    end
     
     attr_reader :t
     
     def initialize(t)
       @t = t
     end
+
+    def in_place(*args)
+      bin = args.shift
+      tmp_path = ::UnixUtils.send(*([bin,path]+args))
+      ::FileUtils.mv tmp_path, path
+    end
     
     def path
       generate unless generated?
@@ -16,7 +74,7 @@ class RemoteTable
     
     def encoded_io
       @encoded_io ||= if ::RUBY_VERSION >= '1.9'
-        ::File.open path, 'rb', :internal_encoding => t.properties.internal_encoding, :external_encoding => t.properties.external_encoding
+        ::File.open path, 'rb', :internal_encoding => t.config.internal_encoding, :external_encoding => t.config.external_encoding
       else
         ::File.open path, 'rb'
       end
@@ -41,14 +99,18 @@ class RemoteTable
     end
         
     def generate
-      tmp_path = Utils.download t.properties.uri, t.properties.form_data
-      if compression = t.properties.compression
-        tmp_path = Utils.decompress tmp_path, compression
+      # sabshere 7/20/11 make web requests move more slowly so you don't get accused of DOS
+      if ::ENV.has_key?('REMOTE_TABLE_DELAY_BETWEEN_REQUESTS')
+        ::Kernel.sleep ::ENV['REMOTE_TABLE_DELAY_BETWEEN_REQUESTS'].to_i
+      end
+      tmp_path = ::UnixUtils.curl t.config.uri.to_s, t.config.form_data
+      if compression = t.config.compression
+        tmp_path = LocalFile.decompress tmp_path, compression
       end
-      if packing = t.properties.packing
-        tmp_path = Utils.unpack tmp_path, packing
+      if packing = t.config.packing
+        tmp_path = LocalFile.unpack tmp_path, packing
       end
-      @path = Utils.pick tmp_path, :filename => t.properties.filename, :glob => t.properties.glob
+      @path = LocalFile.pick tmp_path, :filename => t.config.filename, :glob => t.config.glob
       @generated = true
     end
   end