remote_table 1.4.0 → 2.0.0

data/lib/remote_table/delimited.rb

@@ -0,0 +1,91 @@
+class RemoteTable
+  # Parses plaintext comma-separated (CSV), tab-separated (TSV), or really anything-delimited files using Ruby's CSV parser.
+  module Delimited
+    # Delimited uses Plaintext.
+    def self.extended(base)
+      base.extend Plaintext
+    end
+
+    if ::RUBY_VERSION >= '1.9'
+      require 'csv'
+      Engine = ::CSV
+    else
+      require 'fastercsv'
+      Engine = ::FasterCSV
+    end
+
+    PASSTHROUGH_CSV_SETTINGS = [
+      :unconverted_fields,
+      :col_sep,
+      :row_sep,
+      :return_headers,
+      :header_converters,
+      :quote_char,
+      :converters,
+      :force_quotes,
+    ]
+
+    # Yield each row using Ruby's CSV parser (FasterCSV on Ruby 1.8).
+    def _each
+      delete_harmful!
+      convert_eol_to_unix!
+      transliterate_whole_file_to_utf8!
+      skip_rows!
+
+      Engine.new(local_copy.encoded_io, csv_options).each do |row|
+
+        some_value_present = false
+
+        if not headers
+
+          # represent the row as an array
+          array = row.map do |v|
+            v = v.to_s
+            if not some_value_present and not keep_blank_rows and v.present?
+              some_value_present = true
+            end
+            v
+          end
+          if some_value_present or keep_blank_rows
+            yield array
+          end
+
+        else
+
+          # represent the row as a hash
+          hash = ::ActiveSupport::OrderedHash.new
+          row.each do |k, v|
+            next unless k.present?
+            v = v.to_s
+            if not some_value_present and not keep_blank_rows and v.present?
+              some_value_present = true
+            end
+            hash[k] = v
+          end
+          if some_value_present or keep_blank_rows
+            yield hash
+          end
+
+        end
+      end
+    ensure
+      local_copy.cleanup
+    end
+
+    # Passes user-specified options in PASSTHROUGH_CSV_SETTINGS.
+    #
+    # Also maps:
+    # * +:headers+ directly
+    # * +:keep_blank_rows+ to the CSV option +:skip_blanks+
+    # * +:delimiter+ to the CSV option +:col_sep+
+    #
+    # @return [Hash]
+    def csv_options
+      memo = other_options.slice(*PASSTHROUGH_CSV_SETTINGS)
+      memo[:skip_blanks] = !keep_blank_rows
+      memo[:headers] ||= headers
+      memo[:col_sep] ||= delimiter
+      memo
+    end
+  end
+end

data/lib/remote_table/fixed_width.rb

@@ -0,0 +1,81 @@
+class RemoteTable
+  # Parses plaintext fixed-width files using https://github.com/seamusabshere/fixed_width
+  module FixedWidth
+    def self.extended(base)
+      base.extend Plaintext
+    end
+
+    TRAP_EVERYTHING = proc { |_| true }
+
+    # @private
+    def after_extend
+      @parser_mutex = ::Mutex.new
+      @definition_mutex = ::Mutex.new
+    end
+
+    def _each
+      require 'fixed_width-multibyte'
+
+      delete_harmful!
+      convert_eol_to_unix!
+      transliterate_whole_file_to_utf8!
+      crop_rows!
+      skip_rows!
+      cut_columns!
+
+      parser.parse[:rows].each do |row|
+        some_value_present = false
+        hash = ::ActiveSupport::OrderedHash.new
+        row.each do |k, v|
+          v = v.to_s.strip
+          if not some_value_present and not keep_blank_rows and v.present?
+            some_value_present = true
+          end
+          hash[k] = v
+        end
+        if some_value_present or keep_blank_rows
+          yield hash
+        end
+      end
+    ensure
+      local_copy.cleanup
+    end
+
+    private
+    
+    def parser
+      @parser || @parser_mutex.synchronize do
+        @parser ||= begin
+          if ::FixedWidth::Section.private_instance_methods.map(&:to_sym).include?(:unpacker)
+            raise ::RuntimeError, "[remote_table] You need to use exclusively the fixed_width-multibyte library https://github.com/seamusabshere/fixed_width"
+          end
+          ::FixedWidth::Parser.new definition, local_copy.encoded_io
+        end
+      end
+    end
+    
+    def definition
+      @definition || @definition_mutex.synchronize do
+        @definition ||= if schema_name.is_a?(::String) or schema_name.is_a?(::Symbol)
+          ::FixedWidth.send :definition, schema_name
+        elsif schema.is_a?(::Array)
+          ::FixedWidth.define("remote_table-fixed_with-#{::Kernel.rand}") do |d|
+            d.rows do |row|
+              row.trap(&TRAP_EVERYTHING)
+              schema.each do |name, width, options|
+                name = name.to_s
+                if name == 'spacer'
+                  row.spacer width
+                else
+                  row.column name, width, options
+                end
+              end
+            end
+          end
+        else
+          raise ::ArgumentError, "[remote_table] Expecting :schema_name to be a String or Symbol, or :schema to be an Array"
+        end
+      end
+    end
+  end
+end

data/lib/remote_table/html.rb

@@ -0,0 +1,13 @@
+class RemoteTable
+  # Parses [X]HTML files using Nokogiri's Nokogiri::HTML::Document class.
+  module Html
+    def self.extended(base)
+      base.extend Plaintext
+      base.extend ProcessedByNokogiri
+    end
+    
+    def nokogiri_class
+      ::Nokogiri::HTML::Document
+    end
+  end
+end

data/lib/remote_table/{local_file.rb → local_copy.rb}

@@ -2,7 +2,7 @@ require 'fileutils'
 require 'unix_utils'
 
 class RemoteTable
-  class LocalFile #:nodoc:all
+  class LocalCopy #:nodoc:all
     class << self
       def decompress(input, compression)
         output = case compression
@@ -59,6 +59,8 @@ class RemoteTable
     
     def initialize(t)
       @t = t
+      @encoded_io_mutex = ::Mutex.new
+      @generate_mutex = ::Mutex.new
     end
 
     def in_place(*args)
@@ -68,15 +70,17 @@ class RemoteTable
     end
     
     def path
-      generate unless generated?
+      generate unless @generated
       @path
     end
     
     def encoded_io
-      @encoded_io ||= if ::RUBY_VERSION >= '1.9'
-        ::File.open path, 'rb', :internal_encoding => t.config.internal_encoding, :external_encoding => t.config.external_encoding
-      else
-        ::File.open path, 'rb'
+      @encoded_io || @encoded_io_mutex.synchronize do
+        @encoded_io ||= if ::RUBY_VERSION >= '1.9'
+          ::File.open path, 'rb', :internal_encoding => t.internal_encoding, :external_encoding => RemoteTable::EXTERNAL_ENCODING
+        else
+          ::File.open path, 'rb'
+        end
       end
     end
     
@@ -94,24 +98,24 @@ class RemoteTable
     
     private
     
-    def generated?
-      @generated == true
-    end
-        
     def generate
-      # sabshere 7/20/11 make web requests move more slowly so you don't get accused of DOS
-      if ::ENV.has_key?('REMOTE_TABLE_DELAY_BETWEEN_REQUESTS')
-        ::Kernel.sleep ::ENV['REMOTE_TABLE_DELAY_BETWEEN_REQUESTS'].to_i
-      end
-      tmp_path = ::UnixUtils.curl t.config.uri.to_s, t.config.form_data
-      if compression = t.config.compression
-        tmp_path = LocalFile.decompress tmp_path, compression
-      end
-      if packing = t.config.packing
-        tmp_path = LocalFile.unpack tmp_path, packing
+      return if @generated
+      @generate_mutex.synchronize do
+        return if @generated
+        @generated = true
+        # sabshere 7/20/11 make web requests move more slowly so you don't get accused of DOS
+        if ::ENV.has_key?('REMOTE_TABLE_DELAY_BETWEEN_REQUESTS')
+          ::Kernel.sleep ::ENV['REMOTE_TABLE_DELAY_BETWEEN_REQUESTS'].to_i
+        end
+        tmp_path = ::UnixUtils.curl t.url, t.form_data
+        if compression = t.compression
+          tmp_path = LocalCopy.decompress tmp_path, compression
+        end
+        if packing = t.packing
+          tmp_path = LocalCopy.unpack tmp_path, packing
+        end
+        @path = LocalCopy.pick tmp_path, :filename => t.filename, :glob => t.glob
       end
-      @path = LocalFile.pick tmp_path, :filename => t.config.filename, :glob => t.config.glob
-      @generated = true
     end
   end
 end

data/lib/remote_table/ods.rb

@@ -0,0 +1,17 @@
+class RemoteTable
+  # Parses ODS files using Roo's Openoffice class.
+  #
+  # Know to have issues on JRuby.
+  module Ods
+    def self.extended(base)
+      base.extend ProcessedByRoo
+    end
+
+    def roo_class
+      if ::RUBY_PLATFORM == 'java'
+        ::Kernel.warn "[remote_table] Opening ODS files on JRuby is known to fail because of a flaw in the underlying Roo library"
+      end
+      ::Openoffice
+    end
+  end
+end

data/lib/remote_table/plaintext.rb

@@ -0,0 +1,67 @@
+require 'fileutils'
+
+class RemoteTable
+  # Helper methods that act on plaintext files before they are parsed
+  module Plaintext      
+    CONSIDERED_HARMFUL = [
+      '\xef\xbb\xbf',   # UTF-8 byte order mark
+      '\xc2\xad',       # soft hyphen, often inserted by MS Office (html: ­)
+      '\xad'            # any remaining soft hyphens (sometimes seen in windows-1252)
+    ]
+    EOL_TO_UNIX = 's/\r\n|\n|\r/\n/g'
+
+    # Remove bytes that are both useless and harmful in the vast majority of cases.
+    def delete_harmful!
+      local_copy.in_place :perl, "s/#{CONSIDERED_HARMFUL.join('//g; s/')}//g"
+    end
+    
+    # No matter what the file encoding is SUPPOSED to be, run it through iconv to make sure it's UTF-8
+    #
+    # @example
+    #   iconv -c -t UTF-8//TRANSLIT -f WINDOWS-1252
+    def transliterate_whole_file_to_utf8!
+      local_copy.in_place :iconv, RemoteTable::EXTERNAL_ENCODING_ICONV, internal_encoding
+      # now that we've force-transliterated to UTF-8, act as though this is what the user had specified
+      @internal_encoding = RemoteTable::EXTERNAL_ENCODING
+    end
+    
+    # No matter what the EOL are SUPPOSED to be, run it through Perl with a regex that will convert all EOLS to \n
+    #
+    # @example
+    #   perl -pe 's/\r\n|\n|\r/\n/g'
+    def convert_eol_to_unix!
+      local_copy.in_place :perl, EOL_TO_UNIX
+    end
+    
+    # If the user has specified :skip, use tail
+    #
+    # @example :skip => 6
+    #   tail +7
+    def skip_rows!
+      if skip > 0
+        local_copy.in_place :tail, "+#{skip + 1}"
+      end
+    end
+    
+    # If the user has specified :crop, use a combination of tail and head
+    #
+    # @example :crop => (184..263)
+    #   tail +184 | head 80
+    def crop_rows!
+      if crop
+        local_copy.in_place :tail, "+#{crop.first}"
+        local_copy.in_place :head, (crop.last - crop.first + 1)
+      end
+    end
+    
+    # If the user has specified :cut, use cut
+    #
+    # @example :cut => '13-'
+    #   cut -c 13-
+    def cut_columns!
+      if cut
+        local_copy.in_place :cut, cut
+      end
+    end
+  end
+end

data/lib/remote_table/processed_by_nokogiri.rb

@@ -0,0 +1,76 @@
+class RemoteTable
+  # Mixed in to process XML and XHTML.
+  module ProcessedByNokogiri
+    WHITESPACE = /\s+/
+    SINGLE_SPACE = ' '
+    SOFT_HYPHEN = '­'
+
+    # Yield each row using Nokogiri.
+    def _each
+      require 'nokogiri'
+      require 'cgi'
+      
+      # save this to a local var because we modify it in the loop
+      current_headers = headers
+
+      unless row_css or row_xpath
+        raise ::ArgumentError, "[remote_table] Need :row_css or :row_xpath in order to process XML or HTML"
+      end
+
+      delete_harmful!
+      transliterate_whole_file_to_utf8!
+      
+      xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, RemoteTable::EXTERNAL_ENCODING)
+      (row_css ? xml.css(row_css) : xml.xpath(row_xpath)).each do |row|
+        some_value_present = false
+        values = if column_css
+          row.css column_css
+        elsif column_xpath
+          row.xpath column_xpath
+        else
+          [row]
+        end.map do |cell|
+          memo = cell.content.dup
+          memo = assume_utf8 memo
+          memo.gsub! WHITESPACE, SINGLE_SPACE
+          memo.strip!
+          if not some_value_present and not keep_blank_rows and memo.present?
+            some_value_present = true
+          end
+          memo
+        end
+        if current_headers == :first_row
+          current_headers = values.select(&:present?)
+          next
+        end
+        if keep_blank_rows or some_value_present
+          if not headers
+            yield values
+          else
+            yield zip(current_headers, values)
+          end
+        end
+      end
+    ensure
+      local_copy.cleanup
+    end
+
+    private
+    
+    # http://snippets.dzone.com/posts/show/406
+    def zip(keys, values)
+      hash = ::ActiveSupport::OrderedHash.new
+      keys.zip(values) { |k,v| hash[k]=v }
+      hash
+    end
+
+    # should we be doing this in ruby?
+    def unescaped_xml_without_soft_hyphens
+      str = ::CGI.unescapeHTML local_copy.encoded_io.read
+      local_copy.encoded_io.rewind
+      # get rid of MS Office baddies
+      str.gsub! SOFT_HYPHEN, ''
+      str
+    end
+  end
+end

data/lib/remote_table/processed_by_roo.rb

@@ -0,0 +1,97 @@
+class RemoteTable
+  # Mixed in to process XLS, XLSX, and ODS with the Roo library.
+  module ProcessedByRoo
+    TAG = /<[^>]+>/
+    BLANK = ''
+
+    # Yield each row using Roo.
+    def _each
+      # sometimes Roo forgets to require iconv.
+      require 'iconv'
+      require 'roo'
+
+      spreadsheet = roo_class.new local_copy.path, nil, :ignore
+      if sheet
+        spreadsheet.default_sheet = sheet
+      end
+      
+      first_row = if crop
+        crop.first + 1
+      else
+        skip + 1
+      end
+        
+      last_row = if crop
+        crop.last
+      else
+        spreadsheet.last_row
+      end
+      
+      if not headers
+
+        # create an array to represent this row
+        (first_row..last_row).each do |y|
+          some_value_present = false
+          output = (1..spreadsheet.last_column).map do |x|
+            memo = spreadsheet.cell(y, x).to_s.dup
+            memo = assume_utf8 memo
+            memo.gsub! TAG, BLANK
+            memo.strip!
+            if not some_value_present and not keep_blank_rows and memo.present?
+              some_value_present = true
+            end
+            memo
+          end
+          if keep_blank_rows or some_value_present
+            yield output
+          end
+        end
+
+      else
+        
+        # create a hash to represent this row
+        current_headers = ::ActiveSupport::OrderedHash.new
+        if headers == :first_row
+          (1..spreadsheet.last_column).each do |x|
+            v = spreadsheet.cell(first_row, x)
+            if v.blank?
+              # then look up one
+              v = spreadsheet.cell(first_row - 1, x)
+            end
+            if v.present?
+              v = assume_utf8 v
+              # 'foobar' is found at column 6
+              current_headers[v] = x
+            end
+          end
+          # "advance the cursor"
+          first_row += 1
+        else
+          headers.each_with_index do |k, i|
+            current_headers[k] = i + 1
+          end
+        end
+        (first_row..last_row).each do |y|
+          some_value_present = false
+          output = ::ActiveSupport::OrderedHash.new
+          current_headers.each do |k, x|
+            memo = spreadsheet.cell(y, x).to_s.dup
+            memo = assume_utf8 memo
+            memo.gsub! TAG, BLANK
+            memo.strip!
+            if not some_value_present and not keep_blank_rows and memo.present?
+              some_value_present = true
+            end
+            output[k] = memo
+          end
+          if keep_blank_rows or some_value_present
+            yield output
+          end
+        end
+
+      end
+    ensure
+      local_copy.cleanup
+    end
+  end
+end