remote_table 1.4.0 → 2.0.0

data/lib/remote_table/format.rb

@@ -1,49 +0,0 @@
-require 'iconv'
-if RUBY_VERSION >= '1.9'
-  # for an excellent explanation see http://blog.segment7.net/2010/12/17/from-iconv-iconv-to-string-encode
-  Kernel.warn "[remote_table] Apologies - using iconv because Ruby 1.9.x's String#encode doesn't have transliteration tables (yet)"
-end
-
-require 'remote_table/format/mixins/textual'
-require 'remote_table/format/mixins/processed_by_roo'
-require 'remote_table/format/mixins/processed_by_nokogiri'
-require 'remote_table/format/excel'
-require 'remote_table/format/excelx'
-require 'remote_table/format/delimited'
-require 'remote_table/format/open_office'
-require 'remote_table/format/fixed_width'
-require 'remote_table/format/html'
-require 'remote_table/format/xml'
-require 'remote_table/format/yaml'
-class RemoteTable  
-  class Format
-    
-    attr_reader :t
-
-    def initialize(t)
-      @t = t
-    end
-    
-    def transliterate_to_utf8(str)
-      if str.is_a?(::String)
-        [ iconv.iconv(str), iconv.iconv(nil) ].join
-      end
-    end
-
-    def assume_utf8(str)
-      if str.is_a?(::String) and ::RUBY_VERSION >= '1.9'
-        str.encode! t.config.external_encoding
-      else
-        str
-      end
-    end
-    
-    private
-    
-    def iconv
-      @iconv ||= ::Iconv.new(t.config.external_encoding_iconv, t.config.internal_encoding)
-    end
-    
-    include ::Enumerable
-  end
-end

data/lib/remote_table/format/delimited.rb

@@ -1,60 +0,0 @@
-class RemoteTable
-  class Format
-    class Delimited < Format
-      if ::RUBY_VERSION >= '1.9'
-        require 'csv'
-        Engine = ::CSV
-      else
-        require 'fastercsv'
-        Engine = ::FasterCSV
-      end
-
-      include Textual
-
-      def each(&blk)
-        remove_useless_characters!
-        fix_newlines!
-        transliterate_whole_file_to_utf8!
-        skip_rows!
-        Engine.new(t.local_file.encoded_io, fastercsv_options).each do |row|
-          if row.is_a?(Engine::Row)
-            hash = row.inject(::ActiveSupport::OrderedHash.new) do |memo, (k, v)|
-              if k.present?
-                memo[k] = v.to_s
-              end
-              memo
-            end
-            yield hash if t.config.keep_blank_rows or hash.any? { |k, v| v.present? }
-          elsif row.is_a?(::Array)
-            array = row.map { |v| v.to_s }
-            yield array if t.config.keep_blank_rows or array.any? { |v| v.present? }
-          end
-        end
-      ensure
-        t.local_file.cleanup
-      end
-
-      private
-
-      FASTERCSV_OPTIONS = [
-        :unconverted_fields,
-        :col_sep,
-        :headers,
-        :row_sep,
-        :return_headers,
-        :header_converters,
-        :quote_char,
-        :skip_blanks,
-        :converters,
-        :force_quotes,
-      ]
-
-      def fastercsv_options
-        hsh = t.config.user_specified_options.slice *FASTERCSV_OPTIONS
-        hsh[:skip_blanks] = !t.config.keep_blank_rows
-        hsh.reverse_merge! :headers => t.config.headers
-        hsh.reverse_merge! :col_sep => t.config.delimiter
-      end
-    end
-  end
-end

data/lib/remote_table/format/excel.rb

@@ -1,10 +0,0 @@
-class RemoteTable
-  class Format
-    class Excel < Format
-      include ProcessedByRoo
-      def roo_class
-        ::Excel
-      end
-    end
-  end
-end

data/lib/remote_table/format/excelx.rb

@@ -1,10 +0,0 @@
-class RemoteTable
-  class Format
-    class Excelx < Format
-      include ProcessedByRoo
-      def roo_class
-        ::Excelx
-      end
-    end
-  end
-end

data/lib/remote_table/format/fixed_width.rb

@@ -1,60 +0,0 @@
-class RemoteTable
-  class Format
-    class FixedWidth < Format
-      include Textual
-      def each(&blk)
-        require 'fixed_width-multibyte'
-
-        remove_useless_characters!
-        fix_newlines!
-        transliterate_whole_file_to_utf8!
-        crop_rows!
-        skip_rows!
-        cut_columns!
-        parser.parse[:rows].each do |row|
-          row.reject! { |k, v| k.blank? }
-          row.each do |k, v|
-            row[k] = v.strip
-          end
-          yield row if t.config.keep_blank_rows or row.any? { |k, v| v.present? }
-        end
-      ensure
-        t.local_file.cleanup
-      end
-      
-      private
-      
-      def parser
-        return @parser if @parser.is_a?(::FixedWidth::Parser)
-        if ::FixedWidth::Section.private_instance_methods.map(&:to_sym).include?(:unpacker)
-          raise ::RuntimeError, "You need a different (newer) version of the FixedWidth gem that supports multibyte encoding, sometime after https://github.com/timonk/fixed_width/pull/1 was incorporated"
-        end
-        @parser = ::FixedWidth::Parser.new definition, t.local_file.encoded_io
-      end
-      
-      def definition
-        @definition ||= if t.config.schema_name.is_a?(::String) or t.config.schema_name.is_a?(::Symbol)
-          ::FixedWidth.send :definition, t.config.schema_name
-        elsif t.config.schema.is_a?(::Array)
-          everything = lambda { |_| true }
-          srand # in case this was forked by resque
-          ::FixedWidth.define(rand.to_s) do |d|
-            d.rows do |row|
-              row.trap(&everything)
-              t.config.schema.each do |name, width, options|
-                name = name.to_s
-                if name == 'spacer'
-                  row.spacer width
-                else
-                  row.column name, width, options
-                end
-              end
-            end
-          end
-        else
-          raise ::ArgumentError, "expecting schema_name to be a String or Symbol, or schema to be an Array"
-        end
-      end
-    end
-  end
-end

data/lib/remote_table/format/html.rb

@@ -1,12 +0,0 @@
-class RemoteTable
-  class Format
-    class HTML < Format
-      include Textual
-      include ProcessedByNokogiri
-      
-      def nokogiri_class
-        ::Nokogiri::HTML::Document
-      end
-    end
-  end
-end

data/lib/remote_table/format/mixins/processed_by_nokogiri.rb

@@ -1,70 +0,0 @@
-class RemoteTable
-  class Format
-    module ProcessedByNokogiri
-      def each
-        require 'nokogiri'
-        require 'cgi'
-        
-        raise ::ArgumentError, "Need :row_css or :row_xpath in order to process XML or HTML" unless t.config.row_css or t.config.row_xpath
-        remove_useless_characters!
-        transliterate_whole_file_to_utf8!
-        
-        headers = t.config.headers
-
-        xml = nokogiri_class.parse(unescaped_xml_without_soft_hyphens, nil, 'UTF-8')
-        (row_css? ? xml.css(t.config.row_css) : xml.xpath(t.config.row_xpath)).each do |row|
-          values = if column_css?
-            row.css(t.config.column_css)
-          elsif column_xpath?
-            row.xpath(t.config.column_xpath)
-          else
-            [row]
-          end.map { |cell| assume_utf8 cell.content.gsub(/\s+/, ' ').strip }
-          if headers == :first_row
-            headers = values.select(&:present?)
-            next
-          end
-          output = if t.config.output_class == ::Array
-            values
-          else
-            zip headers, values
-          end
-          if t.config.keep_blank_rows or values.any?
-            yield output
-          end
-        end
-      ensure
-        t.local_file.cleanup
-      end
-
-      private
-
-      def row_css?
-        !!t.config.row_css
-      end
-      
-      def column_css?
-        !!t.config.column_css
-      end
-      
-      def column_xpath?
-        !!t.config.column_xpath
-      end
-      
-      # http://snippets.dzone.com/posts/show/406
-      def zip(keys, values)
-        hash = ::ActiveSupport::OrderedHash.new
-        keys.zip(values) { |k,v| hash[k]=v }
-        hash
-      end
-
-      # should we be doing this in ruby?
-      def unescaped_xml_without_soft_hyphens
-        str = ::CGI.unescapeHTML t.local_file.encoded_io.read
-        # get rid of MS Office baddies
-        str.gsub! '­', ''
-        str
-      end
-    end
-  end
-end

data/lib/remote_table/format/mixins/processed_by_roo.rb

@@ -1,63 +0,0 @@
-class RemoteTable
-  class Format
-    module ProcessedByRoo
-      def each(&blk)
-        require 'iconv'
-        require 'roo'
-
-        spreadsheet = roo_class.new t.local_file.path, nil, :ignore
-        if t.config.sheet
-          spreadsheet.default_sheet = t.config.sheet
-        end
-        
-        first_row = if t.config.crop
-          t.config.crop.first + 1
-        else
-          t.config.skip + 1
-        end
-          
-        last_row = if t.config.crop
-          t.config.crop.last
-        else
-          spreadsheet.last_row
-        end
-        
-        if t.config.output_class == ::Array
-          (first_row..last_row).each do |y|
-            output = (1..spreadsheet.last_column).map do |x|
-              assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
-            end
-            yield output if t.config.keep_blank_rows or output.any? { |v| v.present? }
-          end
-        else
-          headers = ::ActiveSupport::OrderedHash.new
-          if t.config.use_first_row_as_header?
-            (1..spreadsheet.last_column).each do |x|
-              v = spreadsheet.cell(first_row, x)
-              v = spreadsheet.cell(first_row - 1, x) if v.blank? # look up
-              if v.present?
-                v = assume_utf8 v
-                headers[v] = x # 'foobar' is found at column 6
-              end
-            end
-            # "advance the cursor"
-            first_row += 1
-          else
-            t.config.headers.each_with_index do |k, i|
-              headers[k] = i + 1
-            end
-          end
-          (first_row..last_row).each do |y|
-            output = ::ActiveSupport::OrderedHash.new
-            headers.each do |k, x|
-              output[k] = assume_utf8 spreadsheet.cell(y, x).to_s.gsub(/<[^>]+>/, '').strip
-            end
-            yield output if t.config.keep_blank_rows or output.any? { |k, v| v.present? }
-          end
-        end
-      ensure
-        t.local_file.cleanup
-      end
-    end
-  end
-end

data/lib/remote_table/format/mixins/textual.rb

@@ -1,43 +0,0 @@
-require 'fileutils'
-class RemoteTable
-  class Format
-    module Textual      
-      USELESS_CHARACTERS = [
-        '\xef\xbb\xbf',   # UTF-8 byte order mark
-        '\xc2\xad',       # soft hyphen, often inserted by MS Office (html: ­)
-      ]
-      def remove_useless_characters!
-        t.local_file.in_place :perl, "s/#{USELESS_CHARACTERS.join('//g; s/')}//g"
-        if t.config.internal_encoding =~ /windows.?1252/i
-          # soft hyphen again, as I have seen it appear in windows 1252
-          t.local_file.in_place :perl, 's/\xad//g'
-        end
-      end
-      
-      def transliterate_whole_file_to_utf8!
-        t.local_file.in_place :iconv, t.config.external_encoding_iconv, t.config.internal_encoding
-        t.config.user_specified_options[:encoding] = t.config.external_encoding
-      end
-      
-      def fix_newlines!
-        t.local_file.in_place :perl, 's/\r\n|\n|\r/\n/g'
-      end
-      
-      def skip_rows!
-        return unless t.config.skip > 0
-        t.local_file.in_place :tail, "+#{t.config.skip + 1}"
-      end
-      
-      def crop_rows!
-        return unless t.config.crop
-        t.local_file.in_place :tail, "+#{t.config.crop.first}"
-        t.local_file.in_place :head, (t.config.crop.last - t.config.crop.first + 1)
-      end
-      
-      def cut_columns!
-        return unless t.config.cut
-        t.local_file.in_place :cut, t.config.cut
-      end
-    end
-  end
-end

data/lib/remote_table/format/open_office.rb

@@ -1,13 +0,0 @@
-class RemoteTable
-  class Format
-    class OpenOffice < Format
-      include ProcessedByRoo
-      def roo_class
-        if ::RUBY_PLATFORM == 'java'
-          ::Kernel.warn "[remote_table] Opening ODS files on JRuby is known to fail because of a flaw in the underlying Roo library"
-        end
-        ::Openoffice
-      end
-    end
-  end
-end

data/lib/remote_table/format/xml.rb

@@ -1,12 +0,0 @@
-class RemoteTable
-  class Format
-    class XML < Format
-      include Textual
-      include ProcessedByNokogiri
-      
-      def nokogiri_class
-        ::Nokogiri::XML::Document
-      end
-    end
-  end
-end

data/lib/remote_table/format/yaml.rb

@@ -1,14 +0,0 @@
-require 'yaml'
-
-class RemoteTable
-  class Format
-    class Yaml < Format
-      def each(&blk)
-        data = YAML.load_file t.local_file.path
-        data.each &blk
-      ensure
-        t.local_file.cleanup
-      end
-    end
-  end
-end