table_importer 0.0.1

data/lib/table_importer/exceptions.rb

@@ -0,0 +1,29 @@
+module Exceptions
+
+  class ImportError < StandardError;
+  end
+
+  class EmptyFileImportError < ImportError
+    def initialize(message = "The file you uploaded has no valid content to import or the content cannot be read. If there is content in your file please try copying and pasting it in instead.")
+      super(message)
+  	end
+  end
+
+  class EmptyStringImportError < ImportError
+    def initialize(message = "The data you pasted in has no valid content to import or it cannot be read.")
+      super(message)
+    end
+  end
+
+  class IncorrectFileError < ImportError
+    def initialize(message = "Sorry, you didn't upload the type of file you said you did.")
+      super(message)
+    end
+  end
+
+  class HeaderMismatchError < ImportError
+    def initialize(message = "Sorry, we couldn't process your file. Did you correctly check whether your file has headers?")
+      super(message)
+    end
+  end
+end

data/lib/table_importer/source.rb

@@ -0,0 +1,149 @@
+module TableImporter
+# require classes
+# require 'import_sources/excel_source'
+# require 'import_sources/string_source'
+  class Source
+
+    SEPARATORS = {comma: ",", space: " ", tab: "\t", newline_mac: "\n", semicolon: ";", pipe: "|", newline_windows: "\r\n", old_newline_mac: "\r"}
+    require 'table_importer/csv'
+    require 'table_importer/copy_and_paste'
+    require 'table_importer/excel'
+
+    def initialize (data)
+      case data[:type]
+      when 'copy_and_paste'
+        @source = CopyAndPaste.new(data)
+      when 'csv'
+        @source = CSV.new(data)
+      when 'xls', 'xlsx'
+        @source = Excel.new(data)
+      else
+        raise Exceptions::IncorrectFileError.new
+      end
+      @source
+    end
+
+    def get_type
+      @source.get_type
+    end
+
+    def get_column_separator(first_line = "")
+      SEPARATORS.key(@source.get_column_separator(first_line))
+    end
+
+    def get_record_separator(first_line = "")
+      SEPARATORS.key(@source.get_record_separator(first_line))
+    end
+
+    def get_headers
+      @source.get_headers
+    end
+
+    def get_lines(start_point = 0, number = -1)
+      @source.get_lines(start_point, number)
+    end
+
+    def get_preview_lines
+      @source.get_preview_lines
+    end
+
+    def get_chunks(chunk_size = 50)
+      @source.get_chunks(chunk_size)
+    end
+
+    def default_headers(number = 100)
+      return @default_headers if @default_headers
+      @default_headers = 1.upto(number).collect do |n|
+        "column_#{n}".to_sym
+      end
+    end
+
+    def get_sep_count(first_line)
+      SEPARATORS.values.collect do |sep|
+        {sep => first_line.scan(sep).count}
+      end
+    end
+
+    def sort_separators(separators)
+      highest_value = 0
+      highest_key = ""
+      separators.each do |sep|
+        if sep.values[0] >= highest_value
+          highest_value = sep.values[0]
+          highest_key = sep.keys[0]
+        end
+      end
+      highest_key
+    end
+
+    def clean_chunks(chunks, compulsory_headers = {}, delete_empty_columns = false)
+      result = []
+      empty_headers = chunks.first.first.keys
+      chunks.each do |chunk|
+        new_chunk = { :lines => [], :errors => []}
+        chunk.each_with_index do |line, index|
+          line_empty = line_empty?(line)
+          no_compulsory_headers, missing_header = check_compulsory_headers?(line, compulsory_headers)
+          if line_empty || no_compulsory_headers
+            new_chunk[:errors] << format_error(line, line_empty, no_compulsory_headers, compulsory_headers, missing_header)
+          else
+            if delete_empty_columns
+              line.each do |key, value|
+                if value.present? && value.to_s.gsub(/[^A-Za-z0-9]/, '').present?
+                  empty_headers.delete(key)
+                end
+              end
+            end
+            new_chunk[:lines] << line
+          end
+        end
+        result << new_chunk unless new_chunk[:lines] == [] && new_chunk[:errors] == []
+      end
+      if delete_empty_columns
+        remove_empty_columns(result, empty_headers)
+      end
+      result
+    end
+
+    private
+      def line_empty?(line)
+        line.all?{ |item_key, item_value| line_item_is_garbage?(item_value)}
+      end
+
+      def check_compulsory_headers?(line, compulsory_headers)
+        if compulsory_headers.key?(:email)
+          if line.key?(:email)
+            line[:email] = clean_email(line[:email])
+            return true, "email" if line[:email].nil? || !line[:email].to_s.match(/@\S/)
+          end
+          return true, "email" if !line.values.any?{ |value| /@\S/ =~ value.to_s }
+        end
+        # here perform other checks for other compulsory headers we might have.
+        return false
+      end
+
+      def clean_email(email)
+        if email
+          email.to_s.gsub(/\A[^A-Za-z0-9]/, '').reverse.gsub(/\A[^A-Za-z0-9]/, '').reverse
+        end
+      end
+
+      def line_item_is_garbage?(item_value)
+        item_value.blank?
+      end
+
+      def format_error(line, line_empty, no_compulsory_headers, compulsory_headers, missing_header)
+        message = line_empty ? "it did not have any content" : " it did not contain this/these required headers: #{missing_header}"
+        {:level => :error, :message => "The following line was not imported because #{message}.", :data => {:line => line, :line_empty => line_empty, :headers => no_compulsory_headers}}
+      end
+
+      def remove_empty_columns(chunks, headers)
+        chunks.each do |chunk|
+          headers.each do |header|
+            chunk[:lines][0][header] = "empty_column"
+          end
+        end
+        chunks
+      end
+  end
+end

data/lib/table_importer/version.rb

@@ -0,0 +1,3 @@
+module TableImporter
+  VERSION = "0.0.1"
+end

data/lib/table_importer.rb

@@ -0,0 +1,3 @@
+require "table_importer/version"
+require 'table_importer/source'
+

data/spec/copy_and_paste_spec.rb

@@ -0,0 +1,134 @@
+# encoding: UTF-8
+require 'spec_helper'
+
+describe TableImporter::Source do
+
+  context 'when source is a string it' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "creates a source object" do
+      TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "gets the correct copy and paste chunks" do
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source.get_chunks.first[:lines].first[:email].should eql("nick@pr.co")
+    end
+
+    it "has the correct number of lines" do
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source.get_chunks(1).count.should eql(3)
+    end
+
+    it "has the correct number of chunks" do
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source.get_chunks(2).count.should eql(2)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      last_chunk = source.get_chunks(2).last
+      (last_chunk[:lines].count + last_chunk[:errors].count).should eql(1)
+    end
+
+    it "gets the correct preview lines" do
+      @source.get_preview_lines.count.should eql(3)
+    end
+
+    it "can get the correct record separator" do
+      @source.get_record_separator.should eql(:old_newline_mac)
+    end
+
+    it "can get the correct column separator" do
+      @source.get_column_separator.should eql(:old_newline_mac)
+    end
+
+    it "has the correct type" do
+      @source.get_type.should eql("copy_and_paste")
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source is a different string' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => "Nick Dowse <nick@pr.co>, Dennis van der Vliet <dennis@pr.co>, Jeroen Bos <jeroen@pr.co>", :headers_present => false, :headers => {"first_name"=>"0", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "gets the correct chunks" do
+      @source.get_chunks.first[:lines].first[:email].should eql("nick@pr.co")
+    end
+
+    it "has the correct column_separator" do
+      @source.get_column_separator.should eql(nil)
+    end
+
+    it "has the correct record_separator" do
+      @source.get_record_separator.should eql(nil)
+    end
+
+    it "has the correct number of lines" do
+      @source.get_chunks(1).count.should eql(3)
+    end
+
+    it "has the correct number of chunks" do
+      @source.get_chunks(2).count.should eql(2)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      last_chunk = @source.get_chunks(2).last
+      (last_chunk[:lines].count + last_chunk[:errors].count).should eql(1)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source is a bad string' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({
+        :content => "Dennis,denni@pr.co,Amsterdam
+          Nick@test.com,”
+          “, Amsterdam
+          jeroen@, \"jeroe
+          adine, \"
+
+          lorenzo,\"lorenzo@pr.co\"
+          HÐ, “nick¯â@test”, ¾,€",
+        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => :comma, :rec_sep => :newline_mac, :compulsory_headers => {:email => true}})
+    end
+
+    it "has the correct number of lines" do
+      @source.get_lines.count.should eql(8)
+    end
+
+    it "has the correct number of chunks" do
+      @source.get_chunks(4).count.should eql(2)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      last_chunk = @source.get_chunks(3).last
+      (last_chunk[:lines].count + last_chunk[:errors].count).should eql(2)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when string is empty' do
+    it 'raises an error when creating a source object' do
+      expect{
+        TableImporter::Source.new({:content => "", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => :comma, :rec_sep => :newline_mac, :compulsory_headers => {:email => true}})
+      }.to raise_error(Exceptions::EmptyFileImportError)
+    end
+  end
+end

data/spec/csv_spec.rb

@@ -0,0 +1,135 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'smarter_csv'
+
+describe TableImporter::Source do
+
+  context 'when source is a csv file with headers' do
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "creates a source object" do
+      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "has the correct headers" do
+      @source.get_headers.should eql(["country", "medium", "salutation", "first_name|", "last_name", "email", "phone_number", "tags|"])
+    end
+
+    it "has the correct number of chunks" do
+      @source.get_chunks(4).count.should eql(3)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      last_chunk = @source.get_chunks(4).last
+      (last_chunk[:lines].count + last_chunk[:errors].count).should eql(1)
+    end
+
+    it "can get the correct record separator" do
+      @source.get_record_separator.should eql(:newline_mac)
+    end
+
+    it "can get the correct column separator" do
+      @source.get_column_separator.should eql(:semicolon)
+    end
+
+    it "has the correct type" do
+      @source.get_type.should eql("csv")
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source is a csv file without headers it' do
+    before(:each) do
+      @source_headers = "false"
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "creates a source object" do
+      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "has the correct number of chunks" do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+      source.get_chunks(4).count.should eql(3)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+      source.get_chunks(4).last[:lines].count.should eql(1)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source is a large csv file with headers' do
+
+  	before { skip }
+
+    before(:all) do
+      @source_headers = 'true'
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, '/spec/files/csv/with_headers_large.csv'].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "creates a source object" do
+      TableImporter::Source.new({:content => File.open([Dir.pwd, '/spec/files/csv/with_headers_large.csv'].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "has the correct number of chunks" do
+      @source.get_chunks(4).count.should eql(2690)
+    end
+
+    it "does not have extra spaces in the final chunk" do
+      last = @source.get_chunks(4).last
+      (last[:errors].count + last[:lines].count).should eql(3)
+    end
+
+    after(:all) do
+      @source = nil
+    end
+  end
+
+  context 'when source is an edge-case csv file without headers' do
+    before(:each) do
+      @source_headers = "false"
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "creates a source object" do
+      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "has the correct number of chunks" do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      source.get_chunks(4).count.should eql(3)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source is a badly encoded file' do
+    it 'can still get the correct chunks' do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/mexico2013_pressdoc.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      source.get_chunks.first[:lines].count.should eql(49)
+    end
+  end
+
+  context 'when source is an empty csv file' do
+
+    it 'raises an error when creating a source object' do
+      begin
+        TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/no_content.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      rescue Exceptions::EmptyFileImportError => e
+        e.message
+      end
+    end
+  end
+end