table_importer 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c222799798d8ddd74dd9122e21210fdf195eb454
-  data.tar.gz: 742761c4969e4515560024965c34df055b792f8c
+  metadata.gz: 4235b6be7a6e1a540674326e18fd5ff08734672e
+  data.tar.gz: f0085296dac6201c7d47fe4120fec5eb70120154
 SHA512:
-  metadata.gz: 0649b415d943e1967ff3618794bfc3290dd56fc50b795319e988b4ebbc51308c86a4451d92215fe1ba50b4761d236941fa776b2688c4852c1d9be3f7b4bdb845
-  data.tar.gz: 02610c56bba2bde7681fcf22556ab3c46714f115c89bb25f7de3b27d3de982d87e5f1c5de3e31a630c976d3b08f3552935ba0151ff493f46c771589ee5e8b579
+  metadata.gz: fd616c41592477e07e9ab682c6f4b4e40c264317a8169aa5f01cb0909bfe6516363957f73bb92153ebcadce4084138c1824c6a55ed0c42d9e7890d35eba50023
+  data.tar.gz: cdc2f3dc97445c32229fe413a196ed109d887be20e32ac28814a9a5aa984d697c4ecdfca096df2ff6af784e94186b9c28e384712a7410c7a1a718e232898a730

data/README.md

@@ -4,7 +4,7 @@
 Table Importer
 ==============
 
-Given a file (or a string) containing a container, along with options, it will return a hash of those values. Great for importing poorly formatted CSV files.
+Given a file (or a string) containing a container, along with options, it will return a hash of those values. Great for importing poorly formatted CSV files. It can handle CSV, Excel (xls and xlsx), Google Drive Spreadsheet, and a copy and pasted string.
 
 Only works  for ruby versions >= 1.9.3.
 
@@ -19,3 +19,104 @@ Then, you'll need to install bundler and the gem dependencies:
 You should now be able to run the local tests:
 
   `bundle exec rake`
+
+Interact with table_importer by creating a TableImporter instance, and then calling methods on that instance.
+
+  `importer = TableImporter::Source.new({options})`
+
+The options you pass in are:
+
+```
+  # The type of the spreadsheet/input you want to import
+  :type => "google" # Google Drive spreadsheet
+        => "csv" # CSV file
+        => "xls" # Excel spreadsheet
+        => "copy_and_paste" # Copy and pasted input
+  
+  # The content to input. Either a file, a string, or google oauth keys.
+  :content => File.open("path/to/file") # for types csv, xls
+           => "Name, Email, Phone Number
+              Nick, nick@example.com, 6412345678" # For type copy_and_paste
+           =>  "google_access_token, spreadsheet_id" # For type google
+  
+  # Whether the first row of input contains column headers
+  :headers_present => true # First row of input is headers
+                   => false # First row of input is not headers
+
+  # Optionally you can provide mapping for the columns. (This can be incomplete).
+  :user_headers => {
+                      "email"=>"0", 
+                      "organization"=>"4", 
+                      "url"=>"9"
+                   }
+  # Used to separate columns. Pass in 'nil' if using Google Spreadsheet, Excel or you don't know.
+  :column_separator => :comma # ','
+                    => :space # ' '
+                    => :tab # '\t'
+                    => :semicolon # ';'
+                       
+  # Used to separate rows. Pass in 'nil' if using Google Spreadsheet, Excel or you don't know.
+  :record_separator => :newline_mac # '\n'
+                    => :newline_windows # '\r\n'
+                    => :old_newline_mac # '\r' (from OSX 9 days)
+  
+  # A hash of compulsory headers. At the moment only "email" is supported.
+  :compulsory_headers => {
+                            :email => true, false # Does each record require an email address to be valid?
+                          }
+  
+```
+
+There are a few ways to interact with the table importer:
+
+```
+  importer = TableImporter::Source.new({options})
+
+  # get the type
+  puts importer.get_type
+    => "csv"
+    
+  # get the column separator
+  puts importer.get_column_separator
+    => "semicolon"
+    
+  # get the row separator
+  puts importer.get_record_separator
+    => "newline_mac"
+    
+  # Get the headers (either the first row if headers are provided, or else default headers
+  puts importer.get_headers
+   => "column_1, column_2, column_3"
+  
+  # Get the first 8 lines (useful for providing a matching option for the user to map their own headers, like mailchimps contact import.
+  puts importer.get_preview_lines
+    => [{:column_1 => "r1c1", :column_2 => "r1c2", :column_3 => "r1c3"}, {:column_1 => "r2c1", :column_2 => "r2c2", :column_3 => "r2c3"} etc]
+  
+  # Get input chunked in an input size (size defaults to 50)
+  puts importer.get_chunks
+    => All input chunked into 50 line blocks.
+  
+  puts importer.get_chunks(25)
+    => All input chunked into 25 line blocks.
+  
+  # The format for the returned chunks is not a simple array of hashes, like get_preview_lines
+  puts importer.get_chunks(2)
+    => [{:lines => [{:column_1 => "r1c1", :column_2 => "r1c2", :column_3 => "r1c3"}, {:column_1 => "r2c1", :column_2 => "r2c2", :column_3 => "r2c3"}], :errors => []}, {:lines => [{:column_1 => "r3c1", :column_2 => "r3c2", :column_3 => "r3c3"}, {:column_1 => "r4c1", :column_2 => "r4c2", :column_3 => "r4c3"}], :errors => []}]
+
+  # The errors hash is for lines that don't contain the compulsory headers, are blank/empty, or the entire line contains no alphanumeric characters.
+
+  # Gets lines of input returned in an array of hashes (doesn't work for CSV yet)
+  # Pass in start and end points
+  puts importer.get_lines(0, 1)
+    => [{:column_1 => "r1c1", :column_2 => "r1c2", :column_3 => "r1c3"}]
+  
+  # Or let it default to getting all lines
+  puts importer.get_lines
+    => All of the lines
+    
+  puts importer.get_lines(5, 25) 
+    => Line 5 up to line 25
+    
+  puts importer.get_lines(5, -1)
+    => Line 5 to the end of the input.
+    

data/lib/table_importer/copy_and_paste.rb

@@ -5,7 +5,8 @@ module TableImporter
     def initialize(data)
       @data = assign_data(data[:content])
       @column_separator, @record_separator = assign_separators(data[:column_separator], data[:record_separator])
-      @headers, @headers_present = assign_headers(data[:headers], data[:headers_present])
+      @headers, @headers_present = assign_headers(data[:headers_present])
+      @mapping = data[:user_headers]
       @compulsory_headers = data[:compulsory_headers]
       @delete_empty_columns = @data.length < 50000
     end
@@ -39,8 +40,8 @@ module TableImporter
       return col_sep, rec_sep
     end
 
-    def assign_headers(headers, headers_present)
-      headers = headers_present ? get_first_line : get_headers if headers.blank?
+    def assign_headers(headers_present)
+      headers = headers_present ? get_first_line : get_headers
       return headers, headers_present
     end
 
@@ -90,7 +91,7 @@ module TableImporter
     end
 
     def get_chunks(chunk_size)
-      @headers = convert_headers(get_first_line, @headers, @headers_present)
+      @headers = convert_headers(get_first_line, @mapping.present? ? @mapping : @headers, @headers_present)
       lines = get_lines(0, -1).in_groups_of(chunk_size, false)
       clean_chunks(lines, @compulsory_headers)
     end

data/lib/table_importer/csv.rb

@@ -6,7 +6,6 @@ module TableImporter
 
     def initialize(data)
       @headers_present = data[:headers_present] # user has indicated headers are provided
-      @headers = data[:headers]
       @column_separator, @record_separator = initialize_separators(data[:column_separator], data[:record_separator])
       @compulsory_headers = data[:compulsory_headers]
       @file = data[:content]
@@ -18,9 +17,10 @@ module TableImporter
         end
         get_column_separator(first_line)
         raise TableImporter::EmptyFileImportError.new unless file_has_content
-        @headers = @headers_present ? first_line.split(@column_separator) : default_headers(100) if @headers.blank?
+        @headers = @headers_present ? first_line.split(@column_separator) : default_headers(100)
       rescue ArgumentError
         @file = clean_file(@file)
+        @column_separator = get_column_separator
         retry
       end
     end
@@ -35,9 +35,9 @@ module TableImporter
       begin
         SmarterCSV.process(@file.path, default_options({:col_sep => @column_separator.present? ? @column_separator : "\n", :row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 2})) do |chunk|
           if @headers_present
-            return chunk.first.keys[0].to_s
+            return line_count(chunk.first.keys)
           else
-            return chunk.first.values[0].to_s
+            return line_count(chunk.first.values)
           end
         end
       rescue EOFError
@@ -45,6 +45,10 @@ module TableImporter
       end
     end
 
+    def line_count(vals)
+      vals.count == 1 ? vals[0].to_s : vals.join(@column_separator)
+    end
+
     def file_has_content
       begin
         lines = get_preview_lines
@@ -104,6 +108,7 @@ module TableImporter
         clean_chunks(chunks, @compulsory_headers, @delete_empty_columns)
       rescue ArgumentError
         @file = clean_file(@file)
+        @column_separator = get_column_separator
         retry
       end
     end
@@ -146,7 +151,7 @@ module TableImporter
     def clean_file(file)
       contents = file.read
       import = Tempfile.new(["import", ".xls"], :encoding => "UTF-8")
-      utf8_content = contents.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '?').encode('UTF-8').gsub!(/\r\n|\r/, "\n").squeeze("\n")
+      utf8_content = contents.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '?').encode('UTF-8').gsub(/\r\n|\r/, "\n").squeeze("\n")
       clean_contents = utf8_content[0] == "\n" ? utf8_content[1..-1] : utf8_content
       import.write(clean_contents)
       import.close

data/lib/table_importer/excel.rb

@@ -5,29 +5,24 @@ module TableImporter
     def initialize(data)
       begin
         @type = File.extname(data[:content]) == ".xls" ? "xls" : "xlsx"
-        @file_path = data[:content].path
         @headers_present = data[:headers_present]
-        @file = get_file
+        @file = get_file(data[:content].path)
         @compulsory_headers = data[:compulsory_headers]
-        @delete_empty_columns = (File.size(@file_path) < 100000)
-        @mapping = !data[:user_headers].blank? ? data[:user_headers] : data[:headers]
+        @delete_empty_columns = (File.size(data[:content].path) < 100000)
+        @mapping = data[:user_headers]
         raise TableImporter::EmptyFileImportError.new if !@file.first_row
-        if !data[:headers].nil?
-          @headers = data[:headers]
-        else
-          @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
-        end
+        @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
       rescue NoMethodError
         raise TableImporter::HeaderMismatchError.new
       end
     end
 
-    def get_file
+    def get_file(path)
       begin
         if @type == "xls"
-          Roo::Excel.new(@file_path).sheet(0)
+          Roo::Excel.new(path).sheet(0)
         elsif @type == "xlsx"
-          Roo::Excelx.new(@file_path).sheet(0)
+          Roo::Excelx.new(path).sheet(0)
         end
       rescue TypeError
         raise TableImporter::IncorrectFileError.new

data/lib/table_importer/google.rb

@@ -8,13 +8,9 @@ module TableImporter
         @file = get_file(data[:content].split(", ")[1], data[:content].split(", ")[0])
         @compulsory_headers = data[:compulsory_headers]
         @delete_empty_columns = false
-        @mapping = !data[:user_headers].blank? ? data[:user_headers] : data[:headers]
+        @mapping = data[:user_headers] if data[:user_headers].present?
         raise TableImporter::EmptyFileImportError.new if !@file.first_row
-        if !data[:headers].nil?
-          @headers = data[:headers]
-        else
-          @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
-        end
+        @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
       rescue NoMethodError
         raise TableImporter::HeaderMismatchError.new
       end

data/lib/table_importer/roo_spreadsheet_source.rb

@@ -25,7 +25,7 @@ module TableImporter
       finish = [@last_row, start + number_of_lines].min
       mapped_lines = []
       (start...finish).each do |row_number|
-        mapped_lines << Hash[@headers.zip(@file.row(row_number))]
+        mapped_lines << Hash[@headers.zip(@file.row(row_number + 1))]
       end
       mapped_lines
     end

data/lib/table_importer/version.rb

@@ -1,3 +1,3 @@
 module TableImporter
-  VERSION = "0.1.1"
+  VERSION = "0.2.1"
 end

data/spec/files/csv/partway.csv

@@ -0,0 +1,229 @@
+internet@example.com
+radio@example.com
+redactie@example.com
+mvdlaan@example.com
+redactie@example.com
+nieuws@example.com
+info@example.com
+heleen@example.com
+gezondheid@example.com
+mensennatuur@example.com
+wilma@example.com
+editor@example.com
+info@example.com
+m.aandebrugh@example.com
+daphne.van.paassen@example.com
+redactienosop3@example.com
+redactie@example.com
+redactie@example.com
+l.nieber@example.com
+t.vansoest@example.com
+wvhengel@example.com
+hdboer@example.com
+editienl@example.com
+ad@example.com
+radio@example.com
+ct@example.com
+e.kreulen@example.com
+kunststof@example.com
+redactie@example.com
+webred@example.com
+pers@example.com
+nieuwsredactie@example.com
+rsteenhorst@example.com
+wester@example.com
+foto@example.com
+webredactie@example.com
+c.paulussen@example.com
+bnrredactie@example.com
+karlijnmarchildon@example.com
+leon@example.com
+a.gelder@example.com
+barbara.van.gool@example.com
+redactie@example.com
+ps@example.com
+next@example.com
+ahilten@example.com
+weekend@example.com
+redactie@example.com
+info@example.com
+e.van.der.velden@example.com
+merel.izaks@example.com
+deochtend@example.com
+webredactie@example.com
+beleef@example.com
+info@example.com
+h.vanhouwelingen@example.com
+redactie@example.com
+redactie-i@example.com
+sv@example.com
+j.bas@example.com
+albertdelouw@example.com
+saskia.haitsma@example.com
+cosmopolitan@example.com
+bladredactie@example.com
+m.ham@example.com
+stephanie.brandes@example.com
+rtlboulevard@example.com
+redactiesecretariaat@example.com
+nieuwsdienst@example.com
+erik.feenstra@example.com
+j.vdoetelaar@example.com
+a.karimi@example.com
+redactie@example.com
+redactie-flow@example.com
+redactie@example.com
+redactie@example.com
+redactie@example.com
+redactie@example.com
+multimedia@example.com
+pauw@example.com
+deborah.blekkenhorst@example.com
+redactie@example.com
+wetenschap@example.com
+fogteloo@example.com
+info@example.com
+redactie@example.com
+pat.boon@example.com
+patrick.smit@example.com
+mensenleven@example.com
+h.salm@example.com
+cielke@example.com
+m.t.hart@example.com
+r.boxsem@example.com
+nieuwsdienst@example.com
+a.engbers@example.com
+jeugdjournaal@example.com
+b.vandeweijer@example.com
+josefin.hoenders@example.com
+redactie@example.com
+floor.ligtvoet@example.com
+j.koelewijn@example.com
+planning.show@example.com
+noordhuis@example.com
+redactie@example.com
+nathalie.groeneveld@example.com
+redactie@example.com
+pjansen@example.com
+info@example.com
+coleta@example.com
+wart.krol@example.com
+info@example.com
+pers@example.com
+verslaggeverij@example.com
+martijn.verburg@example.com
+mavdmarel@example.com
+redactie@example.com
+e.devisser@example.com
+info@example.com
+info@example.com
+nieuws@example.com
+t.voermans@example.com
+r.brouwer@example.com
+marieke.de.witte@example.com
+redactie@example.com
+binnenland@example.com
+redactie@example.com
+koen.van.huijgevoort@example.com
+redactie@example.com
+msienot@example.com
+kunststoftv@example.com
+opinie@example.com
+vrouwmagazine@example.com
+linda@example.com
+dwdd@example.com
+nrc@example.com
+anna.pruis@example.com
+m.kranenburg@example.com
+info@example.com
+redactie@example.com
+wmeteren@example.com
+amsterdam@example.com
+noud.broekhof@example.com
+nieuwsdienst@example.com
+jildou@example.com
+arjan.poggenklaas@example.com
+marjan.vandenberg@example.com
+binnenland@example.com
+ditisdedag@example.com
+vrij@example.com
+info@example.com
+l.verhoeven@example.com
+m.verburg@example.com
+leven@example.com
+redactiegiel@example.com
+nieuwsdienst@example.com
+foto@example.com
+brittekoppel@example.com
+g.vanteeffelen@example.com
+redactie@example.com
+c.muis@example.com
+servaas.van.der.laan@example.com
+nieuwsdienst@example.com
+erwin@example.com
+show@example.com
+info@example.com
+magazine@example.com
+hart@example.com
+nosbinnenland@example.com
+richt.kooistra@example.com
+almar@example.com
+k.ullah@example.com
+a.crielaard@example.com
+s.lautenbach@example.com
+nosop3@example.com
+redactie@example.com
+autoshow@example.com
+pers@example.com
+boven@example.com
+redactie@example.com
+gezondheid@example.com
+laura.vanbaars@example.com
+esther.monsanto@example.com
+internet@example.com
+frank.thies@example.com
+redactie@example.com
+hvdberge@example.com
+info@example.com
+akerkum@example.com
+redactie@example.com
+saskiavanommen@example.com
+redactie@example.com
+m.kerres@example.com
+s.heijne@example.com
+stad@example.com
+koffietijd@example.com
+redactie@example.com
+c.vanduin@example.com
+webredactie@example.com
+tatiana.pijnenburg@example.com
+bert.heuvelman@example.com
+judith.van.de.hulsbeek@example.com
+dick.van.bolhuis@example.com
+ugamedia@example.com
+lambert.teuwissen@example.com
+kiki.duren@example.com
+redactie@example.com
+rtlnieuws@example.com
+hgillissen@example.com
+blog@example.com
+sjaak@example.com
+jolanda.van.duyvenbode@example.com
+redactie@example.com
+internet@example.com
+t.staal@example.com
+margreet.botter@example.com
+roodshow@example.com
+marijn.lansbergen@example.com
+redactie@example.com
+red@example.com
+redactie@example.com
+nieuws@example.com
+info@example.com
+redactie-i@example.com
+info@example.com
+info@example.com
+info@example.com
+info@example.com
+communicatie@example.com
+advertising@example.com

data/spec/sources/copy_and_paste_spec.rb

@@ -8,32 +8,32 @@ describe TableImporter::Source do
     before(:each) do
       @source = TableImporter::Source.new({
         :content => "nick@pr.co\ndennis@pr.co\nlorenzo@pr.co",
-        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        :headers_present => false, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "creates a source object" do
-      TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "gets the correct copy and paste chunks" do
       source = TableImporter::Source.new({
         :content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co",
-        :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
+        :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks.first[:lines].first[:email].should eql("nick@pr.co")
     end
 
     it "has the correct number of lines" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks(1).count.should eql(3)
     end
 
     it "has the correct number of chunks" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks(2).count.should eql(2)
     end
 
     it "does not have extra spaces in the final chunk" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       last_chunk = source.get_chunks(2).last
       (last_chunk[:lines].count + last_chunk[:errors].count).should eql(1)
     end
@@ -62,7 +62,7 @@ describe TableImporter::Source do
   context 'when source is a different string' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => "Nick Dowse <nick@pr.co>, Dennis van der Vliet <dennis@pr.co>, Jeroen Bos <jeroen@pr.co>", :headers_present => false, :headers => {"first_name"=>"0", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => "Nick Dowse <nick@pr.co>, Dennis van der Vliet <dennis@pr.co>, Jeroen Bos <jeroen@pr.co>", :headers_present => false, :user_headers => {"first_name"=>"0", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "gets the correct chunks" do
@@ -107,7 +107,7 @@ describe TableImporter::Source do
 
           lorenzo,\"lorenzo@pr.co\"
           HÐ, “nick¯â@test”, ¾,€",
-        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+        :headers_present => false, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
     end
 
     it "has the correct number of lines" do
@@ -131,7 +131,7 @@ describe TableImporter::Source do
   context 'when string is empty' do
     it 'raises an error when creating a source object' do
       expect{
-        TableImporter::Source.new({:content => "", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+        TableImporter::Source.new({:content => "", :headers_present => false, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
       }.to raise_error(TableImporter::EmptyFileImportError)
     end
   end

data/spec/sources/csv_spec.rb

@@ -6,7 +6,7 @@ describe TableImporter::Source do
 
   context 'when source is a csv file with headers' do
     before(:each) do
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "has the correct headers" do
@@ -42,20 +42,20 @@ describe TableImporter::Source do
   context 'when source is a csv file without headers it' do
     before(:each) do
       @source_headers = "false"
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "creates a source object" do
-      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "has the correct number of chunks" do
-      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
       source.get_chunks(4).count.should eql(3)
     end
 
     it "does not have extra spaces in the final chunk" do
-      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/without_headers.csv"].join), :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"5", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "csv", :column_separator => :semicolon, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
       source.get_chunks(4).last[:lines].count.should eql(1)
     end
 
@@ -67,15 +67,15 @@ describe TableImporter::Source do
   context 'when source is an edge-case csv file without headers' do
     before(:each) do
       @source_headers = "false"
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "creates a source object" do
-      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "has the correct number of chunks" do
-      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/edge_cases.csv"].join), :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       source.get_chunks(4).count.should eql(3)
     end
 
@@ -86,7 +86,7 @@ describe TableImporter::Source do
 
   context 'when source is a badly encoded file' do
     it 'can still get the correct chunks' do
-      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/mexico2013_pressdoc.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/mexico2013_pressdoc.csv"].join), :headers_present => true, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       source.get_chunks.first[:lines].count.should eql(49)
     end
   end
@@ -95,7 +95,7 @@ describe TableImporter::Source do
 
     it 'raises an error when creating a source object' do
       begin
-        TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/no_content.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/no_content.csv"].join), :headers_present => true, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       rescue TableImporter::EmptyFileImportError => e
         e.message
       end
@@ -105,7 +105,7 @@ describe TableImporter::Source do
   context 'when source has empty lines at start' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/empty_lines_at_start.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/empty_lines_at_start.csv"].join), :headers_present => true, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "Gets the preview lines without error" do
@@ -116,4 +116,19 @@ describe TableImporter::Source do
       @source = nil
     end
   end
+
+  context 'when source is badly encoded partway through the file' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/partway.csv"].join), :headers_present => false, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "Gets the first chunk without error" do
+      @source.get_chunks[0][:lines].count.should eql(50)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
 end

data/spec/sources/excel_spec.rb

@@ -8,11 +8,11 @@ describe TableImporter::Source do
     context 'when mapping has not been set' do
 
       before(:each) do
-        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/with_headers.xls"].join), :headers_present => true, :user_headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/with_headers.xls"].join), :headers_present => true, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       end
 
       it "gets the preview lines" do
-        @source.get_preview_lines.count.should eql(5)
+        @source.get_preview_lines.count.should eql(6)
       end
 
       it "has the correct type" do
@@ -27,7 +27,7 @@ describe TableImporter::Source do
     context 'when mapping has been set' do
 
       before(:each) do
-        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/with_headers.xls"].join), :headers_present => true, :headers_present => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/with_headers.xls"].join), :headers_present => true, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       end
 
       it "has the correct headers" do
@@ -53,12 +53,10 @@ describe TableImporter::Source do
     end
   end
 
-
-
   context 'when source is an xls file without headers' do
     context 'when mapping has not been set' do
       before(:each) do
-        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/without_headers.xls"].join), :headers_present => false, :user_headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/without_headers.xls"].join), :headers_present => false, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       end
 
       it "has the correct number of columns" do
@@ -73,7 +71,7 @@ describe TableImporter::Source do
     context 'when mapping has been set' do
 
       before(:each) do
-        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/without_headers.xls"].join), :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+        @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/without_headers.xls"].join), :headers_present => false, :user_headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
       end
 
       it "has the correct number of lines" do
@@ -121,7 +119,7 @@ describe TableImporter::Source do
   context 'when source has empty lines' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines.xlsx"].join), :headers_present => false, :user_headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines.xlsx"].join), :headers_present => false, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "does not throw an error" do
@@ -136,7 +134,7 @@ describe TableImporter::Source do
   context 'when source has 20 empty lines at the beginning' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines_at_start.xlsx"].join), :headers_present => true, :user_headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines_at_start.xlsx"].join), :headers_present => true, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "does not throw an error" do
@@ -159,16 +157,40 @@ describe TableImporter::Source do
     end
   end
 
-  context 'mediaprofiler' do
+  context 'premapped_1' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/mediaprofiler.xls"].join), :headers_present => "true", :type => "xls", :column_separator => "", :record_separator => "",
-        :user_headers => {:first_name=>0, :last_name_prefix=>1, :last_name=>2, :organization=>3, :email=>5, :second_email=>6, :phone=>7, :phone_mobile=>8, :twitter_username=>9, :url=>10, :street=>11, :street_number=>12, :zipcode=>13, :country=>18}
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/premapped_1.xls"].join), :headers_present => "true", :type => "xls", :column_separator => "", :record_separator => "",
+        :user_headers => {:first_name=>0, :last_name_prefix=>1, :last_name=>2, :organization=>3, :second_email=>5, :email=>6, :phone=>7, :phone_mobile=>8, :twitter_username=>9, :url=>10, :street=>11, :street_number=>12, :zipcode=>13, :country=>18},
+        :compulsory_headers => {:email => true}
       })
     end
 
     it "has correct mapping" do
-      @source.get_preview_lines.first.keys.first.should == :first_name
+      expect(@source.get_preview_lines.first.keys.first).to eql(:first_name)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'premapped_2' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/premapped_2.xls"].join), :headers_present => "true", :type => "xls", :column_separator => "", :record_separator => "",
+        :user_headers => {:organization=>0, :salutation=>2, :first_name=>3, :last_name_prefix=>4, :last_name=>5, :street=>6, :zipcode=>9, :city=>10, :country=>11,
+          :url=>12, :email=>13, :phone=>14, :notes=>18, :secondary_tags=>19, cached_tag_list: 24},
+        :compulsory_headers => {:email => true}
+      })
+    end
+
+    it "has correct mapping" do
+      expect(@source.get_preview_lines.first.keys.first).to eql(:organization)
+    end
+
+    it "gets the correct number of preview lines" do
+      expect(@source.get_preview_lines.count).to eql(1)
     end
 
     after(:each) do

data/table_importer.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "spreadsheet", "0.9.1"
   spec.add_dependency 'roo'
   spec.add_dependency 'google_drive'
-  spec.add_dependency 'smarter_csv'
+  spec.add_dependency 'smarter_csv', '1.0.17'
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_importer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Nick Dowse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-26 00:00:00.000000000 Z
+date: 2014-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spreadsheet
@@ -56,16 +56,16 @@ dependencies:
   name: smarter_csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.17
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.17
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -208,13 +208,15 @@ files:
 - spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
+- spec/files/csv/partway.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
 - spec/files/excel/edge_cases.xls
 - spec/files/excel/empty_lines.xlsx
 - spec/files/excel/empty_lines_at_start.xlsx
-- spec/files/excel/mediaprofiler.xls
 - spec/files/excel/no_content.xlsx
+- spec/files/excel/premapped_1.xls
+- spec/files/excel/premapped_2.xls
 - spec/files/excel/with_headers.xls
 - spec/files/excel/without_headers.xls
 - spec/sources/copy_and_paste_spec.rb
@@ -256,13 +258,15 @@ test_files:
 - spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
+- spec/files/csv/partway.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
 - spec/files/excel/edge_cases.xls
 - spec/files/excel/empty_lines.xlsx
 - spec/files/excel/empty_lines_at_start.xlsx
-- spec/files/excel/mediaprofiler.xls
 - spec/files/excel/no_content.xlsx
+- spec/files/excel/premapped_1.xls
+- spec/files/excel/premapped_2.xls
 - spec/files/excel/with_headers.xls
 - spec/files/excel/without_headers.xls
 - spec/sources/copy_and_paste_spec.rb