table_importer 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7c5fa5aec10db70b4b8391d69151a1da513f755
-  data.tar.gz: f9664947964a4be3e9921c3e04639af19733eed5
+  metadata.gz: 2c087a98855e71813798c295fcf9af1d402ffce0
+  data.tar.gz: 9cdbe7a64cd81f9f4e3e1620cee27b4aea356b7d
 SHA512:
-  metadata.gz: fa51c6c72fd292b64d0c539f28d86c04608952c3c00701048093d82b881d27e81a408e191011de393e6e948eaabf68bdfac0bc2d29139adc3e3f0a2e75c2253d
-  data.tar.gz: 87c4d41ce3edd2b0b68c0bd811a6120fc020dd034f1b8e7aa84d9e3b0bfee97c209a6c7a86f6aa4b36647c8797e4695224a2c7fdcefdb721dc81789488124b94
+  metadata.gz: 4f95de40eb290cc724f13b9916fdf2fd82f59f2919cb8c71e4cf84fd39dc7dd204c18155f39b07a28c730c368af7b7bd44b88e8c38413eb1a4fbde85f45f6a03
+  data.tar.gz: 9d307ff0c93fbce3d07a0d9a5fad25b225b2efd7670488b977acb53b92cea85ee425400785fc507fbba8d0111c057982339b92126d67f7c18edc548e6cdb862c

data/lib/table_importer/copy_and_paste.rb

@@ -4,7 +4,7 @@ module TableImporter
 
     def initialize(data)
       @data = assign_data(data[:content])
-      @column_separator, @record_separator = assign_separators(data[:col_sep], data[:rec_sep])
+      @column_separator, @record_separator = assign_separators(data[:column_separator], data[:record_separator])
       @headers, @headers_present = assign_headers(data[:headers], data[:headers_present])
       @compulsory_headers = data[:compulsory_headers]
       @delete_empty_columns = @data.length < 50000

data/lib/table_importer/csv.rb

@@ -7,8 +7,7 @@ module TableImporter
     def initialize(data)
       @headers_present = data[:headers_present] # user has indicated headers are provided
       @headers = data[:headers]
-      @column_separator = SEPARATORS[data[:column_separator].to_sym] if !data[:column_separator].nil?
-      @record_separator = !data[:record_separator].nil? && data[:record_separator].length > 0 ? SEPARATORS[data[:record_separator].to_sym] : "\n"
+      @column_separator, @record_separator = initialize_separators(data[:column_separator], data[:record_separator])
       @compulsory_headers = data[:compulsory_headers]
       @file = data[:content]
       @delete_empty_columns = File.size(@file) < 100000
@@ -18,7 +17,7 @@ module TableImporter
           raise ArgumentError
         end
         get_column_separator(first_line)
-        @preview_lines = file_has_no_content
+        raise TableImporter::EmptyFileImportError.new unless file_has_content
         @headers = @headers_present ? first_line.split(@column_separator) : default_headers(100) if @headers.blank?
       rescue ArgumentError
         @file = clean_file(@file)
@@ -26,6 +25,12 @@ module TableImporter
       end
     end
 
+    def initialize_separators(col_sep, rec_sep)
+      col_sep = SEPARATORS[col_sep.to_sym] if !col_sep.nil?
+      rec_sep = !rec_sep.nil? && rec_sep.length > 0 ? SEPARATORS[rec_sep.to_sym] : "\n"
+      return col_sep, rec_sep
+    end
+
     def get_first_line
       begin
         SmarterCSV.process(@file.path, default_options({:col_sep => @column_separator.present? ? @column_separator : "\n", :row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 8})) do |chunk|
@@ -40,16 +45,16 @@ module TableImporter
       end
     end
 
-    def file_has_no_content
+    def file_has_content
       begin
         lines = get_preview_lines
         if lines.blank? || lines == 0
-          raise TableImporter::EmptyFileImportError.new
+          return false
         else
-          return lines
+          return true
         end
       rescue NoMethodError
-        raise TableImporter::EmptyFileImportError.new
+        false
       end
     end
 
@@ -77,13 +82,8 @@ module TableImporter
 
     def get_preview_lines
       begin
-        return clean_chunks([@preview_lines], @compulsory_headers, @delete_empty_columns)[0].symbolize_keys[:lines] if !@preview_lines.blank?
-        if @delete_empty_columns
-          chunks = SmarterCSV.process(@file.path, default_options({:row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 50}))
-          return clean_chunks(chunks, @compulsory_headers, true)[0].symbolize_keys[:lines][0..7]
-        end
         SmarterCSV.process(@file.path, default_options({:row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 8})) do |chunk|
-          return clean_chunks([chunk], @compulsory_headers)[0].symbolize_keys[:lines][0..7]
+          return clean_chunks([chunk], @compulsory_headers, @delete_empty_columns)[0].symbolize_keys[:lines][0..7]
         end
       rescue SmarterCSV::HeaderSizeMismatch
         raise TableImporter::HeaderMismatchError.new
@@ -97,14 +97,7 @@ module TableImporter
 
     def get_chunks(chunk_size)
       begin
-        chunks = []
-        if @headers_present
-          key_mapping = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @headers, @headers_present).delete_if{ |key, value| value.blank?}
-          chunks = SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :key_mapping => key_mapping, :remove_unmapped_keys => true, :user_provided_headers => nil}))
-        else
-          user_provided_headers = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @headers, @headers_present).values
-          chunks = SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :user_provided_headers => user_provided_headers, :remove_empty_values => true}))
-        end
+        chunks = @headers_present ? chunks_with_headers(chunk_size) : chunks_without_headers(chunk_size)
         clean_chunks(chunks, @compulsory_headers, @delete_empty_columns)
       rescue ArgumentError
         @file = clean_file(@file)
@@ -112,23 +105,35 @@ module TableImporter
       end
     end
 
+    def chunks_with_headers(chunk_size)
+      key_mapping = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @headers, @headers_present).delete_if{ |key, value| value.blank?}
+      SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :key_mapping => key_mapping, :remove_unmapped_keys => true, :user_provided_headers => nil}))
+    end
+
+    def chunks_without_headers(chunk_size)
+      user_provided_headers = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @headers, @headers_present).values
+      SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :user_provided_headers => user_provided_headers, :remove_empty_values => true}))
+    end
+
     def convert_headers(provided_headers, mapped_headers, headers_present)
       new_headers = []
       old_headers = headers_present ? provided_headers : default_headers
       old_headers.each_with_index do |key, index|
-        key_to_add = "column_#{index}".to_sym
-        mapped_headers.each do |new_key, value|
-          if value.to_s == index.to_s
-            key_to_add = new_key
-          end
-        end
-        new_headers << key_to_add
+        new_headers << map_headers(mapped_headers, index)
       end
       Hash[old_headers.zip(new_headers)]
     end
 
-    # fix quote_char
-    # bit of a hack here to provide the correct number of default headers to the user (rather than just 100)
+    def map_headers(mapped_headers, index)
+      key_to_add = "column_#{index}".to_sym
+      mapped_headers.each do |new_key, value|
+        if value.to_s == index.to_s
+          key_to_add = new_key
+        end
+      end
+      key_to_add
+    end
+
     def default_options(options = {})
       {:col_sep => @column_separator, :row_sep => @record_separator, :force_simple_split => true, :strip_chars_from_headers => /[\-"]/, :remove_empty_values => false,
         :verbose => false, :headers_in_file => @headers_present, :convert_values_to_numeric => false,
@@ -140,6 +145,8 @@ module TableImporter
       import = Tempfile.new(["import", ".xls"], :encoding => "UTF-8")
       import.write(contents.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '?').encode('UTF-8').gsub!(/\r\n|\r/, "\n"))
       import.close
+      SEPARATORS.except!(:newline_windows, :old_newline_mac)
+      @record_separator = "\n"
       return import
     end
   end

data/lib/table_importer/source.rb

@@ -2,7 +2,7 @@ module TableImporter
 
   class Source
 
-    SEPARATORS = {comma: ",", space: " ", tab: "\t", newline_mac: "\n", semicolon: ";", pipe: "|", newline_windows: "\r\n", old_newline_mac: "\r"}
+    SEPARATORS = {comma: ",", space: " ", tab: "\t", newline_mac: "\n", semicolon: ";", newline_windows: "\r\n", old_newline_mac: "\r"}
 
     def initialize (data)
       case data[:type]
@@ -77,7 +77,7 @@ module TableImporter
       chunks.each do |chunk|
         new_chunk = { :lines => [], :errors => []}
         chunk.each_with_index do |line, index|
-          line_empty = line_empty?(line)
+          line, line_empty = line_empty?(line)
           no_compulsory_headers, missing_header = check_compulsory_headers?(line, compulsory_headers)
           if line_empty || no_compulsory_headers
             new_chunk[:errors] << format_error(line, line_empty, no_compulsory_headers, compulsory_headers, missing_header)
@@ -85,7 +85,7 @@ module TableImporter
             if delete_empty_columns
               line.each do |key, value|
                 if value.present? && value.to_s.gsub(/[^A-Za-z0-9]/, '').present?
-                  empty_headers.delete(key)
+                  empty_headers.delete(clean_item(key).to_sym)
                 end
               end
             end
@@ -102,7 +102,20 @@ module TableImporter
 
     private
       def line_empty?(line)
-        line.all?{ |item_key, item_value| line_item_is_garbage?(item_value)}
+        line = clean_line(line)
+        return line, line.all?{ |item_key, item_value| line_item_is_garbage?(item_value)} && line.all?{ |item_key, item_value| line_item_is_garbage?(item_key)}
+      end
+
+      def clean_line(line)
+        map = {}
+        line.each_pair do |key,value|
+          map[clean_item(key).to_sym] = clean_item(value)
+        end
+        map
+      end
+
+      def clean_item(item)
+        item.to_s.delete("\u0000").to_s.delete("\x00")
       end
 
       def check_compulsory_headers?(line, compulsory_headers)
@@ -148,3 +161,4 @@ end
 require 'table_importer/csv'
 require 'table_importer/copy_and_paste'
 require 'table_importer/excel'
+require 'table_importer/exceptions'

data/lib/table_importer/version.rb

@@ -1,3 +1,3 @@
 module TableImporter
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/sources/copy_and_paste_spec.rb

@@ -6,30 +6,34 @@ describe TableImporter::Source do
   context 'when source is a string it' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({
+        :content => "nick@pr.co\ndennis@pr.co\nlorenzo@pr.co",
+        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "creates a source object" do
-      TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+      TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "gets the correct copy and paste chunks" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({
+        :content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co",
+        :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks.first[:lines].first[:email].should eql("nick@pr.co")
     end
 
     it "has the correct number of lines" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks(1).count.should eql(3)
     end
 
     it "has the correct number of chunks" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       source.get_chunks(2).count.should eql(2)
     end
 
     it "does not have extra spaces in the final chunk" do
-      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => :space, :rec_sep => :comma, :compulsory_headers => {:email => true}})
+      source = TableImporter::Source.new({:content => "nick@pr.co, dennis@pr.co, lorenzo@pr.co", :headers_present => false, :headers => {"first_name"=>"", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"0", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => :space, :record_separator => :comma, :compulsory_headers => {:email => true}})
       last_chunk = source.get_chunks(2).last
       (last_chunk[:lines].count + last_chunk[:errors].count).should eql(1)
     end
@@ -39,11 +43,11 @@ describe TableImporter::Source do
     end
 
     it "can get the correct record separator" do
-      @source.get_record_separator.should eql(:old_newline_mac)
+      @source.get_record_separator.should eql(:newline_mac)
     end
 
     it "can get the correct column separator" do
-      @source.get_column_separator.should eql(:old_newline_mac)
+      @source.get_column_separator.should eql(:space)
     end
 
     it "has the correct type" do
@@ -58,7 +62,7 @@ describe TableImporter::Source do
   context 'when source is a different string' do
 
     before(:each) do
-      @source = TableImporter::Source.new({:content => "Nick Dowse <nick@pr.co>, Dennis van der Vliet <dennis@pr.co>, Jeroen Bos <jeroen@pr.co>", :headers_present => false, :headers => {"first_name"=>"0", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :col_sep => "", :rec_sep => "", :compulsory_headers => {:email => true}})
+      @source = TableImporter::Source.new({:content => "Nick Dowse <nick@pr.co>, Dennis van der Vliet <dennis@pr.co>, Jeroen Bos <jeroen@pr.co>", :headers_present => false, :headers => {"first_name"=>"0", "last_name"=>"", "salutation"=>"", "tag_list"=>"", "email"=>"1", "organization"=>"", "url"=>"", "phone"=>"", "job_title"=>"", "second_url"=>"", "notes"=>"", "twitter_username"=>"", "skype_username"=>"", "pinterest_username"=>"", "instagram_username"=>"", "facebook_username"=>"", "last_name_prefix"=>"", "second_email"=>"", "phone_mobile"=>"", "street"=>"", "street_number"=>"", "zipcode"=>"", "city"=>"", "country"=>""}, :user_headers => nil, :type => "copy_and_paste", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
     it "gets the correct chunks" do
@@ -103,7 +107,7 @@ describe TableImporter::Source do
 
           lorenzo,\"lorenzo@pr.co\"
           HÐ, “nick¯â@test”, ¾,€",
-        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => :comma, :rec_sep => :newline_mac, :compulsory_headers => {:email => true}})
+        :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
     end
 
     it "has the correct number of lines" do
@@ -127,7 +131,7 @@ describe TableImporter::Source do
   context 'when string is empty' do
     it 'raises an error when creating a source object' do
       expect{
-        TableImporter::Source.new({:content => "", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :col_sep => :comma, :rec_sep => :newline_mac, :compulsory_headers => {:email => true}})
+        TableImporter::Source.new({:content => "", :headers_present => false, :headers => nil, :user_headers => nil, :type => "copy_and_paste", :column_separator => :comma, :record_separator => :newline_mac, :compulsory_headers => {:email => true}})
       }.to raise_error(TableImporter::EmptyFileImportError)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_importer
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Nick Dowse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-01 00:00:00.000000000 Z
+date: 2014-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spreadsheet