table_importer 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c087a98855e71813798c295fcf9af1d402ffce0
-  data.tar.gz: 9cdbe7a64cd81f9f4e3e1620cee27b4aea356b7d
+  metadata.gz: 43d3bdb6bc28dc89cfd18f3cf85fa28e126817e6
+  data.tar.gz: 2f0b9aa934e7955a4bc2c0277e4603c191569588
 SHA512:
-  metadata.gz: 4f95de40eb290cc724f13b9916fdf2fd82f59f2919cb8c71e4cf84fd39dc7dd204c18155f39b07a28c730c368af7b7bd44b88e8c38413eb1a4fbde85f45f6a03
-  data.tar.gz: 9d307ff0c93fbce3d07a0d9a5fad25b225b2efd7670488b977acb53b92cea85ee425400785fc507fbba8d0111c057982339b92126d67f7c18edc548e6cdb862c
+  metadata.gz: 7ab65d630b278af498ba4d2bc2950fe792d1ab41d4c69ee19cf939b48114410032bae34cfbffec98aa101c2900cdfe03d85b685bb824a44dc83a9160d1173c33
+  data.tar.gz: 0923656a555afc4ba6dd22e07422d2e53f3f93f5635b55af6ae385a69524e04cc2f034fdf804d547a2fcbe22857ca2f81ebc65b81b945f01a03684a1589aafde

data/lib/table_importer/copy_and_paste.rb

@@ -59,7 +59,7 @@ module TableImporter
 
     def get_preview_lines(start_point = @headers_present ? 1 : 0, end_point = 10)
       begin
-        lines = clean_chunks([get_lines(start_point, end_point)], @compulsory_headers, @delete_empty_columns)[0][:lines]
+        lines = clean_chunks([get_lines(start_point, end_point)], {}, @delete_empty_columns)[0][:lines]
         if lines.first.nil?
           get_preview_lines(start_point+10, end_point+10)
         else

data/lib/table_importer/csv.rb

@@ -33,7 +33,7 @@ module TableImporter
 
     def get_first_line
       begin
-        SmarterCSV.process(@file.path, default_options({:col_sep => @column_separator.present? ? @column_separator : "\n", :row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 8})) do |chunk|
+        SmarterCSV.process(@file.path, default_options({:col_sep => @column_separator.present? ? @column_separator : "\n", :row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 2})) do |chunk|
           if @headers_present
             return chunk.first.keys[0].to_s
           else
@@ -80,10 +80,13 @@ module TableImporter
       @record_separator = sort_separators(separators)
     end
 
-    def get_preview_lines
+    def get_preview_lines(start = 0, finish = 7, chunk_size = 8)
       begin
-        SmarterCSV.process(@file.path, default_options({:row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => 8})) do |chunk|
-          return clean_chunks([chunk], @compulsory_headers, @delete_empty_columns)[0].symbolize_keys[:lines][0..7]
+        SmarterCSV.process(@file.path, default_options({:row_sep => @record_separator != nil ? @record_separator : "\n", :chunk_size => chunk_size})) do |chunk|
+          cleaned_chunk = clean_chunks([chunk], @compulsory_headers, @delete_empty_columns)[0].symbolize_keys[:lines]
+          return cleaned_chunk[start..finish] if cleaned_chunk.first.present?
+          @headers_present = false
+          get_preview_lines(start+8, finish+8, chunk_size+8)
         end
       rescue SmarterCSV::HeaderSizeMismatch
         raise TableImporter::HeaderMismatchError.new
@@ -143,11 +146,18 @@ module TableImporter
     def clean_file(file)
       contents = file.read
       import = Tempfile.new(["import", ".xls"], :encoding => "UTF-8")
-      import.write(contents.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '?').encode('UTF-8').gsub!(/\r\n|\r/, "\n"))
+      utf8_content = contents.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '?').encode('UTF-8').gsub!(/\r\n|\r/, "\n").squeeze("\n")
+      clean_contents = utf8_content[0] == "\n" ? utf8_content[1..-1] : utf8_content
+      import.write(clean_contents)
       import.close
+      reset_separators
+      return import
+    end
+
+    def reset_separators
       SEPARATORS.except!(:newline_windows, :old_newline_mac)
       @record_separator = "\n"
-      return import
+      @column_separator = ""
     end
   end
 end

data/lib/table_importer/excel.rb

@@ -15,7 +15,7 @@ module TableImporter
         if !data[:headers].nil?
           @headers = data[:headers]
         else
-          @headers = @headers_present ? @file.row(1).map { |header| header.to_sym unless header.nil?} : default_headers
+          @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
         end
       rescue NoMethodError
         raise TableImporter::HeaderMismatchError.new
@@ -44,11 +44,12 @@ module TableImporter
 
     def get_preview_lines(start_point = 0, end_point = 10)
       begin
-        if clean_chunks([get_lines(start_point, end_point)], @compulsory_headers)[0][:lines].first.nil?
+        lines = clean_chunks([get_lines(start_point, end_point)], @compulsory_headers)[0][:lines]
+        if lines.first.nil?
           get_preview_lines(start_point+10, end_point+10)
         else
           @headers = @mapping.present? ? convert_headers : @headers
-          clean_chunks([get_lines(start_point+1, end_point+1)], @compulsory_headers, @delete_empty_columns)[0][:lines][0..7]
+          lines[0..8]
         end
       rescue SystemStackError
         raise TableImporter::EmptyFileImportError.new

data/lib/table_importer/source.rb

@@ -161,4 +161,3 @@ end
 require 'table_importer/csv'
 require 'table_importer/copy_and_paste'
 require 'table_importer/excel'
-require 'table_importer/exceptions'

data/lib/table_importer/version.rb

@@ -1,3 +1,3 @@
 module TableImporter
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/table_importer.rb

@@ -1,3 +1,4 @@
-require "table_importer/version"
+require 'table_importer/version'
 require 'table_importer/source'
+require 'table_importer/exceptions'
 

data/spec/files/csv/empty_lines_at_start.csv

@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Country;Medium;Salutation;First name;Last name;Email;Phone number;Tags
+Hong Kong;Men's Uno (Hong Kong Edition);Ms;Noelle;Chan;info@chicgroups.com;-28544930;Ressort_Business
+Hong Kong;Hong Kong Commercial Daily;Mr;Ching Wai
+Hong Kong;South China Morning Post;Mr;John;Cremer;john.cremer@scmp.com;-25651370;Ressort_Business
+Hong Kong;Hong Kong Economic Journal;Mr;Toby;Yiu;tcsyiu@hkej.com;-28566702;Ressort_Business
+Hong Kong;Hong KongTrader;Ms;Regina;Deluna;regina.deluna@hktdc.org;-25843524;Ressort_Business
+Hong Kong;GS1 Hong Kong;Ms;Stella;Cheang;info@gs1hk.org;-28611967;Ressort_Business
+Hong Kong;Playtimes;Ms;Jo;Allun;jo.allum@ppp.com.hk;-22018864;gfgsdfgs;Ressort_Business
+Hong Kong;China Tourism;Ms;Mickey;Ching;edit-c@hkctp.com.hk;-25617149;Ressort_Business
+Hong Kong;Metro Daily (Hong Kong);Mr;Jeff;Lee;news@metrohk.com.hk;-31960748;Ressort_Business

data/spec/files/csv/with_headers.csv

@@ -1,10 +1,10 @@
-Country;Medium;Salutation;First name|;Last name;Email;Phone number;Tags|
-Hong Kong;Men's Uno (Hong Kong Edition);Ms;Noelle;Chan;|info@chicgroups.com;-28544930;Ressort_Business|
-Hong Kong;Hong Kong Commercial Daily;Mr;Ching Wai|
-Hong Kong;South China Morning Post;Mr;John;Cremer;|john.cremer@scmp.com;-25651370;Ressort_Business|
-Hong Kong;Hong Kong Economic |Journal;Mr;Toby;Yiu;tcsyiu@hkej.com;-28566702;Ressort_Business
-Hong Kong;Hong Kong |Trader;Ms;Regina;Deluna;regina.deluna@hktdc.org;-25843524;Ressort_Business|
+Country;Medium;Salutation;First name;Last name;Email;Phone number;Tags
+Hong Kong;Men's Uno (Hong Kong Edition);Ms;Noelle;Chan;info@chicgroups.com;-28544930;Ressort_Business
+Hong Kong;Hong Kong Commercial Daily;Mr;Ching Wai
+Hong Kong;South China Morning Post;Mr;John;Cremer;john.cremer@scmp.com;-25651370;Ressort_Business
+Hong Kong;Hong Kong Economic Journal;Mr;Toby;Yiu;tcsyiu@hkej.com;-28566702;Ressort_Business
+Hong Kong;Hong KongTrader;Ms;Regina;Deluna;regina.deluna@hktdc.org;-25843524;Ressort_Business
 Hong Kong;GS1 Hong Kong;Ms;Stella;Cheang;info@gs1hk.org;-28611967;Ressort_Business
-Hong Kong;Playtimes;Ms;Jo;Allun;jo.allum@ppp.com.hk;-22018864;gfgsdfgs;|Ressort_Business
+Hong Kong;Playtimes;Ms;Jo;Allun;jo.allum@ppp.com.hk;-22018864;gfgsdfgs;Ressort_Business
 Hong Kong;China Tourism;Ms;Mickey;Ching;edit-c@hkctp.com.hk;-25617149;Ressort_Business
 Hong Kong;Metro Daily (Hong Kong);Mr;Jeff;Lee;news@metrohk.com.hk;-31960748;Ressort_Business

data/spec/sources/csv_spec.rb

@@ -9,12 +9,8 @@ describe TableImporter::Source do
       @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
     end
 
-    it "creates a source object" do
-      TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
-    end
-
     it "has the correct headers" do
-      @source.get_headers.should eql(["country", "medium", "salutation", "first_name|", "last_name", "email", "phone_number", "tags|"])
+      @source.get_headers.should eql(["country", "medium", "salutation", "first_name", "last_name", "email", "phone_number", "tags"])
     end
 
     it "has the correct number of chunks" do
@@ -105,4 +101,19 @@ describe TableImporter::Source do
       end
     end
   end
+
+  context 'when source has empty lines at start' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/empty_lines_at_start.csv"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "Gets the preview lines without error" do
+      @source.get_preview_lines.count.should eql(7)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
 end

data/spec/sources/excel_spec.rb

@@ -112,6 +112,36 @@ describe TableImporter::Source do
     end
   end
 
+  context 'when source has empty lines' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines.xlsx"].join), :headers_present => false, :headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "does not throw an error" do
+      expect {@source.get_preview_lines}.to_not raise_error
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
+  context 'when source has 20 empty lines at the beginning' do
+
+    before(:each) do
+      @source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/empty_lines_at_start.xlsx"].join), :headers_present => true, :headers => nil, :user_headers => nil, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
+    end
+
+    it "does not throw an error" do
+      @source.get_preview_lines.count.should eql(6)
+    end
+
+    after(:each) do
+      @source = nil
+    end
+  end
+
   context 'when source is an empty xls file' do
 
     it 'raises an error when creating a source object' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_importer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Nick Dowse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-05 00:00:00.000000000 Z
+date: 2014-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spreadsheet
@@ -161,11 +161,14 @@ files:
 - lib/table_importer/source.rb
 - lib/table_importer/version.rb
 - spec/files/csv/edge_cases.csv
+- spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
 - spec/files/excel/edge_cases.xls
+- spec/files/excel/empty_lines.xlsx
+- spec/files/excel/empty_lines_at_start.xlsx
 - spec/files/excel/no_content.xlsx
 - spec/files/excel/with_headers.xls
 - spec/files/excel/without_headers.xls
@@ -202,11 +205,14 @@ summary: Given a file (or a string) containing a container, along with options,
   will return a hash of those values. Great for importing poorly formatted CSV files.
 test_files:
 - spec/files/csv/edge_cases.csv
+- spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
 - spec/files/excel/edge_cases.xls
+- spec/files/excel/empty_lines.xlsx
+- spec/files/excel/empty_lines_at_start.xlsx
 - spec/files/excel/no_content.xlsx
 - spec/files/excel/with_headers.xls
 - spec/files/excel/without_headers.xls