table_importer 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 865e1844a77e5b2deb00d9f8d60c3a3c03c6889d
-  data.tar.gz: 5efeeabd463f044f90c0abf2792ca026c80b9998
+  metadata.gz: 84f4728211cdfa3a07bfec96d6e3dbf35554ec98
+  data.tar.gz: 08fb3b8daf2428a44348b798d335d741ef210dfd
 SHA512:
-  metadata.gz: e554a0f3ba76457f86a42da86c006462b990b627dbae53c2a35c7c0554499160a282dc3b0df0290f72ae29bb2630a72b3fa0a391a04e0e5647ec8cbcf0b0a6c9
-  data.tar.gz: 76d89f4db35c740e6c1dd7a72d99dac88af2c498c5c047d2ea8bb451ba9e8b529105a421ca7946816e45144085be677eec8b1959d5f91f95b943b2144268cdef
+  metadata.gz: f4de9554e43a187a8e09820ff4ee9b6dc3b3c3203648a9866aef834de8e23e951d7c1ecf6d7cdb12468b31f5ab58886820d554ad5342e557df22208118311e02
+  data.tar.gz: 67fd83970aa94616152593e6160bb2311da1174526c1f23b6717c7e3eb42d77b01bcb1c19a254f8d9d1008761daa865f71723d4ae799d43fcabd388494fc82bf

data/README.md

@@ -64,13 +64,18 @@ The options you pass in are:
   :compulsory_headers => {
                             :email => true, false # Does each record require an email address to be valid?
                           }
+
+  # Whether nil values that are a string (ie strings that equal "NULL", "null", "nil", or "undefined") should be replaced with actual nil values.
+  :remove_nil_values => true
+  :remove_nil_values => false
   
 ```
 
 There are a few ways to interact with the table importer:
 
 ```
-  importer = TableImporter::Source.new({options})
+  options = { type: "csv",  }
+  importer = TableImporter::Source.new(options)
 
   # get the type
   puts importer.get_type
@@ -88,7 +93,7 @@ There are a few ways to interact with the table importer:
   puts importer.get_headers
    => "column_1, column_2, column_3"
   
-  # Get the first 8 lines (useful for providing a matching option for the user to map their own headers, like mailchimps contact import.
+  # Get the first 8 lines (useful for providing a matching option for the user to map their own headers, like Mailchimp's contact import.
   puts importer.get_preview_lines
     => [{:column_1 => "r1c1", :column_2 => "r1c2", :column_3 => "r1c3"}, {:column_1 => "r2c1", :column_2 => "r2c2", :column_3 => "r2c3"} etc]
   
@@ -99,7 +104,7 @@ There are a few ways to interact with the table importer:
   puts importer.get_chunks(25)
     => All input chunked into 25 line blocks.
   
-  # The format for the returned chunks is not a simple array of hashes, like get_preview_lines
+  # The format for the returned chunks is not a simple array of hashes, like get_preview_lines, as it also includes per-row errors
   puts importer.get_chunks(2)
     => [{:lines => [{:column_1 => "r1c1", :column_2 => "r1c2", :column_3 => "r1c3"}, {:column_1 => "r2c1", :column_2 => "r2c2", :column_3 => "r2c3"}], :errors => []}, {:lines => [{:column_1 => "r3c1", :column_2 => "r3c2", :column_3 => "r3c3"}, {:column_1 => "r4c1", :column_2 => "r4c2", :column_3 => "r4c3"}], :errors => []}]
 

data/lib/table_importer/copy_and_paste.rb

@@ -2,6 +2,8 @@ module TableImporter
 
   class CopyAndPaste < Source
 
+    attr_accessor :remove_nil_values
+
     def initialize(data)
       @data = assign_data(data[:content])
       @column_separator, @record_separator = assign_separators(data[:column_separator], data[:record_separator])
@@ -9,6 +11,7 @@ module TableImporter
       @mapping = data[:user_headers]
       @compulsory_headers = data[:compulsory_headers]
       @delete_empty_columns = @data.length < 50000
+      @remove_nil_values = data[:remove_nil_values] == true
     end
 
     def assign_data(content)

data/lib/table_importer/csv.rb

@@ -5,12 +5,16 @@ module TableImporter
 
   class CSV < Source
 
+    attr_accessor :remove_nil_values
+
     def initialize(data)
       @headers_present = data[:headers_present] # user has indicated headers are provided
       @column_separator, @record_separator = initialize_separators(data[:column_separator], data[:record_separator])
       @compulsory_headers = data[:compulsory_headers]
       @file = data[:content]
       @delete_empty_columns = File.size(@file) < 100000
+      @remove_nil_values = data[:remove_nil_values] == true
+
       begin
         first_line = get_first_line
         if first_line == 0
@@ -145,9 +149,17 @@ module TableImporter
     end
 
     def default_options(options = {})
-      {:col_sep => @column_separator, :row_sep => @record_separator, :force_simple_split => true, :strip_chars_from_headers => /[\-"]/, :remove_empty_values => false,
-        :verbose => false, :headers_in_file => @headers_present, :convert_values_to_numeric => false,
-        :user_provided_headers => @headers_present ? (@headers == nil || @headers == {} ? nil : @headers) : default_headers(100)}.merge(options)
+      {
+        :col_sep => @column_separator,
+        :row_sep => @record_separator,
+        :force_simple_split => true,
+        :strip_chars_from_headers => /[\-"]/,
+        :remove_empty_values => false,
+        :verbose => false,
+        :headers_in_file => @headers_present,
+        :convert_values_to_numeric => false,
+        :user_provided_headers => @headers_present ? (@headers == nil || @headers == {} ? nil : @headers) : default_headers(100)
+      }.merge(options)
     end
 
     def clean_file(file)

data/lib/table_importer/excel.rb

@@ -2,6 +2,8 @@ module TableImporter
 
   class Excel < RooSpreadsheetSource
 
+    attr_accessor :remove_nil_values
+
     def initialize(data)
       begin
         @type = File.extname(data[:content]) == ".xls" ? "xls" : "xlsx"
@@ -10,6 +12,8 @@ module TableImporter
         @compulsory_headers = data[:compulsory_headers]
         @delete_empty_columns = (File.size(data[:content].path) < 100000)
         @mapping = data[:user_headers]
+        @remove_nil_values = data[:remove_nil_values] == true
+
         raise TableImporter::EmptyFileImportError.new if !@file.first_row
         @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
       rescue NoMethodError

data/lib/table_importer/google.rb

@@ -2,14 +2,21 @@ module TableImporter
 
   class Google < RooSpreadsheetSource
 
+    attr_accessor :remove_nil_values
+
     def initialize(data)
       begin
-        @headers_present = data[:headers_present]
         @file = get_file(data[:content].split(", ")[1], data[:content].split(", ")[0])
+
+        @headers_present    = data[:headers_present]
         @compulsory_headers = data[:compulsory_headers]
+        @mapping            = data[:user_headers] if data[:user_headers].present?
+        @remove_nil_values  = data[:remove_nil_values] == true
+
         @delete_empty_columns = false
-        @mapping = data[:user_headers] if data[:user_headers].present?
+
         raise TableImporter::EmptyFileImportError.new if !@file.first_row
+
         @headers = @headers_present ? @file.row(1).map.with_index { |header, index| header.present? ? header.to_sym : "column_#{index}"} : default_headers
       rescue NoMethodError
         raise TableImporter::HeaderMismatchError.new
@@ -18,7 +25,7 @@ module TableImporter
 
     def get_file(file_key, access_token)
       begin
-        Roo::Google.new(file_key, {:access_token => access_token})
+        Roo::Google.new(file_key, { :access_token => access_token })
       rescue TypeError
         raise TableImporter::IncorrectFileError.new
       end

data/lib/table_importer/source.rb

@@ -1,10 +1,15 @@
 require 'roo'
+require 'roo-xls'
+require 'roo-google'
 
 module TableImporter
 
   class Source
 
-    SEPARATORS = {comma: ",", space: " ", tab: "\t", newline_mac: "\n", semicolon: ";", newline_windows: "\r\n", old_newline_mac: "\r"}
+
+    NIL_VALUES = %w( NULL null nil undefined )
+
+    SEPARATORS = { comma: ",", space: " ", tab: "\t", newline_mac: "\n", semicolon: ";", newline_windows: "\r\n", old_newline_mac: "\r" }
 
     def initialize (data)
       case data[:type]
@@ -108,26 +113,34 @@ module TableImporter
     def check_empty_headers(line, empty_headers)
       line.each do |key, value|
         if value.present? && value.to_s.gsub(/[^A-Za-z0-9]/, '').present?
-          empty_headers.delete(clean_item(key).to_sym)
+          empty_headers.delete(clean_key(key).to_sym)
         end
       end
     end
 
     def line_empty?(line)
       line = clean_line(line)
-      return line, line.all?{ |item_key, item_value| line_item_is_garbage?(item_value)} && line.all?{ |item_key, item_value| line_item_is_garbage?(item_key)}
+      return line, line.all?{ |_, item_value| line_item_is_garbage?(item_value) } && line.all?{ |item_key, _| line_item_is_garbage?(item_key) }
     end
 
     def clean_line(line)
       map = {}
-      line.each_pair do |key,value|
-        map[clean_item(key).to_sym] = clean_item(value)
+      line.each_pair do |key, value|
+        map[clean_key(key).to_sym] = clean_value(value)
+      end
+      return map
+    end
+
+    def clean_value(item)
+      if remove_nil_values == true && NIL_VALUES.include?(item.to_s)
+        return nil
       end
-      map
+
+      return item.to_s.delete("\u0000").to_s.delete("\x00")
     end
 
-    def clean_item(item)
-      item.to_s.delete("\u0000").to_s.delete("\x00")
+    def clean_key(item)
+      return item.to_s.delete("\u0000").to_s.delete("\x00")
     end
 
     def check_compulsory_headers?(line, compulsory_headers)

data/lib/table_importer/version.rb

@@ -1,3 +1,3 @@
 module TableImporter
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/spec/files/csv/null_values.csv

@@ -0,0 +1,10 @@
+Country;Medium;Salutation;First name;Last name;Email;Phone number;Tags
+Hong Kong;Men's Uno (Hong Kong Edition);Ms;Noelle;NULL;info@chicgroups.com;-28544930;Ressort_Business
+Hong Kong;Hong Kong Commercial Daily;Mr;Ching Wai
+Hong Kong;South China Morning Post;Mr;John;Cremer;john.cremer@scmp.com;-25651370;Ressort_Business
+Hong Kong;Hong Kong Economic Journal;Mr;Toby;Yiu;tcsyiu@hkej.com;-28566702;Ressort_Business
+Hong Kong;Hong KongTrader;Ms;Regina;Deluna;regina.deluna@hktdc.org;-25843524;Ressort_Business
+Hong Kong;GS1 Hong Kong;Ms;Stella;Cheang;info@gs1hk.org;-28611967;Ressort_Business
+Hong Kong;Playtimes;Ms;Jo;Allun;jo.allum@ppp.com.hk;-22018864;gfgsdfgs;Ressort_Business
+Hong Kong;China Tourism;Ms;Mickey;Ching;edit-c@hkctp.com.hk;-25617149;Ressort_Business
+Hong Kong;Metro Daily (Hong Kong);Mr;Jeff;Lee;news@metrohk.com.hk;-31960748;Ressort_Business

data/spec/sources/csv_spec.rb

@@ -131,4 +131,17 @@ describe TableImporter::Source do
       @source = nil
     end
   end
+
+  context 'when source has NULL values in it' do
+
+    it "Skips the null values if specified" do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/null_values.csv"].join), :headers_present => false, :user_headers => { "email" => "5" }, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => { email: true }, remove_nil_values: true })
+      expect(source.get_chunks[0][:lines].first[:column_4]).to eql(nil)
+    end
+
+    it "Doesn't skip the null values if not specified" do
+      source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/null_values.csv"].join), :headers_present => false, :user_headers => { "email" => "5" }, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => { email: true }, remove_nil_values: false })
+      expect(source.get_chunks[0][:lines].first[:column_4]).to eql("NULL")
+    end
+  end
 end

data/spec/sources/excel_spec.rb

@@ -238,5 +238,18 @@ describe TableImporter::Source do
         @source = nil
       end
     end
+
+    context 'when source has NULL values in it' do
+
+      it "Skips the null values if specified" do
+        source = TableImporter::Source.new({ content: File.open([Dir.pwd, "/spec/files/excel/null_values.xls"].join), :headers_present => false, :user_headers => { "email" => "0" }, :type => "xls", :column_separator => "", :record_separator => "", :compulsory_headers => { email: true }, remove_nil_values: true })
+        expect(source.get_preview_lines.first[:column_2]).to eql(nil)
+      end
+
+      it "Doesn't skip the null values if not specified" do
+        source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/null_values.xls"].join), :headers_present => false, :user_headers => { "email" => "0" }, :type => "xls", :column_separator => "⁄", :record_separator => "", :compulsory_headers => { email: true }, remove_nil_values: false })
+        expect(source.get_preview_lines.first[:column_2]).to eql("NULL")
+      end
+    end
   end
 end

data/spec/sources/google_spec.rb

@@ -12,12 +12,11 @@ describe TableImporter::Source do
           :content => 'CLIENT_ID, ACCESS_TOKEN',
           :headers_present => true,
           :user_headers => nil,
-          :user_headers => nil,
           :type => "google",
           :column_separator => "",
           :record_separator => "",
           :compulsory_headers =>
-            {:email => true}
+            { :email => true }
           })
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_importer
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Nick Dowse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-15 00:00:00.000000000 Z
+date: 2018-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spreadsheet
@@ -222,6 +222,7 @@ files:
 - spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
+- spec/files/csv/null_values.csv
 - spec/files/csv/partway.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
@@ -229,6 +230,7 @@ files:
 - spec/files/excel/empty_lines.xlsx
 - spec/files/excel/empty_lines_at_start.xlsx
 - spec/files/excel/no_content.xlsx
+- spec/files/excel/null_values.xls
 - spec/files/excel/one_line.xls
 - spec/files/excel/premapped_1.xls
 - spec/files/excel/premapped_2.xls
@@ -273,6 +275,7 @@ test_files:
 - spec/files/csv/empty_lines_at_start.csv
 - spec/files/csv/mexico2013_pressdoc.csv
 - spec/files/csv/no_content.csv
+- spec/files/csv/null_values.csv
 - spec/files/csv/partway.csv
 - spec/files/csv/with_headers.csv
 - spec/files/csv/without_headers.csv
@@ -280,6 +283,7 @@ test_files:
 - spec/files/excel/empty_lines.xlsx
 - spec/files/excel/empty_lines_at_start.xlsx
 - spec/files/excel/no_content.xlsx
+- spec/files/excel/null_values.xls
 - spec/files/excel/one_line.xls
 - spec/files/excel/premapped_1.xls
 - spec/files/excel/premapped_2.xls