smarter_csv 1.1.5 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 042aadb2bc5426a07a64f09e781bccbd728e8052
-  data.tar.gz: ba48c2e303591d4027e05d1208c225381d362857
+  metadata.gz: 251eb1eff306211d163c6be6880ba45e3eb29985
+  data.tar.gz: 9e16d1d4aaab86df7a65d73e6a23ace8f481ba4e
 SHA512:
-  metadata.gz: 58cb92edabb46bdcb48598d4b4b02b5f0f09cc63378e818ac672daf8d722b5fbf1b246df5db262dff306e87943a2bb2bebbb753944adc5449b19cd5a1475c00b
-  data.tar.gz: 31fe30f2b2027274a5252c55b234b120327be6ce652f7ad71232bd8a920e33d30cbae42577fff398d0a61574dc17b3016cd3fa1d520eec3dd4636569cc62860e
+  metadata.gz: e2ad758ddb7d644777df1b9f38b73c7c4c0753d76507834bf68e86394ae017b1f5caa16867efd82f4c0443d4a7c1b4c1d7f704d9c3796e71083bde689ee7aeba
+  data.tar.gz: 8f7126500628a17fda587a05678aca9140d0ebb7ca168fc3144b1d9c00d2be6ec901a43900bd3d558aceb0a8dcfc139aba7347e6dbad01b2ff83c19dc3816763

data/README.md

@@ -193,6 +193,8 @@ The options and the block are optional.
      | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
+     | :required_headers           |   nil    | An array. Eacn of the given headers must be present in the CSV  file,                |
+     |                             |          | or an exception is raised   No validation if nil is given.                           |
      | :remove_unmapped_keys       |   false  | when using :key_mapping option, should non-mapped keys / columns be removed?         |
      | :downcase_header            |   true   | downcase all column headers                                                          |
      | :strings_as_keys            |   false  | use strings instead of symbols as the keys in the result hashes                      |
@@ -293,6 +295,10 @@ Planned in the next releases:
 
 ## Changes
 
+#### 1.2.0 (2018-01-20)
+ * add default validation that a header can only appear once
+ * add option `required_headers`
+
 #### 1.1.5 (2017-11-05)
  * fix issue with invalid byte sequences in header (issue #103, thanks to Dave Myron)
  * fix issue with invalid byte sequences in multi-line data (thanks to Ivan Ushakov)

data/lib/smarter_csv/smarter_csv.rb

@@ -1,8 +1,10 @@
 module SmarterCSV
 
   class HeaderSizeMismatch < Exception; end
-
   class IncorrectOption < Exception; end
+  class DuplicateHeaders < Exception; end
+  class MissingHeaders < Exception; end
+
 
   def SmarterCSV.process(input, options={}, &block)   # first parameter: filename or input object with readline method
     default_options = {:col_sep => ',' , :row_sep => $/ , :quote_char => '"', :force_simple_split => false , :verbose => false ,
@@ -10,7 +12,7 @@ module SmarterCSV
       :convert_values_to_numeric => true, :strip_chars_from_headers => nil , :user_provided_headers => nil , :headers_in_file => true,
       :comment_regexp => /^#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
       :remove_unmapped_keys => false, :keep_original_headers => false, :value_converters => nil, :skip_lines => nil, :force_utf8 => false, :invalid_byte_sequence => '',
-      :auto_row_sep_chars => 500
+      :auto_row_sep_chars => 500, :required_headers => nil
     }
     options = default_options.merge(options)
     options[:invalid_byte_sequence] = '' if options[:invalid_byte_sequence].nil?
@@ -93,6 +95,21 @@ module SmarterCSV
         end
       end
 
+      # header_validations
+      duplicate_headers = []
+      headerA.compact.each do |k|
+        duplicate_headers << k if headerA.select{|x| x == k}.size > 1
+      end
+      raise SmarterCSV::DuplicateHeaders , "ERORR [smarter_csv]: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
+
+      if options[:required_headers] && options[:required_headers].is_a?(Array)
+        missing_headers = []
+        options[:required_headers].each do |k|
+          missing_headers << k unless headerA.include?(k)
+        end
+        raise SmarterCSV::MissingHeaders , "ERORR [smarter_csv]: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
+      end
+
       # in case we use chunking.. we'll need to set it up..
       if ! options[:chunk_size].nil? && options[:chunk_size].to_i > 0
         use_chunks = true

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.1.5"
+  VERSION = "1.2.0"
 end

data/spec/fixtures/duplicate_headers.csv

@@ -0,0 +1,3 @@
+email,firstname,lastname,email,age
+tom@bla.com,Tom,Sawyer,mike@bla.com,34
+eri@bla.com,Eri Chan,tom@bla.com,21

data/spec/fixtures/user_import.csv

@@ -0,0 +1,3 @@
+email,firstname,lastname,manager_email,department
+tom@bla.com,Tom,Sawyer,mike@bla.com,IT
+eri@bla.com,Eri Chan,tom@bla.com,IT

data/spec/smarter_csv/invalid_headers_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'test exceptions for invalid headers' do
+  it 'raises error on duplicate headers' do
+    expect {
+      SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", {})
+    }.to raise_exception(SmarterCSV::DuplicateHeaders)
+  end
+
+  it 'raises error on duplicate given headers' do
+    expect {
+      options = {:user_provided_headers => [:a,:b,:c,:d,:a]}
+      SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
+    }.to raise_exception(SmarterCSV::DuplicateHeaders)
+  end
+
+  it 'raises error on duplicate mapped headers' do
+    expect {
+      # the mapping is right, but the underlying csv file is bad
+      options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
+      SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
+    }.to raise_exception(SmarterCSV::DuplicateHeaders)
+  end
+
+
+  it 'does not raise an error if no required headers are given' do
+    options = {:required_headers => nil} # order does not matter
+    data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
+    data.size.should eq 2
+  end
+
+  it 'does not raise an error if no required headers are given' do
+    options = {:required_headers => []} # order does not matter
+    data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
+    data.size.should eq 2
+  end
+
+  it 'does not raise an error if the required headers are present' do
+    options = {:required_headers => [:lastname,:email,:firstname,:manager_email]} # order does not matter
+    data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
+    data.size.should eq 2
+  end
+
+  it 'raises an error if a required header is missing' do
+    expect {
+      options = {:required_headers => [:lastname,:email,:employee_id,:firstname,:manager_email]} # order does not matter
+      SmarterCSV.process("#{fixture_path}/user_import.csv", options)
+    }.to raise_exception(SmarterCSV::MissingHeaders)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.2.0
 platform: ruby
 authors:
 - 'Tilo Sloboda
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2018-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -56,6 +56,7 @@ files:
 - spec/fixtures/carriage_returns_r.csv
 - spec/fixtures/carriage_returns_rn.csv
 - spec/fixtures/chunk_cornercase.csv
+- spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
@@ -70,6 +71,7 @@ files:
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
 - spec/fixtures/with_dates.csv
@@ -82,6 +84,7 @@ files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/line_ending_spec.rb
@@ -138,6 +141,7 @@ test_files:
 - spec/fixtures/carriage_returns_r.csv
 - spec/fixtures/carriage_returns_rn.csv
 - spec/fixtures/chunk_cornercase.csv
+- spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
@@ -152,6 +156,7 @@ test_files:
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
 - spec/fixtures/with_dates.csv
@@ -164,6 +169,7 @@ test_files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/line_ending_spec.rb