ndr_import 4.1.4 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ce8655ee8ebe40d7df275d8a7a029e5f8a778b5dca6ee94b2a7df857b84fa85
-  data.tar.gz: 128b840a7a26d6f245ee6532b3ec371d190d1c3b7aaadfe233a3039035bab7b8
+  metadata.gz: c3697b01f732fdef03d66858b9f699b0c95673824332a1d4f2d8fa2b845aed24
+  data.tar.gz: 87bb849993db749b7b9a58d2a47be6b1f4ecd5a928f9541f97529452836f7fb1
 SHA512:
-  metadata.gz: 352e35cc8ac5ca10739864fae020ff7960a932da0e91bd1f7d493af841559b40fc8dc1ebafd8608683680bde4a40314a9138e4ea1cfc962cbeb97e4e216bceec
-  data.tar.gz: 9702cecbf759bb74389870d2679a07a3f57d1e8aaac04c5a82218b8e64b4380b42ddc8575be8eb73f27ee090bdc028a84ae6e36ed4c47ba2243ec3c927815993
+  metadata.gz: 445825462bf049b2d8ebacd391695176302924e8b12a5e08c283e9b5ed32bb9600befa48c6777ed038f8a2f6c257604d3c86b899b795fd8d13e7980f85b7911f
+  data.tar.gz: a9c796eb0f8d1914df8fc1dc5c943c49ab7de68f7f109e2e917637a88262d334ef1e1a47f5f23571a910e51664109851f4748988c0b8592cf3c376f8d8c5b0cd

data/.gitignore

@@ -1,3 +1,4 @@
+/.rubocop-https---raw-githubusercontent-com-PublicHealthEngland-ndr-dev-support-master--rubocop-yml
 /.bundle/
 /.yardoc
 /Gemfile.lock

data/.rubocop.yml

@@ -1,32 +1 @@
-Rails:
-  Enabled: true
-
-# Multi-line method chaining should be done with trailing dots.
-Style/DotPosition:
-  EnforcedStyle: trailing
-
-HashSyntax:
-  EnforcedStyle: hash_rockets
-  SupportedStyles:
-    - ruby19
-    - hash_rockets
-
-# Override HoundCI's preference for double quotes:
-Style/StringLiterals:
-  EnforcedStyle: single_quotes
-
-# Would enforce "-> { }", which we cannot use yet.
-Lambda:
-  Enabled: false
-
-# ...
-LineLength:
-  Max: 99
-
-# Relax some of the cops for tests
-Metrics/ClassLength:
-  Exclude:
-    - test/**/*.rb
-Metrics/MethodLength:
-  Exclude:
-    - test/**/*.rb
+inherit_from: 'https://raw.githubusercontent.com/PublicHealthEngland/ndr_dev_support/master/.rubocop.yml'

data/.travis.yml

@@ -1,19 +1,21 @@
 language: ruby
-
 rvm:
-  - 2.2.5
-  - 2.3.1
-
+- 2.3
+- 2.4
+- 2.5
 gemfile:
-  - gemfiles/Gemfile.rails32
-  - gemfiles/Gemfile.rails41
-  - gemfiles/Gemfile.rails42
-  - gemfiles/Gemfile.rails50
-
+- gemfiles/Gemfile.rails42
+- gemfiles/Gemfile.rails50
+- gemfiles/Gemfile.rails51
+- gemfiles/Gemfile.rails52
 sudo: false
-
 cache: bundler
-
-before_script: "bundle update"
-
-script: "bundle exec rake test"
+before_script: bundle update
+script: bundle exec rake test
+notifications:
+  slack:
+    on_pull_requests: false
+    on_success: always
+    on_failure: always
+    rooms:
+      secure: NMd9N/jNbECT/fCUBxPDKy4IMpUnRdMhobwa+9kDRaWYbRAwIkHLd8sI1l63qx7O03SlDpsHLLckjayMKS35gO/wRmgC7T8k9ppW75Xs5D4jR/j3FdTFawnXF+wVx0ddQrpH0K8oftPX1euPEllIh7+bJUYct/Siv0UX5eP+qyHT0gKtbDlI2CMKOcOHE8Q/zwjx0mLVSj/0U+HRx5+sMEDcwPupzhCYLdIAtenHsZ2Zmo2uSqOzK9Ruf3UsIh3gVSKXWGvXQzG8bIxogIJbEbwIk/gQyMr7UXwgNzhltPz1CqOrrTmSonrd3HbLVajJvDWPMBPGDZrUyipiW/MOUKpXgd8CsL46o4icYIZIAYEWLsn/AJkrA+UQZ8S5poR06jNH8xrI0eUQsu6maWhJ/rPfzwb6x2hCCyk2iQsGtO+SPCUrWnT1T3ir8H0HStD+AZ4/yz4cdWlClchko2ZfqS0q3Yv5jF3K4iVOF4GOUo01Ft+LDU3sPC7Ngq51bejoaLTzexiOfEaF0kKvB8XNXYYjV5ATNJ8XWQR7ISJ+na4FCwt4KaKtC864TWWDTxaz6lfr0B3Wf0lZLH22+BhpzrfPWFo3YGyMsOcgPU0rPHrtAzks55rqCTQEOaKTw4rfEmuMEFJK/y80GYS2jTPu0nCnJo4NdV1fVuSSD8iUFAM=

data/code_safety.yml

@@ -3,19 +3,19 @@ file safety:
   ".gitignore":
     comments: whole file re-reviewed
     reviewed_by: josh.pencheon
-    safe_revision: d22b012a99fb1f8d94acbe445b6170f251a7b4a2
+    safe_revision: 3ef51291c413fd5772d61a8394359146a02ae628
   ".hound.yml":
     comments: 
     reviewed_by: timgentry
     safe_revision: df3c0be0e655f4fc95c86d4a57bda6eef1fb8955
   ".rubocop.yml":
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: ac40ad6e6a9d79d139ba063681f3fe6093c73b14
+    reviewed_by: josh.pencheon
+    safe_revision: b09e268ff9c8349b914aa1b7ba888e1d39f97e4a
   ".travis.yml":
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f9de73c5988557e29ea73c279f2fdae19ffa8bb4
+    safe_revision: 661b5b8e71572bba28aa92fb95aa218e3f8444f4
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -40,14 +40,6 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5a7f26cecaabab20f4e666776f9166dcc3fa6bfe
-  gemfiles/Gemfile.rails32:
-    comments: 
-    reviewed_by: timgentry
-    safe_revision: 0c5967732100f43e7b60a97a7f0aae60d3641791
-  gemfiles/Gemfile.rails41:
-    comments: 
-    reviewed_by: timgentry
-    safe_revision: 0c5967732100f43e7b60a97a7f0aae60d3641791
   gemfiles/Gemfile.rails42:
     comments: 
     reviewed_by: timgentry
@@ -56,6 +48,14 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 738577adf8eeacc7a000d08289bd7ae7bf3b5849
+  gemfiles/Gemfile.rails51:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: eedd8f5588edd2a9463250fcddc5602f62ab26f1
+  gemfiles/Gemfile.rails52:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: c158dc783b84cab31380708e76e3812544cc1c2f
   lib/ndr_import.rb:
     comments: 
     reviewed_by: timgentry
@@ -63,7 +63,7 @@ file safety:
   lib/ndr_import/csv_library.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 20939133f61ed102feb2a841b6c43b86f385b972
+    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
   lib/ndr_import/file/all.rb:
     comments: 
     reviewed_by: timgentry
@@ -75,7 +75,7 @@ file safety:
   lib/ndr_import/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 62702f103dd41c188684cfda73a94401b130cd01
+    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
   lib/ndr_import/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -102,8 +102,8 @@ file safety:
     safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: 06fd16643d02957ce856728e4911143c8e0178dc
+    reviewed_by: josh.pencheon
+    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -119,15 +119,15 @@ file safety:
   lib/ndr_import/helpers/file/xml.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4d20f3483a073b8de681b06355e7c532dc68f674
+    safe_revision: d2245268ec6a0e4f60c521d171a820f299632c4f
   lib/ndr_import/helpers/file/zip.rb:
     comments: 
     reviewed_by: timgentry
     safe_revision: 6c6f204fab2f4232d81cb76aa523c26b0c490ae7
   lib/ndr_import/mapper.rb:
-    comments: digests changed on branch
+    comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 0627a1e730c4a3ede183ddac90fb8eef7e8da5dd
+    safe_revision: fbd4e1bfda8acd9ae026601dc4ecdabca5be4bc7
   lib/ndr_import/mapping_error.rb:
     comments: 
     reviewed_by: timgentry
@@ -161,10 +161,9 @@ file safety:
     reviewed_by: josh.pencheon
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   lib/ndr_import/table.rb:
-    comments: 'uses File.basename. Revision 5c8b12ca9b88a3bba972aaa2e38cdae25403e033:
-      digests changed on branch'
+    comments: uses File.basename
     reviewed_by: josh.pencheon
-    safe_revision: 5c8b12ca9b88a3bba972aaa2e38cdae25403e033
+    safe_revision: 14775e7262dbec3b6d0f69930ff526f856a94214
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -172,11 +171,11 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: fa2ddc1ad2bda3c2a262a4cc82c78e1983718052
+    safe_revision: 1df87077495c1f595023bdbbd1e0bfb220dc5da5
   ndr_import.gemspec:
-    comments: digests changed on branch
+    comments: 
     reviewed_by: josh.pencheon
-    safe_revision: bd53fc8ae3ad5fb9caab9e79fe05fe578202849e
+    safe_revision: b3f39d7243c41a44594336c3feed3b29c2386a0e
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -184,19 +183,19 @@ file safety:
   test/file/delimited_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 62702f103dd41c188684cfda73a94401b130cd01
+    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
   test/file/excel_test.rb:
     comments: 
     reviewed_by: joshpencheon
     safe_revision: 334d53bb9e8698bba3d92b4dbd71b36faacae629
   test/file/pdf_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
+    reviewed_by: josh.pencheon
+    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
   test/file/registry_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
+    reviewed_by: josh.pencheon
+    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -219,8 +218,8 @@ file safety:
     safe_revision: 377c060bd1e012d6e162a4ab8923a3609aacdf57
   test/helpers/file/pdf_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
+    reviewed_by: josh.pencheon
+    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
   test/helpers/file/word_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -234,10 +233,9 @@ file safety:
     reviewed_by: josh.pencheon
     safe_revision: 9abdd6ced1d0c90ce8dd88abee4eb6472c7ff0d6
   test/mapper_test.rb:
-    comments: 'exposes Mapper internals to test them. Revision f707f42686cabe21d80e2547e9a24fe094b6c7d1:
-      digests changed on branch'
+    comments: exposes Mapper internals to test them
     reviewed_by: josh.pencheon
-    safe_revision: f707f42686cabe21d80e2547e9a24fe094b6c7d1
+    safe_revision: 3cea78767a73fd0d0ae64a4af3f07389e45349e9
   test/non_tabular/mapping_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -304,8 +302,8 @@ file safety:
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
   test/resources/hello_world.pdf:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: ed2360bd226dd6babb7683f2f888ba9547ff7e53
+    reviewed_by: josh.pencheon
+    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
   test/resources/hello_world.txt:
     comments: 
     reviewed_by: timgentry
@@ -404,12 +402,12 @@ file safety:
     safe_revision: 3c7f827d17aacbf7b811eea67e27553f3b039070
   test/table_test.rb:
     comments: 
-    reviewed_by: joshpencheon
-    safe_revision: e70ece909c44b26208eaff3cf27ada7145e03f11
+    reviewed_by: josh.pencheon
+    safe_revision: 14775e7262dbec3b6d0f69930ff526f856a94214
   test/test_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 53cc0af4321b64746367231b65821fda0f8d8a0a
+    safe_revision: 72750275da2193cc9ed362da85b88e9b5054f2d7
   test/universal_importer_helper_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/gemfiles/Gemfile.rails51

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec path: '..'
+
+# Minitest 5.10.2 doesn't work with Rails version
+gem 'minitest', '< 5.10.2'

data/gemfiles/Gemfile.rails52

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec path: '..'
+
+gem 'activesupport', '~> 5.2.0'
+
+gem 'ndr_support', tag: 'v5.4.0', git: 'https://github.com/PublicHealthEngland/ndr_support'

data/lib/ndr_import/csv_library.rb

@@ -10,6 +10,14 @@ class << CSVLibrary
     not self.const_defined?(:Reader)
   end
 
+  # Ensure that we can pass "mode" straight through the underlying IO object
+  def foreach(path, **options, &block)
+    return to_enum(__method__, path, options) unless block
+    open(path, options.delete(:mode) || 'r', options) do |csv|
+      csv.each(&block)
+    end
+  end
+
   def write_csv_to_string(data)
     self.generate do |csv|
       data.each { |line| csv << line }

data/lib/ndr_import/file/delimited.rb

@@ -46,17 +46,17 @@ module NdrImport
       def determine_encodings!(safe_path)
         # delimiter encoding => # FasterCSV encoding string
         supported_encodings = {
-          'UTF-8'        => 'bom|utf-8',
-          'Windows-1252' => 'windows-1252:utf-8'
+          'UTF-8'        => 'r:bom|utf-8',
+          'Windows-1252' => 'r:windows-1252:utf-8'
         }
 
         successful_options = nil
-        supported_encodings.each do |delimiter_encoding, csv_encoding|
+        supported_encodings.each do |delimiter_encoding, access_mode|
           begin
             col_sep = @options['col_sep']
             options = {
               :col_sep  => (col_sep || ',').force_encoding(delimiter_encoding),
-              :encoding => csv_encoding
+              :mode     => access_mode
             }
 
             row_num = 0

data/lib/ndr_import/helpers/file/delimited.rb

@@ -48,16 +48,16 @@ module NdrImport
         def determine_encodings!(safe_path, col_sep = nil)
           # delimiter encoding => # FasterCSV encoding string
           supported_encodings = {
-            'UTF-8'        => 'bom|utf-8',
-            'Windows-1252' => 'windows-1252:utf-8'
+            'UTF-8'        => 'r:bom|utf-8',
+            'Windows-1252' => 'r:windows-1252:utf-8'
           }
 
           successful_options = nil
-          supported_encodings.each do |delimiter_encoding, csv_encoding|
+          supported_encodings.each do |delimiter_encoding, access_mode|
             begin
               options = {
-                :col_sep  => (col_sep || ',').force_encoding(delimiter_encoding),
-                :encoding => csv_encoding
+                :col_sep => (col_sep || ',').force_encoding(delimiter_encoding),
+                :mode    => access_mode
               }
 
               row_num = 0

data/lib/ndr_import/helpers/file/xml.rb

@@ -31,13 +31,16 @@ module NdrImport
           # We let slide any warnings about xml declared as one of our
           # auto encodings, but parsed as UTF-8:
           encoding_pattern = AUTO_ENCODINGS.map { |name| Regexp.escape(name) }.join('|')
-          encoding_warning = /\A(\d+:\d+: FATAL: )?Document labelled (#{encoding_pattern}) but has UTF-8 content\z/
+          encoding_warning = /Document labelled (#{encoding_pattern}) but has UTF-8 content\z/
           fatal_errors     = document.errors.select do |error|
             error.fatal? && (encoding_warning !~ error.message)
           end
 
           return unless fatal_errors.any?
-          fail Nokogiri::XML::SyntaxError, "The file had #{fatal_errors.length} fatal error(s)!"
+          raise Nokogiri::XML::SyntaxError, <<~MSG
+            The file had #{fatal_errors.length} fatal error(s)!"
+            #{fatal_errors.join("\n")}
+          MSG
         end
       end
     end

data/lib/ndr_import/mapper.rb

@@ -177,11 +177,14 @@ module NdrImport::Mapper
         raise e2
       end
     elsif field_mapping.include?(Strings::CLEAN)
-      return original_value.blank? ? nil : original_value.clean(field_mapping[Strings::CLEAN])
+      return nil if original_value.blank?
+
+      cleaners = Array(field_mapping[Strings::CLEAN])
+      return cleaners.inject(original_value) { |a, e| a.clean(e) }
     elsif field_mapping.include?(Strings::MAP)
-      return field_mapping[Strings::MAP] ? field_mapping[Strings::MAP][original_value] : nil
+      return field_mapping[Strings::MAP].fetch(original_value, original_value)
     elsif field_mapping.include?(Strings::MATCH)
-      # WARNING:TVB Thu Aug  9 17:09:25 BST 2012 field_mapping['match'] regexp
+      # WARNING:TVB Thu Aug  9 17:09:25 BST 2012 field_mapping[Strings::MATCH] regexp
       # may need to be escaped
       matches = Regexp.new(field_mapping[Strings::MATCH]).match(original_value)
       return matches[1].strip if matches && matches.size > 0

data/lib/ndr_import/table.rb

@@ -62,13 +62,16 @@ module NdrImport
       return enum_for(:process_line, line) unless block
 
       if @row_index < header_lines
-        validate_header(line, @columns)
+        consume_header_line(line, @columns)
       else
-        fail_unless_header_complete(@columns)
         transform_line(line, @row_index, &block)
       end
 
       @row_index += 1
+
+      # We've now seen enough lines to have consumed a valid header; is this the case?
+      fail_unless_header_complete(@columns) if @row_index == header_lines
+
       @notifier.try(:processed, @row_index)
     end
 
@@ -163,20 +166,21 @@ module NdrImport
       fail ArgumentError, "Unrecognised options: #{unrecognised_options.inspect}"
     end
 
-    # if there is a header, then check the column headings are as expected in the correct order
-    def validate_header(line, column_mappings)
+    # Process `line` as a candidate header row. Header validation is done
+    # lazily, once all expected header lines have been consumed.
+    def consume_header_line(line, column_mappings)
       columns = column_names(column_mappings)
 
-      unless line.length == columns.length
-        fail "Number of columns does not match; expected #{columns.length}, got #{line.length}!"
-      end
-
-      header_guess = line.map(&:downcase)
+      header_guess = line.map { |column| column.to_s.downcase }
 
+      # The "best guess" is only if/when the header eventually deemed to be
+      # invalid - in which case, it builds the informative error message.
       @header_best_guess = header_guess if header_guess.any?(&:present?)
       @header_valid      = true         if header_guess == columns
     end
 
+    # Once all header lines have been consumed, blow up in a
+    # helpful fashion if no valid row was found:
     def fail_unless_header_complete(column_mappings)
       return unless header_lines > 0 && !header_valid?
 

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '4.1.4'.freeze
+  VERSION = '5.0.0'.freeze
 end

data/ndr_import.gemspec

@@ -19,16 +19,16 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.1'
-  spec.add_dependency 'ndr_support', '>= 4.1.2', '< 6'
+  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.3'
+  spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
   spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1'
   spec.add_dependency 'roo', '~> 2.0'
 
-  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.2'
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'spreadsheet', '1.0.3'
-  spec.add_dependency 'pdf-reader', '1.2.0'
+  spec.add_dependency 'pdf-reader', '1.2.0' # Raises warnings on Ruby 2.4+
   spec.add_dependency 'msworddoc-extractor', '0.2.0'
 
   spec.required_ruby_version = '>= 2.2'
@@ -37,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
-  spec.add_development_dependency 'ndr_dev_support', '~> 1.1', '>= 1.1.3'
+  spec.add_development_dependency 'ndr_dev_support', '~> 3.0'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'guard-test'

data/test/file/delimited_test.rb

@@ -143,7 +143,7 @@ module NdrImport
         file_path = @permanent_test_files.join('normal.csv')
         handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
 
-        handler.expects(determine_encodings!: { encoding: 'bom|utf-8', col_sep: ',' }).once
+        handler.expects(determine_encodings!: { mode: 'r:bom|utf-8', col_sep: ',' }).once
 
         2.times do
           handler.tables.each do |tablename, sheet|

data/test/file/pdf_test.rb

@@ -15,7 +15,7 @@ module NdrImport
         handler.tables.each do |tablename, sheet|
           assert_nil tablename
           assert_instance_of Enumerator, sheet
-          assert_equal ['Hello World'], sheet.to_a
+          assert_equal ['Hello World ', 'Goodbye Universe ', ' '], sheet.to_a
         end
       end
 

data/test/file/registry_test.rb

@@ -35,7 +35,7 @@ module NdrImport
         tables.each do |tablename, sheet|
           assert_nil tablename
           assert_instance_of Enumerator, sheet
-          assert_equal ['Hello World'], sheet.to_a
+          assert_equal ['Hello World ', 'Goodbye Universe ', ' '], sheet.to_a
         end
       end
 

data/test/helpers/file/pdf_test.rb

@@ -15,7 +15,7 @@ class PdfTest < ActiveSupport::TestCase
 
   test 'read_pdf_file helper should read pdf file' do
     file_content = @importer.send(:read_pdf_file, @permanent_test_files.join('hello_world.pdf'))
-    assert_equal file_content, ['Hello World']
+    assert_equal ['Hello World ', 'Goodbye Universe ', ' '], file_content
   end
 
   test 'read_pdf_file helper should raise exception on invalid pdf file' do

data/test/mapper_test.rb

@@ -21,6 +21,7 @@ class MapperTest < ActiveSupport::TestCase
   clean_ethniccategory_mapping = { 'clean' => :ethniccategory }
   clean_icd_mapping = { 'clean' => :icd }
   clean_opcs_mapping = { 'clean' => :code_opcs }
+  clean_code_and_upcase_mapping = { 'clean' => [:code, :upcase] }
   map_mapping = { 'map' => { 'A' => '1' } }
   replace_mapping = { 'replace' => { '.0' => '' } }
   daysafter_mapping = { 'daysafter' => '2012-05-16' }
@@ -102,6 +103,21 @@ class MapperTest < ActiveSupport::TestCase
         priority: 1
   YML
 
+  cross_populate_map_reverse_priority_mapping = YAML.load <<-YML
+    - column: referringclinicianname
+      mappings:
+      - field: consultantname
+      - field: consultantcode
+        priority: 1
+        map:
+          "Bob Fossil": "C5678"
+          "Bolo": ""
+    - column: referringcliniciancode
+      mappings:
+      - field: consultantcode
+        priority: 2
+  YML
+
   cross_populate_order_mapping = YAML.load <<-YML
     - column: referringclinicianname
       mappings:
@@ -282,7 +298,7 @@ class MapperTest < ActiveSupport::TestCase
   end
 
   test 'map should return nil' do
-    assert_nil TestMapper.new.mapped_value('B', map_mapping)
+    assert_equal 'B', TestMapper.new.mapped_value('B', map_mapping)
   end
 
   test 'map should return correct date format' do
@@ -333,11 +349,16 @@ class MapperTest < ActiveSupport::TestCase
     assert_equal 'U212 Y973', TestMapper.new.mapped_value('U212,Y973,X1', clean_opcs_mapping)
     assert_equal '', TestMapper.new.mapped_value('98', clean_opcs_mapping)
     assert_equal '', TestMapper.new.mapped_value('TooLong', clean_opcs_mapping)
-    assert_equal nil, TestMapper.new.mapped_value('', clean_opcs_mapping)
+    assert_nil TestMapper.new.mapped_value('', clean_opcs_mapping)
     assert_equal 'ABCD', TestMapper.new.mapped_value('AbcD', clean_opcs_mapping)
     assert_equal '1234', TestMapper.new.mapped_value('1234', clean_opcs_mapping)
   end
 
+  test 'map should use multiple cleans' do
+    assert_equal 'U3 Y2 X1',
+                 TestMapper.new.mapped_value('u3,y2,x1', clean_code_and_upcase_mapping)
+  end
+
   test 'map should handle array original value' do
     original_value = ['C9999998', %w(Addenbrookes RGT01)]
     mapped_value = TestMapper.new.mapped_line(original_value, replace_array_mapping)
@@ -373,7 +394,7 @@ class MapperTest < ActiveSupport::TestCase
   test 'line mapping should create valid hash with blank cleaned value' do
     assert_equal '', TestMapper.new.mapped_value('98', clean_opcs_mapping)
     line_hash = TestMapper.new.mapped_line(['98'], simple_mapping_with_clean_opcs)
-    assert_equal nil, line_hash['primaryprocedures']
+    assert_nil line_hash['primaryprocedures']
     assert_equal '98', line_hash[:rawtext]['primaryprocedures']
   end
 
@@ -453,13 +474,52 @@ class MapperTest < ActiveSupport::TestCase
     assert_equal 'C5678', line_hash['consultantcode']
   end
 
+  test 'should create valid hash with used cross populate with map with no p2 map' do
+    line_hash = TestMapper.new.mapped_line(['something', ''], cross_populate_map_mapping)
+    assert_equal 'something', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'something', line_hash['consultantname']
+    assert_equal 'something', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate with map with p1 mapped' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', 'P2'],
+                                           cross_populate_map_reverse_priority_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'P2', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'C5678', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate with map with p1 mapped out' do
+    line_hash = TestMapper.new.mapped_line(['Bolo', 'P2'],
+                                           cross_populate_map_reverse_priority_mapping)
+    assert_equal 'Bolo', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'P2', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bolo', line_hash['consultantname']
+    assert_equal 'P2', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate with map with p1 no map' do
+    line_hash = TestMapper.new.mapped_line(['something', 'P2'],
+                                           cross_populate_map_reverse_priority_mapping)
+    assert_equal 'something', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'P2', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'something', line_hash['consultantname']
+    assert_equal 'something', line_hash['consultantcode']
+  end
+
   test 'should create valid hash with used cross populate without map' do
     line_hash = TestMapper.new.mapped_line(['Bob Smith', ''], cross_populate_map_mapping)
     assert_equal 'Bob Smith', line_hash[:rawtext]['referringclinicianname']
     assert_equal '', line_hash[:rawtext]['referringcliniciancode']
 
     assert_equal 'Bob Smith', line_hash['consultantname']
-    assert_nil line_hash['consultantcode']
+    assert_equal 'Bob Smith', line_hash['consultantcode']
   end
 
   test 'should create valid hash with used cross populate without map and priorities' do

data/test/table_test.rb

@@ -100,11 +100,22 @@ class TableTest < ActiveSupport::TestCase
                                  :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
 
     output = []
-    table.process_line(%w(CARROT POTATO)).each do |klass, fields, index|
-      output << [klass, fields, index]
-    end
+    table.process_line(%w(ONE TWO)).each { |*stuff| output << stuff }
+    table.process_line(%w(CARROT POTATO)).each { |*stuff| output << stuff }
 
-    assert_equal [], output
+    expected_output = [
+      ['SomeTestKlass', { :rawtext => { 'one' => 'CARROT', 'two' => 'POTATO' } }, 1]
+    ]
+    assert_equal expected_output, output
+  end
+
+  def test_process_line_with_unsatisifed_header
+    table = NdrImport::Table.new(:header_lines => 1, :footer_lines => 0,
+                                 :klass => 'SomeTestKlass',
+                                 :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
+
+    exception = assert_raises(RuntimeError) { table.process_line(%w(ONE THREE)).to_a }
+    assert_equal 'Header is not valid! missing: ["two"] unexpected: ["three"]', exception.message
   end
 
   def test_transform_line
@@ -243,11 +254,59 @@ class TableTest < ActiveSupport::TestCase
     assert_equal expected_output, output
   end
 
-  def test_invalid_header_length
+  def test_varying_header_line_lengths_with_valid_header_row_including_nils
+    lines = [
+      [nil] << 'RIGHTALIGN1' << 'RIGHTALIGN2',
+      %w(ONE TWO),
+      %w(LEFTALIGN) << nil,
+      %w(CENTRE1) << nil << 'CENTRE2',
+      %w(UNO DOS)
+    ].each
+
+    table = NdrImport::Table.new(:header_lines => 4, :footer_lines => 0,
+                                 :klass => 'SomeTestKlass',
+                                 :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
+
+    output = []
+    table.transform(lines).each do |klass, fields, index|
+      output << [klass, fields, index]
+    end
+
+    assert table.header_valid?
+
+    expected_output = [['SomeTestKlass', { :rawtext => { 'one' => 'UNO', 'two' => 'DOS' } }, 4]]
+    assert_equal expected_output, output
+  end
+
+  def test_varying_header_line_lengths_with_valid_header_row
     lines = [
       %w(NOTHEADING1 NOTHEADING2 UHOH3 UHOH4),
       %w(ONE TWO),
-      %w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2)
+      %w(DEFINITELYNOTHEADING1 DEFINITELYNOTHEADING2),
+      %w(UNO DOS)
+    ].each
+
+    table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
+                                 :klass => 'SomeTestKlass',
+                                 :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
+
+    output = []
+    table.transform(lines).each do |klass, fields, index|
+      output << [klass, fields, index]
+    end
+
+    assert table.header_valid?
+
+    expected_output = [['SomeTestKlass', { :rawtext => { 'one' => 'UNO', 'two' => 'DOS' } }, 3]]
+    assert_equal expected_output, output
+  end
+
+  def test_varying_header_line_lengths_without_valid_header_row
+    lines = [
+      %w(NOTHEADING1 NOTHEADING2 UHOH3 UHOH4),
+      %w(ONE TWO NOPE),
+      %w(NOT_HERE OR_HERE),
+      %w(UNO DOS)
     ].each
 
     table = NdrImport::Table.new(:header_lines => 3, :footer_lines => 0,
@@ -255,7 +314,10 @@ class TableTest < ActiveSupport::TestCase
                                  :columns => [{ 'column' => 'one' }, { 'column' => 'two' }])
 
     exception = assert_raises(RuntimeError) { table.transform(lines).to_a }
-    assert_match(/expected 2, got 4/, exception.message)
+
+    assert_match(/Header is not valid!/, exception.message)
+    assert_match(/missing: \["one", "two"\]/, exception.message)
+    assert_match(/unexpected: \["not_here", "or_here"\]/, exception.message)
   end
 
   def test_jumbled_header

data/test/test_helper.rb

@@ -25,4 +25,4 @@ NdrImport::StandardMappings.mappings = YAML.load_file(
   File.expand_path(File.dirname(__FILE__) + '/resources/standard_mappings.yml')
 )
 
-require 'mocha/mini_test'
+require 'mocha/minitest'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 4.1.4
+  version: 5.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-20 00:00:00.000000000 Z
+date: 2018-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -19,7 +19,7 @@ dependencies:
         version: 3.2.18
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,14 +29,14 @@ dependencies:
         version: 3.2.18
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.3'
 - !ruby/object:Gem::Dependency
   name: ndr_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.2
+        version: 5.3.2
     - - "<"
       - !ruby/object:Gem::Version
         version: '6'
@@ -46,7 +46,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.2
+        version: 5.3.2
     - - "<"
       - !ruby/object:Gem::Version
         version: '6'
@@ -90,14 +90,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
 - !ruby/object:Gem::Dependency
   name: roo-xls
   requirement: !ruby/object:Gem::Requirement
@@ -216,20 +222,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.3
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.3
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
@@ -317,10 +317,10 @@ files:
 - README.md
 - Rakefile
 - code_safety.yml
-- gemfiles/Gemfile.rails32
-- gemfiles/Gemfile.rails41
 - gemfiles/Gemfile.rails42
 - gemfiles/Gemfile.rails50
+- gemfiles/Gemfile.rails51
+- gemfiles/Gemfile.rails52
 - lib/ndr_import.rb
 - lib/ndr_import/csv_library.rb
 - lib/ndr_import/file/all.rb

data/gemfiles/Gemfile.rails32

@@ -1,5 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec path: '..'
-
-gem 'activesupport', '~> 3.2.18'

data/gemfiles/Gemfile.rails41

@@ -1,5 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec path: '..'
-
-gem 'activesupport', '~> 4.1.0'