ndr_import 3.1.6 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d79b720417de07bd9f4db6de53c08600ee190ac
-  data.tar.gz: d267da10cc8446fe3272b6aab0e6792e6d77b3e2
+  metadata.gz: ee9c5c86bbcd00af6877225040d6beb01007575d
+  data.tar.gz: 7dabb1d68c3467314e90f916f7ccb573be33ea1a
 SHA512:
-  metadata.gz: 47a472805cd80bede14e404c026dbfdaa4a0b1f7cd1cd48f33d4c83fc27e66b0b41f91f3a64b4fdd77486f302d2ee58a0fd2c3280a9d93e128f1fc55aace7634
-  data.tar.gz: 515cd22f7ae8a0c2b5faa8b3d2b9f5aac08932772637bd189bfe791c3446f2daa42317a87993af8dd642969b289ef0ab880bc647a795ea9e7a9c06f89c6359ad
+  metadata.gz: d9a13b367b3b094c041221b989456af8dfa1571395fd0b4ba11e1fcae10d830ea6b50b89a8f59f7bd72bc49209419c3f9bfbbfe86f9c0b47cbdf600b5e754508
+  data.tar.gz: a8da74311916a789f8203407fc1bfd7db5540940efdf28a1d96b615fa8f52321f7c541596d82a69f8ee704ad50863bf547ec812a86d138d82dfda59d79ba3b9d

data/code_safety.yml

@@ -70,12 +70,12 @@ file safety:
     safe_revision: a8451809db86e0e023b0d93826114a33ff896c4f
   lib/ndr_import/file/delimited.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
+    reviewed_by: joshpencheon
+    safe_revision: 4f7588ef0246bfced14f42b247436507568422a2
   lib/ndr_import/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
-    safe_revision: 334d53bb9e8698bba3d92b4dbd71b36faacae629
+    safe_revision: e108b1635e41e89e4cba816d47514dcd9d0d6433
   lib/ndr_import/file/pdf.rb:
     comments: 
     reviewed_by: timgentry
@@ -98,12 +98,12 @@ file safety:
     safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
   lib/ndr_import/helpers/file/delimited.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 277e5d8159542b539fe7548fb30b449d9273c8f9
+    reviewed_by: joshpencheon
+    safe_revision: 06fd16643d02957ce856728e4911143c8e0178dc
   lib/ndr_import/helpers/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
-    safe_revision: 80ce15cc5ca15019400d4cd24c3353191bec00cc
+    safe_revision: 06fd16643d02957ce856728e4911143c8e0178dc
   lib/ndr_import/helpers/file/pdf.rb:
     comments: 
     reviewed_by: timgentry
@@ -122,8 +122,8 @@ file safety:
     safe_revision: 6c6f204fab2f4232d81cb76aa523c26b0c490ae7
   lib/ndr_import/mapper.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: 74ea81bd4c4119912012ff9f3b7c031e592438f9
+    reviewed_by: joshpencheon
+    safe_revision: 582ce42b27ce3813efa7f93e09928e8f228ba6df
   lib/ndr_import/mapping_error.rb:
     comments: 
     reviewed_by: timgentry
@@ -163,7 +163,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: joshpencheon
-    safe_revision: 749aa9bb66f6e07b52acab1096f6f866034fc676
+    safe_revision: 04a29ed03b6772091d7208403f0acc5032117149
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
@@ -202,12 +202,12 @@ file safety:
     safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
   test/helpers/file/delimited_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: fd2904c1c8166b5284bd6213965401c40f292246
+    reviewed_by: joshpencheon
+    safe_revision: 377c060bd1e012d6e162a4ab8923a3609aacdf57
   test/helpers/file/excel_test.rb:
     comments: 
     reviewed_by: joshpencheon
-    safe_revision: 80ce15cc5ca15019400d4cd24c3353191bec00cc
+    safe_revision: 377c060bd1e012d6e162a4ab8923a3609aacdf57
   test/helpers/file/pdf_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -398,5 +398,5 @@ file safety:
     safe_revision: 6bb588c17d33a604c96c648c6eebfe2d063dfbc0
   test/test_helper.rb:
     comments: 
-    reviewed_by: josh.pencheon
-    safe_revision: c244390ab15a9745d4e8c4cbfbaff67d7ef80cb4
+    reviewed_by: joshpencheon
+    safe_revision: 646c08329cba0ba8ea2c37226cf26434da29c747

data/lib/ndr_import/file/delimited.rb

@@ -32,7 +32,7 @@ module NdrImport
         # By now, we know `encodings` should let us read the whole
         # file succesfully; if there are problems, we should crash.
         CSVLibrary.foreach(safe_path, encodings) do |line|
-          yield line.map(&:to_s) unless line.length <= 5
+          yield line.map(&:to_s)
         end
       end
 

data/lib/ndr_import/file/excel.rb

@@ -64,12 +64,12 @@ module NdrImport
       def xls_rows(workbook, sheet_name)
         return enum_for(:xls_rows, workbook, sheet_name) unless block_given?
 
-        activate_sheet!(workbook, sheet_name)
-        return unless workbook.first_row
-        (workbook.first_row..workbook.last_row).each do |row|
-          activate_sheet!(workbook, sheet_name)
-          columns = (workbook.first_column..workbook.last_column)
-          yield columns.map { |col| cast_excel_value(workbook.cell(row, col)) }
+        return unless workbook.first_row(sheet_name)
+        rows    = workbook.first_row(sheet_name)..workbook.last_row(sheet_name)
+        columns = workbook.first_column(sheet_name)..workbook.last_column(sheet_name)
+
+        rows.each do |row|
+          yield columns.map { |col| cast_excel_value(workbook.cell(row, col, sheet_name)) }
         end
       end
 
@@ -78,9 +78,7 @@ module NdrImport
       def xlsx_rows(workbook, sheet_name)
         return enum_for(:xlsx_rows, workbook, sheet_name) unless block_given?
 
-        activate_sheet!(workbook, sheet_name)
-
-        workbook.each_row_streaming(:pad_cells => true) do |row|
+        workbook.each_row_streaming(sheet: sheet_name, pad_cells: true) do |row|
           yield row.map { |cell| cast_excel_value(cell.value) }
         end
       end
@@ -126,14 +124,6 @@ module NdrImport
         FileUtils.mkdir_p(SafeFile.safepath_to_string(SafeFile.dirname(dest)))
         FileUtils.cp(SafeFile.safepath_to_string(source), SafeFile.safepath_to_string(dest))
       end
-
-      # Sets `sheet_name' to be the default (i.e. active) sheet in `workbook'.
-      # This setting applies to the Roo object, so needs to re-applied if (for example)
-      # streaming rows from multiple sheets simultaneously - something that was problematic
-      # with Workbook#each_with_pagename.
-      def activate_sheet!(workbook, sheet_name)
-        workbook.sheet(sheet_name)
-      end
     end
 
     Registry.register(Excel, 'xls', 'xlsx')

data/lib/ndr_import/helpers/file/delimited.rb

@@ -27,13 +27,6 @@ module NdrImport
           yield nil, delimited_rows(path, col_sep)
         end
 
-        # Deprecated method
-        def each_delimited_table(path, col_sep = nil, &block)
-          Kernel.warn '[warning] each_delimited_table will be deprecated,' \
-                      ' please use delimited_tables instead.'
-          delimited_tables(path, col_sep, &block)
-        end
-
         # Iterate through the file line by line, yielding each one in turn.
         def delimited_rows(path, col_sep = nil)
           return enum_for(:delimited_rows, path, col_sep) unless block_given?
@@ -44,17 +37,10 @@ module NdrImport
           # By now, we know `encodings` should let us read the whole
           # file succesfully; if there are problems, we should crash.
           CSVLibrary.foreach(safe_path, encodings) do |line|
-            yield line.map(&:to_s) unless line.length <= 5
+            yield line.map(&:to_s)
           end
         end
 
-        # Deprecated method
-        def each_delimited_row(path, col_sep = nil, &block)
-          Kernel.warn '[warning] each_delimited_row will be deprecated,' \
-                      ' please use delimited_rows instead.'
-          delimited_rows(path, col_sep, &block)
-        end
-
         private
 
         # Derive the source encoding by trying all supported encodings.

data/lib/ndr_import/helpers/file/excel.rb

@@ -45,13 +45,6 @@ module NdrImport
           end
         end
 
-        # Deprecated method
-        def each_excel_table(path, &block)
-          Kernel.warn '[warning] each_excel_table will be deprecated,' \
-                      ' please use excel_tables instead.'
-          excel_tables(path, &block)
-        end
-
         private
 
         def read_excel_file(path, selected_sheet = nil)
@@ -85,13 +78,6 @@ module NdrImport
           end
         end
 
-        # Deprecated method
-        def each_excel_row(workbook, sheet, &block)
-          Kernel.warn '[warning] each_excel_row will be deprecated,' \
-                      ' please use excel_rows instead.'
-          excel_rows(workbook, sheet, &block)
-        end
-
         # Iterate through an xls sheet line by line, yielding each one in turn.
         def xls_rows(sheet)
           return enum_for(:xls_rows, sheet) unless block_given?
@@ -106,13 +92,6 @@ module NdrImport
           end
         end
 
-        # Deprecated method
-        def each_xls_row(sheet, &block)
-          Kernel.warn '[warning] each_xls_row will be deprecated,' \
-                      ' please use xls_rows instead.'
-          xls_rows(sheet, &block)
-        end
-
         # Iterate through an xlsx sheet line by line, yielding each one in turn.
         # This method uses streaming https://github.com/roo-rb/roo#excel-xlsx-and-xlsm-support
         def xlsx_rows(sheet)
@@ -123,13 +102,6 @@ module NdrImport
           end
         end
 
-        # Deprecated method
-        def each_xlsx_row(sheet, &block)
-          Kernel.warn '[warning] each_xlsx_row will be deprecated,' \
-                      ' please use xlsx_rows instead.'
-          xlsx_rows(sheet, &block)
-        end
-
         def get_excel_sheets_name(path)
           workbook = load_workbook(path)
           workbook.sheets

data/lib/ndr_import/mapper.rb

@@ -21,40 +21,37 @@ module NdrImport::Mapper
 
     [field_mapping['replace']].flatten.each do |field_replacement|
       field_replacement.each do |pattern, replacement|
-        if original_value.is_a?(Array)
-          original_value.map { |val| val.gsub!(pattern, replacement) }
-        else
-          original_value.gsub!(pattern, replacement)
-        end
+        Array(original_value).each { |val| val.gsub!(pattern, replacement) }
       end
     end
   end
 
   # Returns the standard_mapping hash specified
-  # Assumes mappping exists
+  # Assumes mapping exists
   def standard_mapping(mapping_name, column_mapping)
     standard_mapping = NdrImport::StandardMappings.mappings[mapping_name]
-    return nil if standard_mapping.nil?
-
-    # Work on deep copies of reference standard mapping,
-    # as well as the given column_mapping:
-    reference = YAML.load(standard_mapping.to_yaml)
-    incoming  = YAML.load(column_mapping.to_yaml)
+    return unless standard_mapping
 
-    # Merge the 'mappings' key into the reference value,
-    # but replace any other keys using `incoming`:
-    reference['mappings'].concat(incoming.delete('mappings')) if incoming['mappings']
-    reference.merge!(incoming)
+    column_mapping.each_with_object(standard_mapping.dup) do |(key, value), result|
+      if 'mappings' == key
+        # Column mapping appends mappings to the standard mapping...
+        result[key] += value
+      else
+        # ...but overwrites other values.
+        result[key] = value
+      end
+    end
   end
 
   # This takes an array of raw values and their associated mappings and returns an attribute hash
   # It accepts a block to alter the raw value that is stored in the raw text (if necessary),
   # enabling it to work for different sources
   def mapped_line(line, line_mappings)
-    attributes = {}
-    rawtext = {}
     validate_line_mappings(line_mappings)
 
+    rawtext = {}
+    data    = {}
+
     line.each_with_index do |raw_value, col|
       column_mapping = line_mappings[col]
       if column_mapping.nil?
@@ -67,7 +64,6 @@ module NdrImport::Mapper
       if column_mapping['standard_mapping']
         column_mapping = standard_mapping(column_mapping['standard_mapping'], column_mapping)
       end
-      field_mappings = column_mapping['mappings'] || []
 
       # Establish the rawtext column name we are to use for this column
       rawtext_column_name = (column_mapping['rawtext_name'] || column_mapping['column']).downcase
@@ -85,68 +81,56 @@ module NdrImport::Mapper
       # Store the raw column value
       rawtext[rawtext_column_name] = raw_value
 
-      field_mappings.each do |field_mapping|
+      next unless column_mapping.key?('mappings')
+      column_mapping['mappings'].each do |field_mapping|
         # create a duplicate of the raw value we can manipulate
         original_value = raw_value ? raw_value.dup : nil
 
         replace_before_mapping(original_value, field_mapping)
         value = mapped_value(original_value, field_mapping)
 
+        # We don't care about blank values, unless we're mapping a :join
+        # field (in which case, :compact may or may not be being used).
+        next if value.blank? && !field_mapping['join']
+
         field = field_mapping['field']
 
-        # Assumes join is specified in first joined field
-        joined = field_mapping['join'] ? true : false
-
-        # Currently assuming already validated YAML, s.t. no fields have the
-        # same priorities
-        #
-        # This has become really messy...
-        unless value.blank? && !joined
-          attributes[field] = {} unless attributes[field]
-          attributes[field][:priority] = {} unless attributes[field][:priority]
-          if field_mapping['order']
-            attributes[field][field_mapping['order']] = value
-            attributes[field][:join] = field_mapping['join'] if field_mapping['join']
-            attributes[field][:compact] = field_mapping['compact'] if field_mapping.include?('compact')
-          elsif field_mapping['priority']
-            attributes[field][:priority][field_mapping['priority']] = value
-          else
-            # Check if already a mapped-to field, and assign default low
-            # priority
-            attributes[field][:priority][1] = value
-            attributes[field][:value] = value
-          end
+        data[field] ||= {}
+        data[field][:values] ||= [] # "better" values come earlier
+        data[field][:compact]  = true unless data[field].key?(:compact)
+
+        if field_mapping['order']
+          data[field][:join] ||= field_mapping['join']
+          data[field][:compact] = field_mapping['compact'] if field_mapping.key?('compact')
+
+          data[field][:values][field_mapping['order'] - 1] = value
+        elsif field_mapping['priority']
+          data[field][:values][field_mapping['priority']] = value
+        else
+          data[field][:values].unshift(value) # new "best" value
         end
       end
     end
 
+    attributes = {}
+
     # tidy up many to one field mappings
     # and one to many, for cross-populating
-    attributes.each do |field, value|
-      if value.include?(:join)
-        join_string = value.delete(:join) || ','
-        value.delete(:value)
-        value.delete(:priority)
-        if value.include?(:compact)
-          compact = value.delete(:compact)
+    data.each do |field, field_data|
+      values = field_data[:values]
+
+      attributes[field] =
+        if field_data.key?(:join)
+          # Map "blank" values to nil:
+          values = values.map { |value| value if value.present? }
+          values.compact! if field_data[:compact]
+          values.join(field_data[:join])
         else
-          compact = true
-        end
-        t = value.sort.map do |_part_order, part_value|
-          part_value.blank? ? nil : part_value
+          values.detect(&:present?)
         end
-        if compact
-          attributes[field] = t.compact.join(join_string)
-        else
-          attributes[field] = t.join(join_string)
-        end
-      else
-        attributes[field][:priority].reject! { |_k, v| v.blank? }
-        attributes[field] = attributes[field][:priority].sort.first[1]
-      end
     end
 
-    attributes[:rawtext] = rawtext
+    attributes[:rawtext] = rawtext # Assign last
     attributes
   end
 
@@ -186,8 +170,9 @@ module NdrImport::Mapper
           fail "Standard mapping \"#{column_mapping['standard_mapping']}\" does not exist"
         end
       end
-      field_mappings = column_mapping['mappings'] || []
-      field_mappings.each do |field_mapping|
+
+      next unless column_mapping.key?('mappings')
+      column_mapping['mappings'].each do |field_mapping|
         field = field_mapping['field']
         if field_mapping['priority']
           fail 'Cannot have duplicate priorities' if priority[field] == field_mapping['priority']

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '3.1.6'.freeze
+  VERSION = '4.0.0'.freeze
 end

data/test/helpers/file/delimited_test.rb

@@ -87,27 +87,4 @@ class DelimitedTest < ActiveSupport::TestCase
       assert_equal ['2'] * 26, sheet[2]
     end
   end
-
-  def test_each_delimited_table_should_be_deprecated
-    original_stderr = $stderr
-    $stderr = StringIO.new
-
-    table = @importer.send(:each_delimited_table, @permanent_test_files.join('normal.csv'))
-    table.each do |tablename, sheet|
-      assert_nil tablename
-      sheet = sheet.to_a
-      assert_equal(('A'..'Z').to_a, sheet[0])
-      assert_equal ['1'] * 26, sheet[1]
-      assert_equal ['2'] * 26, sheet[2]
-    end
-
-    assert_match(/\A\[warning\] each_delimited_table will be deprecated/, $stderr.string)
-  ensure
-    $stderr = original_stderr
-  end if Gem::Requirement.new('< 4.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
-
-  def test_deprecated_methods_removed_in_v3
-    refute @importer.public_methods.include?(:each_delimited_table), 'should be removed in v4.0.0'
-    refute @importer.public_methods.include?(:each_delimited_row), 'should be removed in v4.0.0'
-  end if Gem::Requirement.new('>= 4.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
 end

data/test/helpers/file/excel_test.rb

@@ -79,26 +79,4 @@ class ExcelTest < ActiveSupport::TestCase
       assert_equal %w(1A 1B), sheet.first
     end
   end
-
-  def test_each_excel_table_should_be_deprecated
-    original_stderr = $stderr
-    $stderr = StringIO.new
-
-    table = @importer.send(:each_excel_table, @permanenttestfiles.join('sample_xlsx.xlsx'))
-    table.each do |tablename, sheet|
-      assert_equal 'Sheet1', tablename
-      assert_equal %w(1A 1B), sheet.first
-    end
-
-    assert_match(/\A\[warning\] each_excel_table will be deprecated/, $stderr.string)
-  ensure
-    $stderr = original_stderr
-  end if Gem::Requirement.new('< 4.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
-
-  def test_deprecated_methods_removed_in_v3
-    refute @importer.protected_methods.include?(:each_excel_table), 'should be removed in v4.0.0'
-    refute @importer.private_methods.include?(:each_excel_row), 'should be removed in v4.0.0'
-    refute @importer.private_methods.include?(:each_xls_row), 'should be removed in v4.0.0'
-    refute @importer.private_methods.include?(:each_xlsx_row), 'should be removed in v4.0.0'
-  end if Gem::Requirement.new('>= 4.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
 end

data/test/test_helper.rb

@@ -14,6 +14,8 @@ rescue LoadError
   # Rails 4+ only
 end
 
+ActiveSupport.test_order = :random if ActiveSupport.respond_to?(:test_order=)
+
 # The default changes to UTC in Rails 4.
 # TODO: ndr_support should cope...
 ActiveRecord::Base.default_timezone = :local

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 3.1.6
+  version: 4.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2016-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport