ndr_import 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b9dfb908b8fd54add87023de9f374e6b3a74af9
-  data.tar.gz: a0d67ce391abf53242f15e221df6cb297f83cb14
+  metadata.gz: 8b7020625dad38c6d789cf4b081fa3b770252bad
+  data.tar.gz: 145338f57d67c6cf72a12077e8a38918390aac51
 SHA512:
-  metadata.gz: 46478e258b56cf0cebe45f8255087bbd48371bb0f4e5a1c30ce69e2b40d1500fa24427f8eea949a21a72ab22ef6e5e147283706d99fbe900f490502422b2b5a5
-  data.tar.gz: de540d72430687fd0c2731bd810910fc65ba0edf0c05798e6970f437ebbc451b58f9c7a77df35862b830e5b31f973fb3bc264d9646967de42d62a6d6d5db5d41
+  metadata.gz: 8b6050f2f7c69de8322113eaf21f96c0fbd97524ac7e0afebd2f9d576f945794add6323d14b53be9fe5cd01317e222305e7a1c9017e715ca8c6d6ce210efa1b4
+  data.tar.gz: 9351ef5aedeb46798f73b12b84531c617430b4f9726370a2cfceed5800fca63fb37149043f3ee1b818050160d51acf2f72d6bc70b773dcf6140463ad5167fe82

data/.travis.yml

@@ -1,14 +1,14 @@
 language: ruby
 
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3.0
+  - 2.2.5
+  - 2.3.1
 
 gemfile:
   - gemfiles/Gemfile.rails32
   - gemfiles/Gemfile.rails41
   - gemfiles/Gemfile.rails42
+  - gemfiles/Gemfile.rails50
 
 sudo: false
 

data/Rakefile

@@ -6,7 +6,8 @@ Rake::TestTask.new do |t|
   t.libs << 'lib'
   t.libs << 'test'
   t.pattern = 'test/**/*_test.rb'
-  t.verbose = true
+  t.verbose = false
+  t.warning = false
 end
 
 desc 'Run tests'

data/code_safety.yml

@@ -15,7 +15,7 @@ file safety:
   ".travis.yml":
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: c87afff2ef0c63a1b648037e532b94523f84c0ab
+    safe_revision: f9de73c5988557e29ea73c279f2fdae19ffa8bb4
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -39,7 +39,7 @@ file safety:
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a4d925f8c4076fadedbc6a3f5b672dfeca041572
+    safe_revision: 5a7f26cecaabab20f4e666776f9166dcc3fa6bfe
   gemfiles/Gemfile.rails32:
     comments: 
     reviewed_by: timgentry
@@ -52,6 +52,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 0c5967732100f43e7b60a97a7f0aae60d3641791
+  gemfiles/Gemfile.rails50:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 738577adf8eeacc7a000d08289bd7ae7bf3b5849
   lib/ndr_import.rb:
     comments: 
     reviewed_by: timgentry
@@ -160,14 +164,18 @@ file safety:
     comments: uses File.basename
     reviewed_by: joshpencheon
     safe_revision: e70ece909c44b26208eaff3cf27ada7145e03f11
+  lib/ndr_import/universal_importer_helper.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 36143ff4f48e0433a7b04b63143ef6d6d2481fec
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: joshpencheon
-    safe_revision: 00e02b83f5ae640becdb98d378eddfb67f8ae5c5
+    reviewed_by: josh.pencheon
+    safe_revision: 42f08fa7fdb0ff325b0f830af66d1829efa9b273
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 52e4bf7ab1f897a5184a291f64d0926f44272a6a
+    safe_revision: 0a8fccbac2a264adcdae7e46df54da25dc9cfdcd
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -227,7 +235,7 @@ file safety:
   test/mapper_test.rb:
     comments: exposes Mapper internals to test them
     reviewed_by: josh.pencheon
-    safe_revision: e1cc1153509ac371438aa68708a6c316dbf09eee
+    safe_revision: f707f42686cabe21d80e2547e9a24fe094b6c7d1
   test/non_tabular/mapping_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -400,3 +408,7 @@ file safety:
     comments: 
     reviewed_by: joshpencheon
     safe_revision: 646c08329cba0ba8ea2c37226cf26434da29c747
+  test/universal_importer_helper_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 36143ff4f48e0433a7b04b63143ef6d6d2481fec

data/gemfiles/Gemfile.rails50

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec path: '..'
+
+gem 'activesupport', '~> 5.0.0'

data/lib/ndr_import/universal_importer_helper.rb

@@ -0,0 +1,61 @@
+require 'ndr_import/file/registry'
+
+module NdrImport
+  # This mixin provides file importer helper methods that abstract away some of the
+  # complexity of enumerating over files and tables (which should be universally useful).
+  module UniversalImporterHelper
+    def table_enumerators(filename)
+      table_enumerators = {}
+
+      extract(filename).each do |table, rows|
+        table_enumerators[table.canonical_name] = table.transform(rows)
+      end
+
+      table_enumerators
+    end
+
+    # This method returns the correct NdrImport::{,NonTabular::}Table for the given
+    # filename/tablename. It requires all the mappings to be stored in the table_mappings
+    # instance variable.
+    def get_table_mapping(filename, tablename)
+      @table_mappings.find { |mapping| mapping.match(filename, tablename) }
+    end
+
+    # Iterate through the file(s) line by line, yielding each one in turn, using
+    # get_table_mapping to select the mapping relevant to this file.
+    def extract(source_file, unzip_path, &block)
+      return enum_for(:extract, source_file, unzip_path) unless block
+
+      files = NdrImport::File::Registry.files(source_file,
+                                              'unzip_path' => unzip_path)
+      files.each do |filename|
+        # now at the individual file level, can we find the table mapping?
+        table_mapping = get_table_mapping(filename, nil)
+
+        tables = NdrImport::File::Registry.tables(filename,
+                                                  table_mapping.try(:format),
+                                                  'unzip_path' => unzip_path)
+        yield_tables_and_their_content(filename, tables, &block)
+      end
+    end
+
+    # This method does the table row yielding for the extract method, setting the notifier
+    # so that we can monitor progress
+    def yield_tables_and_their_content(filename, tables, &block)
+      tables.each do |tablename, table_content|
+        mapping = get_table_mapping(filename, tablename)
+        next if mapping.nil?
+
+        total_records = table_content.count unless table_content.is_a?(Enumerator)
+        mapping.notifier = get_notifier(total_records)
+
+        yield(mapping, table_content)
+      end
+    end
+
+    # This method needs to be implemented where this mixin is used.
+    def get_notifier(_total_records)
+      fail 'Implement get_notifier'
+    end
+  end
+end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '4.0.1'.freeze
+  VERSION = '4.1.0'.freeze
 end

data/ndr_import.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.0.0'
-  spec.add_dependency 'ndr_support', '>= 3.3.0', '< 5'
+  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.1'
+  spec.add_dependency 'ndr_support', '>= 4.1.2', '< 6'
 
   spec.add_dependency 'rubyzip', '~> 1.1'
   spec.add_dependency 'roo', '~> 2.0'
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'pdf-reader', '1.2.0'
   spec.add_dependency 'msworddoc-extractor', '0.2.0'
 
-  spec.required_ruby_version = '>= 2.1.0'
+  spec.required_ruby_version = '>= 2.2'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/test/mapper_test.rb

@@ -19,7 +19,7 @@ class MapperTest < ActiveSupport::TestCase
   format_mapping_yyyymmdd = { 'format' => 'yyyymmdd' }
   clean_name_mapping = { 'clean' => :name }
   clean_ethniccategory_mapping = { 'clean' => :ethniccategory }
-  clean_icd_mapping = { 'clean' => :code_icd }
+  clean_icd_mapping = { 'clean' => :icd }
   clean_opcs_mapping = { 'clean' => :code_opcs }
   map_mapping = { 'map' => { 'A' => '1' } }
   replace_mapping = { 'replace' => { '.0' => '' } }
@@ -325,7 +325,7 @@ class MapperTest < ActiveSupport::TestCase
   end
 
   test 'map should clean icd code' do
-    assert_equal 'C34.3 R93.2 Z51.5',
+    assert_equal 'C343 R932 Z515',
                  TestMapper.new.mapped_value('C34.3,R93.2,Z51.5', clean_icd_mapping)
   end
 

data/test/universal_importer_helper_test.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+require 'test_helper'
+require 'ndr_import/universal_importer_helper'
+
+# This tests the UniversalImporterHelper mixin
+class UniversalImporterHelperTest < ActiveSupport::TestCase
+  # This is a test importer class to test the excel file helper mixin
+  class TestImporter
+    include NdrImport::UniversalImporterHelper
+
+    def initialize
+      @table_mappings = [
+        NdrImport::Table.new(:filename_pattern => /\.xls\z/i,
+                             :tablename_pattern => /\Asheet1\z/i)
+      ]
+    end
+
+    def get_notifier(_)
+    end
+  end
+
+  def setup
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @test_importer = TestImporter.new
+  end
+
+  test 'extract with matching mapping' do
+    source_file = @permanent_test_files.join('sample_xls.xls')
+    unzip_path = SafePath.new('test_space_rw').to_s
+    enumerator_ran = false
+    @test_importer.extract(source_file, unzip_path) do |table, rows|
+      assert_instance_of NdrImport::Table, table
+      assert_instance_of Enumerator, rows
+      enumerator_ran = true
+    end
+    assert enumerator_ran
+  end
+
+  test 'extract without matching mapping' do
+    source_file = @permanent_test_files.join('sample_xlsx.xlsx')
+    unzip_path = SafePath.new('test_space_rw').to_s
+    enumerator_ran = false
+    @test_importer.extract(source_file, unzip_path) do |_table, _rows|
+      enumerator_ran = true
+    end
+    refute enumerator_ran
+  end
+
+  test 'get_notifier' do
+    class TestImporterWithoutNotifier
+      include NdrImport::UniversalImporterHelper
+    end
+
+    assert_raise(RuntimeError) do
+      TestImporterWithoutNotifier.new.get_notifier(10_000)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-01 00:00:00.000000000 Z
+date: 2016-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -19,7 +19,7 @@ dependencies:
         version: 3.2.18
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,27 +29,27 @@ dependencies:
         version: 3.2.18
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: ndr_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: 4.1.2
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: 4.1.2
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -300,6 +300,7 @@ files:
 - gemfiles/Gemfile.rails32
 - gemfiles/Gemfile.rails41
 - gemfiles/Gemfile.rails42
+- gemfiles/Gemfile.rails50
 - lib/ndr_import.rb
 - lib/ndr_import/csv_library.rb
 - lib/ndr_import/file/all.rb
@@ -327,6 +328,7 @@ files:
 - lib/ndr_import/non_tabular_file_helper.rb
 - lib/ndr_import/standard_mappings.rb
 - lib/ndr_import/table.rb
+- lib/ndr_import/universal_importer_helper.rb
 - lib/ndr_import/version.rb
 - ndr_import.gemspec
 - test/file/base_test.rb
@@ -387,6 +389,7 @@ files:
 - test/standard_mappings_test.rb
 - test/table_test.rb
 - test/test_helper.rb
+- test/universal_importer_helper_test.rb
 homepage: https://github.com/PublicHealthEngland/ndr_import
 licenses:
 - MIT
@@ -399,7 +402,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -407,7 +410,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.5
 signing_key: 
 specification_version: 4
 summary: NDR Import
@@ -470,3 +473,4 @@ test_files:
 - test/standard_mappings_test.rb
 - test/table_test.rb
 - test/test_helper.rb
+- test/universal_importer_helper_test.rb