ndr_import 6.4.1 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4a3d81eb60f8edbb23c9bcb3a3c9563e2af8727d3556bd903c74acfe799e1b5
-  data.tar.gz: d21272466eb8916ffd83dccd200f7e1b9a37d0104111472137b083c44dc2d356
+  metadata.gz: 1a5dd34ac83682dce8b3bd5d53f3610c750e3db86defc7e6f808738d4c6a47d9
+  data.tar.gz: 942ca0f0d52d949c39d338938a6ab1d354f1c037675bcb8bae940c9357e672a0
 SHA512:
-  metadata.gz: bd72bcca29c9e8215939a53c5a48f63752ba1c859b0ec06b6a384831615a3f2d8e0ff3d7567842bf4806c65e25ee08987d02e6a3660c764998adf1d31990de65
-  data.tar.gz: 6210a99d38241e29c33a06ee94093ce6ada95cf79e9ecc0ed8532700c44434b063197adde1c5f9b7a202168f9c0d329de75039472dff22da815564afc004bf53
+  metadata.gz: 370e387ead245e596ac48c4d9669e86083754e43b3d9521b20501a13569286c3820e8a699b3492121410b4de2ceada0e9c9957d3e07469afc95820f42ab3ffaa
+  data.tar.gz: 1e74f1eb05e132fd611250730c5f720d10930488401f07774bd12afa853dde31f22e0d1e54e3fd0d24f93a77633b8136e852932a8fc634057fda4ccba7954113

data/code_safety.yml

@@ -183,11 +183,11 @@ file safety:
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    safe_revision: 3e3a852b58e8b169535e29029e535a10f6b9cd42
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: c830ea8f3fa25efe88eb8712573ff6933978bdeb
+    safe_revision: 3f1743adb7a0ae612e480ffb9b39ba14bfb0493f
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -195,7 +195,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 2cb4ecf1f8d8e86e9f34871782b96590665b5cf0
+    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -215,11 +215,11 @@ file safety:
   test/file/pdf_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
+    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -251,7 +251,7 @@ file safety:
   test/helpers/file/pdf_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
+    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
   test/helpers/file/word_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -471,7 +471,7 @@ file safety:
   test/universal_importer_helper_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+    safe_revision: 3e3a852b58e8b169535e29029e535a10f6b9cd42
   test/xml/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/universal_importer_helper.rb

@@ -3,6 +3,7 @@ require 'ndr_import/file/registry'
 module NdrImport
   # This mixin provides file importer helper methods that abstract away some of the
   # complexity of enumerating over files and tables (which should be universally useful).
+  # It is assumed that the host module/class defines `unzip_path`.
   module UniversalImporterHelper
     def table_enumerators(filename)
       table_enumerators = {}
@@ -23,8 +24,8 @@ module NdrImport
 
     # Iterate through the file(s) line by line, yielding each one in turn, using
     # get_table_mapping to select the mapping relevant to this file.
-    def extract(source_file, unzip_path, &block)
-      return enum_for(:extract, source_file, unzip_path) unless block
+    def extract(source_file, &block)
+      return enum_for(:extract, source_file) unless block
 
       files = NdrImport::File::Registry.files(source_file,
                                               'unzip_path' => unzip_path)
@@ -47,20 +48,35 @@ module NdrImport
     # This method does the table row yielding for the extract method, setting the notifier
     # so that we can monitor progress
     def yield_tables_and_their_content(filename, tables, &block)
+      return enum_for(:yield_tables_and_their_content, filename, tables) unless block_given?
+
       tables.each do |tablename, table_content|
         mapping = get_table_mapping(filename, tablename)
         next if mapping.nil?
 
-        total_records = table_content.count unless table_content.is_a?(Enumerator)
-        mapping.notifier = get_notifier(total_records)
+        mapping.notifier = get_notifier(record_total(filename, table_content))
 
         yield(mapping, table_content)
       end
     end
 
+    def mapped_tables(filename)
+      @mapped_tables ||= table_enumerators(filename)
+    end
+
     # This method needs to be implemented where this mixin is used.
     def get_notifier(_total_records)
-      fail 'Implement get_notifier'
+      raise NotImplementedError, 'get_notifier must be defined!'
+    end
+
+    def record_total(filename, table_content)
+      if '.csv' == ::File.extname(filename).downcase
+        return `wc -l #{Shellwords.escape(filename)}`.strip.match(/\A(\d+)/)[1].to_i
+      elsif table_content.is_a?(Enumerator)
+        nil # Avoid slurping
+      else
+        table_content.size
+      end
     end
   end
 end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '6.4.1'.freeze
+  VERSION = '7.0.0'.freeze
 end

data/ndr_import.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'docx', '~> 0.3'
   spec.add_dependency 'msworddoc-extractor', '0.2.0'
   spec.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.5'
-  spec.add_dependency 'pdf-reader', '1.2.0' # Raises warnings on Ruby 2.4+
+  spec.add_dependency 'pdf-reader', '~> 2.1'
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'spreadsheet', '1.0.3'
 

data/test/file/pdf_test.rb

@@ -15,7 +15,8 @@ module NdrImport
         handler.tables.each do |tablename, sheet|
           assert_nil tablename
           assert_instance_of Enumerator, sheet
-          assert_equal ['Hello World ', 'Goodbye Universe ', ' '], sheet.to_a
+          assert_equal ['Hello                                         World', '',
+                        'Goodbye                                       Universe'], sheet.to_a
         end
       end
 

data/test/file/registry_test.rb

@@ -35,7 +35,8 @@ module NdrImport
         tables.each do |tablename, sheet|
           assert_nil tablename
           assert_instance_of Enumerator, sheet
-          assert_equal ['Hello World ', 'Goodbye Universe ', ' '], sheet.to_a
+          assert_equal ['Hello                                         World', '',
+                        'Goodbye                                       Universe'], sheet.to_a
         end
       end
 

data/test/helpers/file/pdf_test.rb

@@ -15,7 +15,8 @@ class PdfTest < ActiveSupport::TestCase
 
   test 'read_pdf_file helper should read pdf file' do
     file_content = @importer.send(:read_pdf_file, @permanent_test_files.join('hello_world.pdf'))
-    assert_equal ['Hello World ', 'Goodbye Universe ', ' '], file_content
+    assert_equal ['Hello                                         World', '',
+                  'Goodbye                                       Universe'], file_content
   end
 
   test 'read_pdf_file helper should raise exception on invalid pdf file' do

data/test/universal_importer_helper_test.rb

@@ -17,6 +17,10 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
 
     def get_notifier(_)
     end
+
+    def unzip_path
+      SafePath.new('test_space_rw')
+    end
   end
 
   def setup
@@ -26,9 +30,8 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
 
   test 'extract with matching mapping' do
     source_file = @permanent_test_files.join('sample_xls.xls')
-    unzip_path = SafePath.new('test_space_rw').to_s
     enumerator_ran = false
-    @test_importer.extract(source_file, unzip_path) do |table, rows|
+    @test_importer.extract(source_file) do |table, rows|
       assert_instance_of NdrImport::Table, table
       assert_instance_of Enumerator, rows
       enumerator_ran = true
@@ -38,9 +41,8 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
 
   test 'extract without matching mapping' do
     source_file = @permanent_test_files.join('sample_xlsx.xlsx')
-    unzip_path = SafePath.new('test_space_rw').to_s
     enumerator_ran = false
-    @test_importer.extract(source_file, unzip_path) do |_table, _rows|
+    @test_importer.extract(source_file) do |_table, _rows|
       enumerator_ran = true
     end
     refute enumerator_ran
@@ -59,9 +61,8 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
                                      { 'column' => 'three' }])
     ]
     source_file = @permanent_test_files.join('not_sign_delimited.txt')
-    unzip_path = SafePath.new('test_space_rw').to_s
     @test_importer.stubs(:get_table_mapping).returns(table_mappings.first)
-    @test_importer.extract(source_file, unzip_path) do |table, rows|
+    @test_importer.extract(source_file) do |table, rows|
       mapped_rows = table.transform(rows)
 
       assert_instance_of NdrImport::Table, table
@@ -78,7 +79,7 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
       include NdrImport::UniversalImporterHelper
     end
 
-    assert_raise(RuntimeError) do
+    assert_raise(NotImplementedError) do
       TestImporterWithoutNotifier.new.get_notifier(10_000)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 6.4.1
+  version: 7.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-18 00:00:00.000000000 Z
+date: 2018-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -136,16 +136,16 @@ dependencies:
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: roo-xls
   requirement: !ruby/object:Gem::Requirement