ndr_import 5.1.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 411899f19141a2e2da38b712d76d4de02b00bda7
-  data.tar.gz: 9feaaf50a3bb840974a0ce38ef66fee5ab32c19e
+  metadata.gz: 7781d10d01e028e81ddb0cccaacd8189a1e4e8dc
+  data.tar.gz: 1a0f2b9288d43be1f3caf0dcc68510b9feb95c22
 SHA512:
-  metadata.gz: 4d10e6a1d6370dd513364cf6332daf14ab6fe75e9b320906fa6a93e4e08422e54ebe16755bfbbd966e05c130bf53eab329e84ef2b5e766acd600e7059fd939f6
-  data.tar.gz: c824e74a89cfe9c9ded3f4a95b42f1be7bc31ae475b503264aa10e14621223691eb936dcd550792592ce7104c6eb278f7cb43059dacba916a26fc0f9ce4b8f6c
+  metadata.gz: 40f73db1effc623fde1c36b43ab3a016edac4ae308bc36a9aac7b9f4c515e94e8891e99413b758e75c306b30c2f9609e8e41577b491678b89d1b48bb884addf2
+  data.tar.gz: d352544945d48ebee58e62dd0a72ad7ab12101578d4ef6ae740e2f8c15925d023a54f0ef85746e45a47e227d99f3ef14e93d6eebd97632af0eafba5005a5f93a

data/code_safety.yml

@@ -75,7 +75,7 @@ file safety:
   lib/ndr_import/file/delimited.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   lib/ndr_import/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -167,19 +167,19 @@ file safety:
   lib/ndr_import/table.rb:
     comments: uses File.basename
     reviewed_by: josh.pencheon
-    safe_revision: 14775e7262dbec3b6d0f69930ff526f856a94214
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 36143ff4f48e0433a7b04b63143ef6d6d2481fec
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   lib/ndr_import/version.rb:
     comments: another check?
-    reviewed_by: ollie.tulloch
-    safe_revision: fa21aae027f436337eb0de481e390cdbab348123
+    reviewed_by: josh.pencheon
+    safe_revision: 5adc16ab99f43a20c1bedcea8724258188ab64bd
   ndr_import.gemspec:
     comments: 
-    reviewed_by: ollie.tulloch
-    safe_revision: a76d7fd03894158417411d0c9620ea4c1d5a03d6
+    reviewed_by: josh.pencheon
+    safe_revision: ea0a8c2eca882e5fe43d6dde203490033086a197
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -187,7 +187,7 @@ file safety:
   test/file/delimited_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   test/file/excel_test.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -199,7 +199,7 @@ file safety:
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f82e6839f7e3f9932a6f92242c03d94234fc1d82
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -250,8 +250,8 @@ file safety:
     safe_revision: 63b1e8c275bd63377309d9b124176f39f75576a9
   test/non_tabular/table_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: c9136301653bd7e6c4d16a8cfb8f40ad9ef6e836
+    reviewed_by: josh.pencheon
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   test/non_tabular_file_helper_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -320,6 +320,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: e13005efa08f7168c1c65c60ce97057287ace534
+  test/resources/high_ascii_delimited_example_two.txt:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   test/resources/malformed.xml:
     comments: 
     reviewed_by: timgentry
@@ -348,6 +352,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
+  test/resources/not_sign_delimited.txt:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
   test/resources/sample_xls.xls:
     comments: 
     reviewed_by: timgentry
@@ -419,4 +427,4 @@ file safety:
   test/universal_importer_helper_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 36143ff4f48e0433a7b04b63143ef6d6d2481fec
+    safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10

data/lib/ndr_import/file/delimited.rb

@@ -9,11 +9,8 @@ module NdrImport
     # This class is a delimited file handler that returns a single table.
     class Delimited < Base
       DELIMITED_COL_SEP = {
-        'csv' => nil,
-        'pipe' => '|',
-        'thorn' => "\xfe" # high-ascii (lower case thorn) delimited files
+        'csv' => nil
       }
-
       def initialize(filename, format, options = {})
         super
 
@@ -86,6 +83,6 @@ module NdrImport
       end
     end
 
-    Registry.register(Delimited, 'csv', 'pipe', 'thorn')
+    Registry.register(Delimited, 'csv', 'delimited')
   end
 end

data/lib/ndr_import/table.rb

@@ -10,7 +10,8 @@ module NdrImport
     include NdrImport::Mapper
 
     def self.all_valid_options
-      %w(canonical_name filename_pattern tablename_pattern header_lines footer_lines format klass columns)
+      %w[canonical_name delimiter filename_pattern tablename_pattern header_lines footer_lines
+         format klass columns]
     end
 
     def all_valid_options

data/lib/ndr_import/universal_importer_helper.rb

@@ -34,7 +34,8 @@ module NdrImport
 
         tables = NdrImport::File::Registry.tables(filename,
                                                   table_mapping.try(:format),
-                                                  'unzip_path' => unzip_path)
+                                                  'unzip_path' => unzip_path,
+                                                  'col_sep' => table_mapping.try(:delimiter))
         yield_tables_and_their_content(filename, tables, &block)
       end
     end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '5.1.0'.freeze
+  VERSION = '6.0.0'.freeze
 end

data/ndr_import.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
-  spec.add_development_dependency 'ndr_dev_support', '~> 3.0'
+  spec.add_development_dependency 'ndr_dev_support', '~> 3.1', '>= 3.1.3'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'guard-test'

data/test/file/delimited_test.rb

@@ -23,7 +23,7 @@ module NdrImport
 
       test 'should read pipe correctly' do
         file_path = @permanent_test_files.join('normal_pipe.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'pipe', 'col_sep' => nil)
+        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => '|')
         handler.tables.each do |tablename, sheet|
           assert_nil tablename
           sheet = sheet.to_a
@@ -35,7 +35,7 @@ module NdrImport
 
       test 'should read thorn correctly' do
         file_path = @permanent_test_files.join('normal_thorn.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'thorn', 'col_sep' => nil)
+        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfe")
         handler.tables.each do |tablename, sheet|
           assert_nil tablename
           sheet = sheet.to_a
@@ -71,7 +71,7 @@ module NdrImport
 
       test 'should read acsii-delimited csv' do
         file_path = @permanent_test_files.join('high_ascii_delimited.txt')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => "\xfe")
+        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfe")
         handler.tables.each do |tablename, sheet|
           assert_nil tablename
           assert_instance_of Enumerator, sheet
@@ -82,6 +82,23 @@ module NdrImport
         end
       end
 
+      test 'should read acsii-delimited txt' do
+        rows = []
+        file_path = @permanent_test_files.join('high_ascii_delimited_example_two.txt')
+        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfd")
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet.each do |row|
+            rows << row
+          end
+        end
+
+        assert_equal(('A'..'Z').to_a, rows[0])
+        assert_equal ['1'] * 26, rows[1]
+        assert_equal ['2'] * 26, rows[2]
+      end
+
       test 'should read line-by-line' do
         rows = []
         file_path = @permanent_test_files.join('normal.csv')

data/test/file/registry_test.rb

@@ -12,7 +12,7 @@ module NdrImport
 
       test 'Registry.handlers' do
         assert_instance_of Hash, NdrImport::File::Registry.handlers
-        assert_equal %w(csv doc pdf pipe thorn txt xls xlsx zip),
+        assert_equal %w[csv delimited doc pdf txt xls xlsx zip],
                      NdrImport::File::Registry.handlers.keys.sort
       end
 

data/test/non_tabular/table_test.rb

@@ -47,10 +47,10 @@ STR
   end
 
   def test_all_valid_options
-    valid_options = %w(
-      canonical_name capture_start_line columns end_in_a_record end_line_pattern filename_pattern
-      format klass remove_lines start_in_a_record start_line_pattern
-    )
+    valid_options = %w[
+      canonical_name capture_start_line columns delimiter end_in_a_record end_line_pattern
+      filename_pattern format klass remove_lines start_in_a_record start_line_pattern
+    ]
     assert_equal valid_options.sort,
                  NdrImport::NonTabular::Table.all_valid_options.sort
   end

data/test/resources/high_ascii_delimited_example_two.txt

@@ -0,0 +1,3 @@
+A�B�C�D�E�F�G�H�I�J�K�L�M�N�O�P�Q�R�S�T�U�V�W�X�Y�Z
+1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1�1
+2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2�2

data/test/resources/not_sign_delimited.txt

@@ -0,0 +1,3 @@
+one¬two¬three
+some¬data¬here
+more¬data¬here

data/test/universal_importer_helper_test.rb

@@ -46,6 +46,33 @@ class UniversalImporterHelperTest < ActiveSupport::TestCase
     refute enumerator_ran
   end
 
+  test 'extract and transform with bespoke delimiter' do
+    table_mappings = [
+      NdrImport::Table.new(filename_pattern: /\.txt\z/i,
+                           format: 'delimited',
+                           delimiter: '¬',
+                           header_lines: 1,
+                           footer_lines: 0,
+                           klass: 'SomeTestClass',
+                           columns: [{ 'column' => 'one' },
+                                     { 'column' => 'two' },
+                                     { 'column' => 'three' }])
+    ]
+    source_file = @permanent_test_files.join('not_sign_delimited.txt')
+    unzip_path = SafePath.new('test_space_rw').to_s
+    @test_importer.stubs(:get_table_mapping).returns(table_mappings.first)
+    @test_importer.extract(source_file, unzip_path) do |table, rows|
+      mapped_rows = table.transform(rows)
+
+      assert_instance_of NdrImport::Table, table
+      assert_instance_of Enumerator, rows
+      expected_mapped_data = [{ rawtext: { 'one' => 'some', 'two' => 'data', 'three' => 'here' } },
+                              { rawtext: { 'one' => 'more', 'two' => 'data', 'three' => 'here' } }]
+
+      assert_equal expected_mapped_data, (mapped_rows.to_a.map { |_klass, fields| fields })
+    end
+  end
+
   test 'get_notifier' do
     class TestImporterWithoutNotifier
       include NdrImport::UniversalImporterHelper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 5.1.0
+  version: 6.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-15 00:00:00.000000000 Z
+date: 2018-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -222,14 +222,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.3
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
@@ -386,6 +392,7 @@ files:
 - test/resources/hello_world.pdf
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
+- test/resources/high_ascii_delimited_example_two.txt
 - test/resources/malformed.xml
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
@@ -393,6 +400,7 @@ files:
 - test/resources/normal_thorn.csv
 - test/resources/not_a_pdf.pdf
 - test/resources/not_a_word_file.doc
+- test/resources/not_sign_delimited.txt
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx
 - test/resources/sheet_streaming.xls
@@ -471,6 +479,7 @@ test_files:
 - test/resources/hello_world.pdf
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
+- test/resources/high_ascii_delimited_example_two.txt
 - test/resources/malformed.xml
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
@@ -478,6 +487,7 @@ test_files:
 - test/resources/normal_thorn.csv
 - test/resources/not_a_pdf.pdf
 - test/resources/not_a_word_file.doc
+- test/resources/not_sign_delimited.txt
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx
 - test/resources/sheet_streaming.xls