ndr_import 3.0.0

data/test/file/excel_test.rb

@@ -0,0 +1,85 @@
+require 'test_helper'
+require 'ndr_import/file/excel'
+
+module NdrImport
+  module File
+    # Excel file handler tests
+    class ExcelTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read xls file' do
+        file_path = @permanent_test_files.join('sample_xls.xls')
+        handler = NdrImport::File::Excel.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_equal 'Sheet1', tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal %w(1A 1B), sheet.first
+        end
+      end
+
+      test 'should read xlsx file' do
+        file_path = @permanent_test_files.join('sample_xlsx.xlsx')
+        handler = NdrImport::File::Excel.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_equal 'Sheet1', tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal %w(1A 1B), sheet.first
+        end
+      end
+
+      test 'should read xlsx file with the incorrect xls extension' do
+        file_path = @permanent_test_files.join('xlsx_file_xls_extension.xls')
+        handler = NdrImport::File::Excel.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_equal 'Sheet1', tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal %w(1A 1B), sheet.first
+        end
+
+        SafeFile.delete @permanent_test_files.join('xlsx_file_xls_extension_amend.xlsx')
+      end
+
+      test 'read_excel_file helper should handle exceptions' do
+        # txt file
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('flat_file.txt')
+          handler = NdrImport::File::Excel.new(file_path, nil)
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+
+        # .txt file in .xls extension
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('txt_file_xls_extension.xls')
+          handler = NdrImport::File::Excel.new(file_path, 'txt')
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+
+        # .txt file in .xlsx extension
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('txt_file_xlsx_extension.xlsx')
+          handler = NdrImport::File::Excel.new(file_path, 'txt')
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+
+        SafeFile.delete @permanent_test_files.join('txt_file_xls_extension_amend.xlsx')
+      end
+    end
+  end
+end

data/test/file/pdf_test.rb

@@ -0,0 +1,35 @@
+require 'test_helper'
+require 'ndr_import/file/pdf'
+
+module NdrImport
+  module File
+    # PDF file handler tests
+    class PdfTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read pdf correctly' do
+        file_path = @permanent_test_files.join('hello_world.pdf')
+        handler = NdrImport::File::Pdf.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal ['Hello World'], sheet.to_a
+        end
+      end
+
+      test 'should raise exception on invalid pdf file' do
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('not_a_pdf.pdf')
+          handler = NdrImport::File::Pdf.new(file_path, nil)
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+      end
+    end
+  end
+end

data/test/file/registry_test.rb

@@ -0,0 +1,60 @@
+require 'test_helper'
+require 'ndr_import/file/registry'
+
+module NdrImport
+  module File
+    # Registry file handler tests
+    class RegistryTest < ActiveSupport::TestCase
+      def setup
+        @home = SafePath.new('test_space_rw')
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'Registry.handlers' do
+        assert_instance_of Hash, NdrImport::File::Registry.handlers
+        assert_equal %w(csv doc pdf pipe thorn txt xls xlsx zip),
+                     NdrImport::File::Registry.handlers.keys.sort
+      end
+
+      test 'should fail to enumerate unknown format' do
+        exception = assert_raises(RuntimeError) do
+          file_path = @permanent_test_files.join('normal.csv')
+          tables = NdrImport::File::Registry.tables(file_path, 'mp3')
+          tables.each do |tablename, sheet|
+            assert_nil tablename
+            sheet.to_a
+          end
+        end
+
+        assert_equal 'Error: Unknown file format "mp3"', exception.message
+      end
+
+      test 'should enumerate pdf file table' do
+        file_path = @permanent_test_files.join('hello_world.pdf')
+        tables = NdrImport::File::Registry.tables(file_path, nil)
+        tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal ['Hello World'], sheet.to_a
+        end
+      end
+
+      test 'should enumerate zip file tables' do
+        options = { 'unzip_path' => @home }
+        file_path = @permanent_test_files.join('normal.csv.zip')
+        files = NdrImport::File::Registry.files(file_path, options)
+        files.each do |filename|
+          tables = NdrImport::File::Registry.tables(filename, nil, options)
+
+          tables.each do |tablename, sheet|
+            assert_nil tablename
+            sheet = sheet.to_a
+            assert_equal(('A'..'Z').to_a, sheet[0])
+            assert_equal ['1'] * 26, sheet[1]
+            assert_equal ['2'] * 26, sheet[2]
+          end
+        end
+      end
+    end
+  end
+end

data/test/file/text_test.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+require 'test_helper'
+require 'ndr_import/file/text'
+
+module NdrImport
+  module File
+    # Text file handler tests
+    class TextTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read text file' do
+        file_path = @permanent_test_files.join('hello_world.txt')
+        handler = NdrImport::File::Text.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal ['Hello world,', 'this is a text document'], sheet.to_a
+        end
+      end
+
+      test 'should read text file with UTF-8 encoding' do
+        file_path = @permanent_test_files.join('hello_utf8.txt')
+        handler = NdrImport::File::Text.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+
+          lines = sheet.to_a
+
+          assert_equal ['Hello world', 'This is a thorny þ issue!'], lines
+          assert lines.all? { |line| line.encoding.name == 'UTF-8' && line.valid_encoding? }
+        end
+      end
+
+      test 'should read text file with UTF-16 [BE] encoding' do
+        file_path = @permanent_test_files.join('hello_utf16be.txt')
+        handler = NdrImport::File::Text.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+
+          lines = sheet.to_a
+
+          assert_equal ['Hello world', 'This is a thorny þ issue!'], lines
+          assert lines.all? { |line| line.encoding.name == 'UTF-8' && line.valid_encoding? }
+        end
+      end
+
+      test 'should read text file with UTF-16 [LE] encoding' do
+        file_path = @permanent_test_files.join('hello_utf16le.txt')
+        handler = NdrImport::File::Text.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+
+          lines = sheet.to_a
+
+          assert_equal ['Hello world', 'This is a thorny þ issue!'], lines
+          assert lines.all? { |line| line.encoding.name == 'UTF-8' && line.valid_encoding? }
+        end
+      end
+
+      test 'should read text file with Windows-1252 encoding' do
+        file_path = @permanent_test_files.join('hello_windows.txt')
+        handler = NdrImport::File::Text.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+
+          lines = sheet.to_a
+
+          assert_equal ['Hello windows world', 'This is a thorny þ issue!'], lines
+          assert lines.all? { |line| line.encoding.name == 'UTF-8' && line.valid_encoding? }
+        end
+      end
+
+      test 'should raise exception on invalid text file' do
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('hello_world.pdf')
+          handler = NdrImport::File::Text.new(file_path, nil)
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+      end
+    end
+  end
+end

data/test/file/word_test.rb

@@ -0,0 +1,35 @@
+require 'test_helper'
+require 'ndr_import/file/pdf'
+
+module NdrImport
+  module File
+    # Word document file handler tests
+    class WordTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read word file' do
+        file_path = @permanent_test_files.join('hello_world.doc')
+        handler = NdrImport::File::Word.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal ['Hello world, this is a word document'], sheet.to_a
+        end
+      end
+
+      test 'should raise exception on invalid word file' do
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('not_a_word_file.doc')
+          handler = NdrImport::File::Word.new(file_path, nil)
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+      end
+    end
+  end
+end

data/test/file/zip_test.rb

@@ -0,0 +1,47 @@
+require 'test_helper'
+require 'ndr_import/file/zip'
+require 'zip'
+
+module NdrImport
+  module File
+    # Zip file handler tests
+    class ZipTest < ActiveSupport::TestCase
+      def setup
+        @home = SafePath.new('test_space_rw')
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should reject non SafePath arguments' do
+        file_path = @home.join('imaginary.zip')
+
+        assert_raises ArgumentError do
+          NdrImport::File::Zip.new(file_path.to_s, nil, 'unzip_path' => @home.to_s)
+        end
+
+        assert_raises ArgumentError do
+          NdrImport::File::Zip.new(file_path.to_s, nil, 'unzip_path' => @home)
+        end
+
+        assert_raises ArgumentError do
+          NdrImport::File::Zip.new(file_path, nil, 'unzip_path' => @home.to_s)
+        end
+      end
+
+      test 'should read table correctly' do
+        options = { 'unzip_path' => @home }
+        file_path = @permanent_test_files.join('normal.csv.zip')
+
+        handler = NdrImport::File::Zip.new(file_path, nil, options)
+        handler.files.each do |filename|
+          assert_instance_of SafePath, filename
+          assert_equal 'normal.csv', ::File.basename(filename)
+        end
+
+        exception = assert_raises RuntimeError do
+          handler.tables
+        end
+        assert_equal 'Zip#tables should never be called', exception.message
+      end
+    end
+  end
+end

data/test/helpers/file/delimited_test.rb

@@ -0,0 +1,113 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/delimited'
+
+# Delimited file helper tests
+class DelimitedTest < ActiveSupport::TestCase
+  # This is a test importer class to test the Delimited file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Delimited
+  end
+
+  def setup
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test 'should read csv correctly' do
+    rows = @importer.read_delimited_file(@permanent_test_files.join('normal.csv'), nil)
+    assert_equal(('A'..'Z').to_a, rows[0])
+    assert_equal ['1'] * 26, rows[1]
+    assert_equal ['2'] * 26, rows[2]
+  end
+
+  test 'should read csv with a BOM' do
+    rows = @importer.read_delimited_file(@permanent_test_files.join('bomd.csv'), nil)
+    assert_equal(('A'..'Z').to_a, rows[0])
+    assert_equal ['1'] * 26, rows[1]
+    assert_equal ['2'] * 26, rows[2]
+  end
+
+  test 'should read windows-1252 csv' do
+    rows = @importer.read_delimited_file(@permanent_test_files.join('windows.csv'), nil)
+    assert_equal 1, rows.length
+  end
+
+  test 'should read acsii-delimited csv' do
+    rows = @importer.read_delimited_file(@permanent_test_files.join('high_ascii_delimited.txt'),
+                                         "\xfe")
+    assert_equal 2, rows.length
+  end
+
+  test 'should read line-by-line' do
+    rows = []
+    @importer.delimited_rows(@permanent_test_files.join('normal.csv')) { |row| rows << row }
+    assert_equal(('A'..'Z').to_a, rows[0])
+    assert_equal ['1'] * 26, rows[1]
+    assert_equal ['2'] * 26, rows[2]
+  end
+
+  test 'should read line-by-line with custom delimiter' do
+    count = 0
+    file  = @permanent_test_files.join('high_ascii_delimited.txt')
+
+    @importer.delimited_rows(file, "\xfe") { count += 1 }
+    assert_equal 2, count
+  end
+
+  test 'should report addition details upon failure to slurp csv' do
+    exception = assert_raises(CSVLibrary::MalformedCSVError) do
+      @importer.read_delimited_file(@permanent_test_files.join('broken.csv'), nil)
+    end
+
+    msg = 'Invalid CSV format on row 2 of broken.csv. Original: Missing or stray quote in line 2'
+    assert_equal msg, exception.message
+  end
+
+  test 'should report addition details upon failure to read csv line-by-line' do
+    rows_yielded = []
+    exception    = assert_raises(CSVLibrary::MalformedCSVError) do
+      @importer.delimited_rows(@permanent_test_files.join('broken.csv')) do |row|
+        rows_yielded << row
+      end
+    end
+
+    assert rows_yielded.empty?, 'no rows should have been yielded'
+
+    msg = 'Invalid CSV format on row 2 of broken.csv. Original: Missing or stray quote in line 2'
+    assert_equal msg, exception.message
+  end
+
+  test 'delimited_tables should read table correctly' do
+    table = @importer.send(:delimited_tables, @permanent_test_files.join('normal.csv'))
+    table.each do |tablename, sheet|
+      assert_nil tablename
+      sheet = sheet.to_a
+      assert_equal(('A'..'Z').to_a, sheet[0])
+      assert_equal ['1'] * 26, sheet[1]
+      assert_equal ['2'] * 26, sheet[2]
+    end
+  end
+
+  def test_each_delimited_table_should_be_deprecated
+    original_stderr = $stderr
+    $stderr = StringIO.new
+
+    table = @importer.send(:each_delimited_table, @permanent_test_files.join('normal.csv'))
+    table.each do |tablename, sheet|
+      assert_nil tablename
+      sheet = sheet.to_a
+      assert_equal(('A'..'Z').to_a, sheet[0])
+      assert_equal ['1'] * 26, sheet[1]
+      assert_equal ['2'] * 26, sheet[2]
+    end
+
+    assert_match(/\A\[warning\] each_delimited_table will be deprecated/, $stderr.string)
+  ensure
+    $stderr = original_stderr
+  end if Gem::Requirement.new('< 3.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
+
+  def test_deprecated_methods_removed_in_v3
+    refute @importer.public_methods.include?(:each_delimited_table), 'should be removed in v3.0.0'
+    refute @importer.public_methods.include?(:each_delimited_row), 'should be removed in v3.0.0'
+  end if Gem::Requirement.new('>= 3.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
+end