ndr_import 8.5.0 → 8.5.1

data/gemfiles/Gemfile.rails50

@@ -1,8 +0,0 @@
-source 'https://rubygems.org'
-
-# ooxml_decrypt is not published to rubygems.org so we need to specify where it is
-gem 'ooxml_decrypt', git: 'https://github.com/timgentry/ooxml_decrypt'
-
-gemspec path: '..'
-
-gem 'activesupport', '~> 5.0.0'

data/gemfiles/Gemfile.rails51

@@ -1,9 +0,0 @@
-source 'https://rubygems.org'
-
-# ooxml_decrypt is not published to rubygems.org so we need to specify where it is
-gem 'ooxml_decrypt', git: 'https://github.com/timgentry/ooxml_decrypt'
-
-gemspec path: '..'
-
-# Minitest 5.10.2 doesn't work with Rails version
-gem 'minitest', '< 5.10.2'

data/test/file/acro_form_test.rb

@@ -1,39 +0,0 @@
-require 'test_helper'
-require 'ndr_import/file/acro_form'
-
-module NdrImport
-  module File
-    # Acro Form file handler tests
-    class AcroFormTest < ActiveSupport::TestCase
-      def setup
-        @permanent_test_files = SafePath.new('permanent_test_files')
-      end
-
-      test 'should read pdf correctly' do
-        file_path = @permanent_test_files.join('acro_form.pdf')
-        handler = NdrImport::File::AcroForm.new(file_path, nil)
-
-        expected_row = { 'Group3'                      => nil,
-                         'Textbox1_required'           => nil,
-                         'Textbox2_required'           => nil,
-                         'List Box_required'           => '3',
-                         'Dropdown_required'           => '3',
-                         'Textbox1_optional'           => nil,
-                         'Textbox2_optional'           => nil,
-                         'List Box_optional'           => '3',
-                         'Dropdown_optional'           => '3',
-                         'Date_required'               => nil,
-                         'Date__optional'              => nil,
-                         'Textbox3_numerical_required' => nil,
-                         'Textbox3_numerical_optional' => nil }
-
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal 1, sheet.to_a.size
-          assert_equal expected_row, sheet.to_a.first
-        end
-      end
-    end
-  end
-end

data/test/file/base_test.rb

@@ -1,54 +0,0 @@
-require 'test_helper'
-require 'ndr_import/file/registry'
-
-module NdrImport
-  module File
-    # Base file handler tests
-    class BaseTest < ActiveSupport::TestCase
-      # Handles a single table file, but for test purposes,
-      # I could be bothered to implement it fully
-      class SingleTableLazyDeveloper < ::NdrImport::File::Base
-      end
-
-      def setup
-        @permanent_test_files = SafePath.new('permanent_test_files')
-      end
-
-      test 'should fail on not implementing rows' do
-        begin
-          Registry.register(SingleTableLazyDeveloper, 'lazy_dev')
-
-          exception = assert_raises(RuntimeError) do
-            file_path = @permanent_test_files.join('normal.csv')
-            handler = SingleTableLazyDeveloper.new(file_path, 'lazy_dev')
-
-            handler.tables.each do |tablename, sheet|
-              assert_nil tablename
-              assert_instance_of Enumerator, sheet
-              sheet.to_a
-            end
-          end
-
-          msg = 'Implement NdrImport::File::BaseTest::SingleTableLazyDeveloper#rows'
-          assert_equal msg, exception.message
-        ensure
-          Registry.unregister('lazy_dev')
-        end
-      end
-
-      test 'should not fail when set up with an readable safepath' do
-        assert Base.new(SafePath.new('test_space_r'), nil)
-        assert Base.new(SafePath.new('test_space_rw'), nil)
-      end
-
-      test 'should fail when set up with an unreadable safepath' do
-        assert_raises(SecurityError) { Base.new(SafePath.new('test_space_w'), nil) }
-      end
-
-      test 'should fail when set up with a non-safepath' do
-        exception = assert_raises(ArgumentError) { Base.new(NdrImport.root, nil) }
-        assert exception.message =~ /file_name should be of type SafePath, but it is String/
-      end
-    end
-  end
-end

data/test/file/delimited_test.rb

@@ -1,233 +0,0 @@
-require 'test_helper'
-require 'ndr_import/file/delimited'
-
-module NdrImport
-  module File
-    # Delimited file handler tests
-    class DelimitedTest < ActiveSupport::TestCase
-      def setup
-        @permanent_test_files = SafePath.new('permanent_test_files')
-      end
-
-      test 'should read csv correctly' do
-        file_path = @permanent_test_files.join('normal.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          sheet = sheet.to_a
-          assert_equal(('A'..'Z').to_a, sheet[0])
-          assert_equal ['1'] * 26, sheet[1]
-          assert_equal ['2'] * 26, sheet[2]
-        end
-      end
-
-      test 'should read pipe correctly' do
-        file_path = @permanent_test_files.join('normal_pipe.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => '|')
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          sheet = sheet.to_a
-          assert_equal(('A'..'Z').to_a, sheet[0])
-          assert_equal ['1'] * 26, sheet[1]
-          assert_equal ['2'] * 26, sheet[2]
-        end
-      end
-
-      test 'should read malformed pipe correctly' do
-        file_path = @permanent_test_files.join('malformed_pipe.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => '|',
-                                                                         'liberal_parsing' => 'true')
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          sheet = sheet.to_a
-          assert_equal(('A'..'Z').to_a, sheet[0])
-          assert_equal ['1'] * 26, sheet[1]
-          expected_row = ['2'] * 25
-          expected_row << '2"malformed"'
-          assert_equal expected_row, sheet[2].sort
-        end
-      end
-
-      test 'should read thorn correctly' do
-        file_path = @permanent_test_files.join('normal_thorn.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfe")
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          sheet = sheet.to_a
-          assert_equal(('A'..'Z').to_a, sheet[0])
-          assert_equal ['1'] * 26, sheet[1]
-          assert_equal ['2'] * 26, sheet[2]
-        end
-      end
-
-      test 'should read csv with a BOM' do
-        file_path = @permanent_test_files.join('bomd.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet = sheet.to_a
-          assert_equal(('A'..'Z').to_a, sheet[0])
-          assert_equal ['1'] * 26, sheet[1]
-          assert_equal ['2'] * 26, sheet[2]
-        end
-      end
-
-      test 'should read windows-1252 csv' do
-        file_path = @permanent_test_files.join('windows.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet = sheet.to_a
-          assert_equal 1, sheet.length
-        end
-      end
-
-      test 'should read acsii-delimited csv' do
-        file_path = @permanent_test_files.join('high_ascii_delimited.txt')
-        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfe")
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet = sheet.to_a
-          assert_equal 2, sheet.length
-          assert_equal '1234567890', sheet[0][1]
-          assert_equal '1234567890', sheet[1][1]
-        end
-      end
-
-      test 'should read acsii-delimited txt' do
-        rows = []
-        file_path = @permanent_test_files.join('high_ascii_delimited_example_two.txt')
-        handler = NdrImport::File::Delimited.new(file_path, 'delimited', 'col_sep' => "\xfd")
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet.each do |row|
-            rows << row
-          end
-        end
-
-        assert_equal(('A'..'Z').to_a, rows[0])
-        assert_equal ['1'] * 26, rows[1]
-        assert_equal ['2'] * 26, rows[2]
-      end
-
-      test 'should read malformed delimited txt' do
-        rows = []
-        file_path = @permanent_test_files.join('malformed.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil,
-                                                                   'liberal_parsing' => 'true')
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet.each do |row|
-            rows << row
-          end
-        end
-
-        assert_equal(('A'..'Z').to_a, rows[0])
-        assert_equal ['1'] * 26, rows[1]
-        expected_row = ['2'] * 25
-        expected_row << '2"malformed"'
-        assert_equal expected_row, rows[2].sort
-      end
-
-      test 'should fail to read malformed delimited txt without liberal_parsing' do
-        rows_yielded = []
-        exception    = assert_raises(CSVLibrary::MalformedCSVError) do
-          file_path = @permanent_test_files.join('malformed.csv')
-          handler = NdrImport::File::Delimited.new(file_path, 'csv')
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.each do |row|
-              rows_yielded << row
-            end
-          end
-        end
-
-        assert rows_yielded.empty?, 'no rows should have been yielded'
-
-        msg = 'Invalid CSV format on row 3 of malformed.csv. Original: Illegal quoting in line 3.'
-        assert_equal msg, exception.message
-      end
-
-      test 'should read line-by-line' do
-        rows = []
-        file_path = @permanent_test_files.join('normal.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv')
-
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          sheet.each do |row|
-            rows << row
-          end
-        end
-
-        assert_equal(('A'..'Z').to_a, rows[0])
-        assert_equal ['1'] * 26, rows[1]
-        assert_equal ['2'] * 26, rows[2]
-      end
-
-      test 'should report addition details upon failure to slurp csv' do
-        exception = assert_raises(CSVLibrary::MalformedCSVError) do
-          file_path = @permanent_test_files.join('broken.csv')
-          handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.to_a
-          end
-        end
-
-        assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
-        assert_match(CORRUPTED_QUOTES_MESSAGE_PATTERN, exception.message)
-        assert_match(/in line 2/, exception.message)
-      end
-
-      test 'should report addition details upon failure to read csv line-by-line' do
-        rows_yielded = []
-        exception    = assert_raises(CSVLibrary::MalformedCSVError) do
-          file_path = @permanent_test_files.join('broken.csv')
-          handler = NdrImport::File::Delimited.new(file_path, 'csv')
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.each do |row|
-              rows_yielded << row
-            end
-          end
-        end
-
-        assert rows_yielded.empty?, 'no rows should have been yielded'
-
-        assert_match(/Invalid CSV format on row 2 of broken\.csv\./, exception.message)
-        assert_match(CORRUPTED_QUOTES_MESSAGE_PATTERN, exception.message)
-        assert_match(/in line 2/, exception.message)
-      end
-
-      test 'should only determine encodings once' do
-        file_path = @permanent_test_files.join('normal.csv')
-        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
-
-        handler.expects(try_each_encoding: { mode: 'r:bom|utf-8', col_sep: ',' }).once
-
-        2.times do
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            sheet = sheet.to_a
-            assert_equal(('A'..'Z').to_a, sheet[0])
-            assert_equal ['1'] * 26, sheet[1]
-            assert_equal ['2'] * 26, sheet[2]
-          end
-        end
-      end
-    end
-  end
-end

data/test/file/docx_test.rb

@@ -1,53 +0,0 @@
-require 'test_helper'
-require 'ndr_import/file/docx'
-
-module NdrImport
-  module File
-    # Word .docx document file handler tests
-    class DocxTest < ActiveSupport::TestCase
-      def setup
-        @permanent_test_files = SafePath.new('permanent_test_files')
-      end
-
-      test 'should read word file' do
-        file_path = @permanent_test_files.join('hello_world.docx')
-        handler = NdrImport::File::Docx.new(file_path, nil)
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal [
-            'Hello world, this is a modern word document',
-            'With more than one line of text',
-            'Three in fact'
-          ], sheet.to_a
-        end
-      end
-
-      test 'should read password protected word file' do
-        file_path = @permanent_test_files.join('password_protected_hello_world.docx')
-        handler = NdrImport::File::Docx.new(file_path, nil, file_password: 'salad')
-        handler.tables.each do |tablename, sheet|
-          assert_nil tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal [
-            'Hello world, this is a modern word document',
-            'With more than one line of text',
-            'Three in fact'
-          ], sheet.to_a
-        end
-      end
-
-      test 'should raise exception on invalid word file' do
-        assert_raises RuntimeError do
-          file_path = @permanent_test_files.join('not_a_word_file.docx')
-          handler = NdrImport::File::Docx.new(file_path, nil)
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.to_a
-          end
-        end
-      end
-    end
-  end
-end

data/test/file/excel_test.rb

@@ -1,124 +0,0 @@
-require 'test_helper'
-require 'ndr_import/file/excel'
-
-module NdrImport
-  module File
-    # Excel file handler tests
-    class ExcelTest < ActiveSupport::TestCase
-      def setup
-        @permanent_test_files = SafePath.new('permanent_test_files')
-      end
-
-      test 'should read xls file' do
-        file_path = @permanent_test_files.join('sample_xls.xls')
-        handler = NdrImport::File::Excel.new(file_path, nil)
-        handler.tables.each do |tablename, sheet|
-          assert_equal 'Sheet1', tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal %w(1A 1B), sheet.first
-        end
-      end
-
-      test 'should read xlsx file' do
-        file_path = @permanent_test_files.join('sample_xlsx.xlsx')
-        handler = NdrImport::File::Excel.new(file_path, nil)
-        handler.tables.each do |tablename, sheet|
-          assert_equal 'Sheet1', tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal %w(1A 1B), sheet.first
-        end
-      end
-
-      test 'should read xlsx file with null sheet' do
-        file_path = @permanent_test_files.join('blank_tab_test.xlsx')
-        handler   = NdrImport::File::Excel.new(file_path, nil)
-
-        sheets = handler.tables.map { |_tablename, sheet| sheet }
-
-        assert_equal %w(column_a column_b column_c column_d), sheets[0].next
-        assert_equal %w(11111 ABC123 8888888888 2), sheets[0].next
-        assert_equal %w(column_a column_b column_c column_d), sheets[1].next
-        assert_equal %w(11111 ABC123 8888888888 3), sheets[1].next
-        assert_raises(StopIteration) { sheets[2].next }
-      end
-
-      %w(sheet_streaming.xlsx sheet_streaming.xls).each do |filename|
-        test "should be able to stream from multiple sheets at once from #{filename}" do
-          file_path = @permanent_test_files.join(filename)
-          handler   = NdrImport::File::Excel.new(file_path, nil)
-
-          sheets = handler.tables.map { |_tablename, sheet| sheet }
-
-          assert_equal %w(1A1 1B1), sheets[0].next
-          assert_raises(StopIteration) { sheets[2].next }
-          assert_equal %w(2A1 2B1), sheets[1].next
-          assert_equal %w(2A2 2B2), sheets[1].next
-          assert_equal %w(1A2 1B2), sheets[0].next
-          assert_raises(StopIteration) { sheets[2].next }
-        end
-      end
-
-      test 'should read xlsx file with the incorrect xls extension' do
-        file_path = @permanent_test_files.join('xlsx_file_xls_extension.xls')
-        handler = NdrImport::File::Excel.new(file_path, nil)
-        handler.tables.each do |tablename, sheet|
-          assert_equal 'Sheet1', tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal %w(1A 1B), sheet.first
-        end
-
-        SafeFile.delete @permanent_test_files.join('xlsx_file_xls_extension_amend.xlsx')
-      end
-
-      test 'should read password protected xlsx file' do
-        file_path = @permanent_test_files.join('password_protected_sample_xlsx.xlsx')
-        handler = NdrImport::File::Excel.new(file_path, nil, file_password: 'carrot')
-        handler.tables.each do |tablename, sheet|
-          assert_equal 'Sheet1', tablename
-          assert_instance_of Enumerator, sheet
-          assert_equal %w(1A 1B), sheet.first
-        end
-      end
-
-      test 'read_excel_file helper should handle exceptions' do
-        # txt file
-        assert_raises RuntimeError do
-          file_path = @permanent_test_files.join('flat_file.txt')
-          handler = NdrImport::File::Excel.new(file_path, nil)
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.to_a
-          end
-        end
-
-        # .txt file in .xls extension
-        assert_raises RuntimeError do
-          file_path = @permanent_test_files.join('txt_file_xls_extension.xls')
-          handler = NdrImport::File::Excel.new(file_path, 'txt')
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.to_a
-          end
-        end
-
-        # .txt file in .xlsx extension
-        assert_raises RuntimeError do
-          file_path = @permanent_test_files.join('txt_file_xlsx_extension.xlsx')
-          handler = NdrImport::File::Excel.new(file_path, 'txt')
-
-          handler.tables.each do |tablename, sheet|
-            assert_nil tablename
-            assert_instance_of Enumerator, sheet
-            sheet.to_a
-          end
-        end
-
-        SafeFile.delete @permanent_test_files.join('txt_file_xls_extension_amend.xlsx')
-      end
-    end
-  end
-end