ndr_import 3.0.0

data/test/helpers/file/excel_test.rb

@@ -0,0 +1,97 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/excel'
+
+# Excel file helper tests
+class ExcelTest < ActiveSupport::TestCase
+  # This is a test importer class to test the excel file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Excel
+  end
+
+  def setup
+    @permanenttestfiles = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test 'read_excel_file helper should read xls file' do
+    file_content = @importer.send(:read_excel_file, @permanenttestfiles.join('sample_xls.xls'))
+    assert_equal file_content.count, 2
+    assert_equal file_content.first, %w(1A 1B)
+  end
+
+  test 'read_excel_file helper should read xlsx file' do
+    file_content = @importer.send(:read_excel_file, @permanenttestfiles.join('sample_xlsx.xlsx'))
+    assert_equal file_content.count, 2
+    assert_equal file_content.first, %w(1A 1B)
+  end
+
+  test 'read_excel_file helper should read xlsx file with the incorrect xls extension' do
+    file_path = @permanenttestfiles.join('xlsx_file_xls_extension.xls')
+    file_content = @importer.send(:read_excel_file, file_path)
+    assert_equal file_content.count, 2
+    assert_equal file_content.first, %w(1A 1B)
+
+    SafeFile.delete @permanenttestfiles.join('xlsx_file_xls_extension_amend.xlsx')
+  end
+
+  test 'read_excel_file helper should handle exceptions' do
+    # txt file
+    SafeFile.open(@permanenttestfiles.join('temp.txt'), 'w') { |f| f.write 'dummy line' }
+    assert_raises RuntimeError do
+      @importer.send(:read_excel_file, @permanenttestfiles.join('temp.txt'))
+    end
+
+    # .txt file in .xls extension
+    File.rename @permanenttestfiles.join('temp.txt'), @permanenttestfiles.join('temp.xls')
+    assert_raises RuntimeError do
+      @importer.send(:read_excel_file, @permanenttestfiles.join('temp.xls'))
+    end
+
+    # .txt file in .xlsx extension
+    File.rename @permanenttestfiles.join('temp.xls'), @permanenttestfiles.join('temp.xlsx')
+    assert_raises RuntimeError do
+      @importer.send(:read_excel_file, @permanenttestfiles.join('temp.xlsx'))
+    end
+
+    SafeFile.delete @permanenttestfiles.join('temp.xlsx')
+    SafeFile.delete @permanenttestfiles.join('temp_amend.xlsx')
+  end
+
+  test 'excel_tables helper should read xls table correctly' do
+    table = @importer.send(:excel_tables, @permanenttestfiles.join('sample_xls.xls'))
+    table.each do |tablename, sheet|
+      assert_equal 'Sheet1', tablename
+      assert_equal %w(1A 1B), sheet.first
+    end
+  end
+
+  test 'excel_tables helper should read xlsx table correctly' do
+    table = @importer.send(:excel_tables, @permanenttestfiles.join('sample_xlsx.xlsx'))
+    table.each do |tablename, sheet|
+      assert_equal 'Sheet1', tablename
+      assert_equal %w(1A 1B), sheet.first
+    end
+  end
+
+  def test_each_excel_table_should_be_deprecated
+    original_stderr = $stderr
+    $stderr = StringIO.new
+
+    table = @importer.send(:each_excel_table, @permanenttestfiles.join('sample_xlsx.xlsx'))
+    table.each do |tablename, sheet|
+      assert_equal 'Sheet1', tablename
+      assert_equal %w(1A 1B), sheet.first
+    end
+
+    assert_match(/\A\[warning\] each_excel_table will be deprecated/, $stderr.string)
+  ensure
+    $stderr = original_stderr
+  end if Gem::Requirement.new('< 3.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
+
+  def test_deprecated_methods_removed_in_v3
+    refute @importer.protected_methods.include?(:each_excel_table), 'should be removed in v3.0.0'
+    refute @importer.private_methods.include?(:each_excel_row), 'should be removed in v3.0.0'
+    refute @importer.private_methods.include?(:each_xls_row), 'should be removed in v3.0.0'
+    refute @importer.private_methods.include?(:each_xlsx_row), 'should be removed in v3.0.0'
+  end if Gem::Requirement.new('>= 3.0.0').satisfied_by?(Gem::Version.new(NdrImport::VERSION))
+end

data/test/helpers/file/pdf_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/pdf'
+
+# PDF file helper tests
+class PdfTest < ActiveSupport::TestCase
+  # This is a test importer class to test the PDF file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Pdf
+  end
+
+  def setup
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test 'read_pdf_file helper should read pdf file' do
+    file_content = @importer.send(:read_pdf_file, @permanent_test_files.join('hello_world.pdf'))
+    assert_equal file_content, ['Hello World']
+  end
+
+  test 'read_pdf_file helper should raise exception on invalid pdf file' do
+    assert_raises RuntimeError do
+      @importer.send(:read_pdf_file, @permanent_test_files.join('not_a_pdf.pdf'))
+    end
+  end
+end

data/test/helpers/file/word_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/word'
+
+# Word file helper tests
+class WordTest < ActiveSupport::TestCase
+  # This is a test importer class to test the Word file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Word
+  end
+
+  def setup
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test 'read_word_file helper should read word file' do
+    file_content = @importer.send(:read_word_file, @permanent_test_files.join('hello_world.doc'))
+    assert_equal file_content, ['Hello world, this is a word document']
+  end
+
+  test 'read_word_file helper should raise exception on invalid word file' do
+    assert_raises RuntimeError do
+      @importer.send(:read_word_file, @permanent_test_files.join('not_a_word_file.doc'))
+    end
+  end
+end

data/test/helpers/file/xml_test.rb

@@ -0,0 +1,131 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/xml'
+require 'nokogiri'
+
+# XML file helper tests
+class XmlTest < ActiveSupport::TestCase
+  # This is a test importer class to test the XML file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Xml
+  end
+
+  def setup
+    @home = SafePath.new('test_space_rw')
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test 'import_xml_file should handle incoming UTF-8' do
+    doc   = @importer.send(:read_xml_file, @permanent_test_files.join('utf-8_xml.xml'))
+    greek = doc.xpath('//letter').map(&:text).join
+
+    assert_equal 'UTF-8', doc.encoding
+
+    assert greek.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), greek.encoding
+    assert_equal 2, greek.chars.to_a.length
+    assert_equal [206, 177, 206, 178], greek.bytes.to_a # 2-bytes each for alpha and beta
+  end
+
+  test 'import_xml_file should handle incoming UTF-16 (big endian)' do
+    doc   = @importer.send(:read_xml_file, @permanent_test_files.join('utf-16be_xml.xml'))
+    greek = doc.xpath('//letter').map(&:text).join
+
+    assert_equal 'UTF-8', doc.encoding
+
+    assert greek.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), greek.encoding
+    assert_equal 2, greek.chars.to_a.length
+    assert_equal [206, 177, 206, 178], greek.bytes.to_a # 2-bytes each for alpha and beta
+  end
+
+  test 'import_xml_file should handle incoming UTF-16 (little endian)' do
+    doc   = @importer.send(:read_xml_file, @permanent_test_files.join('utf-16le_xml.xml'))
+    greek = doc.xpath('//letter').map(&:text).join
+
+    assert_equal 'UTF-8', doc.encoding
+
+    assert greek.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), greek.encoding
+    assert_equal 2, greek.chars.to_a.length
+    assert_equal [206, 177, 206, 178], greek.bytes.to_a # 2 bytes each for alpha and beta
+  end
+
+  test 'import_xml_file should handle incoming UTF-16 with declaration' do
+    doc   = @importer.send(:read_xml_file,
+                           @permanent_test_files.join('utf-16be_xml_with_declaration.xml'))
+    greek = doc.xpath('//letter').map(&:text).join
+
+    assert greek.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), greek.encoding
+    assert_equal 2, greek.chars.to_a.length
+    assert_equal [206, 177, 206, 178], greek.bytes.to_a # 2 bytes each for alpha and beta
+
+    # The document should be UTF-8, and we shouldn't
+    # get encoding mismatches when interrogating it:
+    assert_equal 'UTF-8', doc.encoding
+    assert_equal 1, doc.css('note[id=alpha]').length
+  end
+
+  test 'import_xml_file should handle incoming Windows-1252' do
+    doc   = @importer.send(:read_xml_file, @permanent_test_files.join('windows-1252_xml.xml'))
+    punct = doc.xpath('//letter').map(&:text).join
+
+    assert_equal 'UTF-8', doc.encoding
+
+    assert punct.valid_encoding?
+    assert_equal Encoding.find('UTF-8'), punct.encoding
+    assert_equal 2, punct.chars.to_a.length
+    assert_equal [226, 128, 153, 226, 128, 147], punct.bytes.to_a # 3 bytes each for apostrophe and dash
+  end
+
+  test 'import_xml_file with malformed XML file' do
+    assert_raises Nokogiri::XML::SyntaxError do
+      @importer.send(:read_xml_file, @permanent_test_files.join('malformed.xml'))
+    end
+  end
+
+  test '.import_xml_file should reject non safe path arguments' do
+    assert_raises ArgumentError do
+      @importer.send(:read_xml_file, @home.join('simple.xml').to_s)
+    end
+  end
+
+  test '.import_xml_file should accept safepath' do
+    builder = Nokogiri::XML::Builder.new do |xml|
+      xml.root do
+        xml.note(:id => 1) do
+          xml.time 'Thu Dec 13 13:12:00 UTC 2012'
+          xml.title 'Note 1'
+          xml.body 'Note 1 body blabla bla'
+        end
+        xml.note(:id => 2) do
+          xml.time 'Thu Dec 14 12:11:00 UTC 2012'
+          xml.title 'note 2'
+          xml.body 'note 2 body blablabala'
+        end
+      end
+    end
+    SafeFile.open(@home.join('simple.xml'), 'w') { |f| f.write builder.to_xml }
+
+    doc = @importer.send(:read_xml_file, @home.join('simple.xml'))
+
+    assert_equal 1, doc.children.reject { |c| c.text =~ /\A\n *\Z/ }.length
+    assert_equal 'root', doc.children.reject { |c| c.text =~ /\A\n *\Z/ }[0].name
+    assert_equal 2, doc.
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }.length
+    assert_equal 'note', doc.children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].name
+    assert_equal 3, doc.
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }.length
+    assert_equal 'Thu Dec 13 13:12:00 UTC 2012', doc.
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].
+      children.reject { |c| c.text =~ /\A\n *\Z/ }[0].text
+
+    SafeFile.delete @home.join('simple.xml')
+  end
+end

data/test/helpers/file/zip_test.rb

@@ -0,0 +1,75 @@
+require 'test_helper'
+require 'ndr_import/helpers/file/zip'
+require 'zip'
+
+# Zip file helper tests
+class ZipTest < ActiveSupport::TestCase
+  # This is a test importer class to test the Zip file helper mixin
+  class TestImporter
+    include NdrImport::Helpers::File::Zip
+  end
+
+  def setup
+    @home = SafePath.new('test_space_rw')
+    @permanent_test_files = SafePath.new('permanent_test_files')
+    @importer = TestImporter.new
+  end
+
+  test '.unzip should reject non SafePath arguments' do
+    zip = @home.join('imaginary.zip')
+
+    assert_raises ArgumentError do
+      @importer.send(:unzip_file, zip.to_s, @home.to_s)
+    end
+
+    assert_raises ArgumentError do
+      @importer.send(:unzip_file, zip.to_s, @home)
+    end
+
+    assert_raises ArgumentError do
+      @importer.send(:unzip_file, zip, @home.to_s)
+    end
+  end
+
+  test '.unzip unzip zip file' do
+    zip_name = @home.join('test.zip')
+
+    files = [
+      @home.join('f1'),
+      @home.join('f2'),
+      @home.join('f3')
+    ]
+
+    files.each do |fname|
+      File.open(fname, 'w') { |f| f.write "test #{fname}" }
+    end
+
+    ::Zip::File.open(zip_name, Zip::File::CREATE) do |zipfile|
+      files.each do |fname|
+        zipfile.add(File.basename(fname.to_s), fname.to_s)
+      end
+    end
+
+    File.delete(*files)
+
+    files.each do |fname|
+      assert !File.exist?(fname)
+    end
+
+    assert File.exist?(zip_name)
+    dest = @home.join('unziped')
+
+    @importer.send(:unzip_file, zip_name, dest)
+
+    files.each do |fname|
+      assert File.exist?(dest.join(File.basename(fname)))
+    end
+
+    files.each do |fname|
+      File.delete(dest.join(File.basename(fname)))
+    end
+
+    File.delete(zip_name)
+    FileUtils.rm_r(dest)
+  end
+end

data/test/mapper_test.rb

@@ -0,0 +1,551 @@
+require 'test_helper'
+
+# expose private methods
+class TestMapper
+  include NdrImport::Mapper
+
+  public :fixed_width_columns, :mapped_line, :mapped_value, :replace_before_mapping
+
+  # TODO: test fixed_width_columns
+end
+
+# This tests the NdrImport::Mapper class
+class MapperTest < ActiveSupport::TestCase
+  def setup
+    @permanent_test_files = SafePath.new('permanent_test_files')
+  end
+
+  format_mapping = { 'format' => 'dd/mm/yyyy' }
+  format_mapping_yyyymmdd = { 'format' => 'yyyymmdd' }
+  clean_name_mapping = { 'clean' => :name }
+  clean_ethniccategory_mapping = { 'clean' => :ethniccategory }
+  clean_icd_mapping = { 'clean' => :code_icd }
+  clean_opcs_mapping = { 'clean' => :code_opcs }
+  map_mapping = { 'map' => { 'A' => '1' } }
+  replace_mapping = { 'replace' => { '.0' => '' } }
+  daysafter_mapping = { 'daysafter' => '2012-05-16' }
+  # TODO: match_mapping = {}
+
+  simple_mapping = [{ 'column' => 'patient address', 'mappings' => ['field' => 'address'] }]
+
+  simple_mapping_with_clean_opcs = YAML.load <<-YML
+    - column: primaryprocedures
+      mappings:
+      - field: primaryprocedures
+        clean: :code_opcs
+  YML
+
+  join_mapping = YAML.load <<-YML
+    - column: forename1
+      mappings:
+      - field: forenames
+        order: 1
+        join: " "
+    - column: forename2
+      mappings:
+      - field: forenames
+        order: 2
+  YML
+
+  join_compact_mapping = YAML.load <<-YML
+    - column: forename1
+      mappings:
+      - field: forenames
+        order: 1
+        join: " "
+        compact: false
+    - column: forename2
+      mappings:
+      - field: forenames
+        order: 2
+  YML
+
+  unused_mapping = [{ 'column' => 'extra', 'rawtext_name' => 'extra' }]
+
+  cross_populate_mapping = YAML.load <<-YML
+    - column: referringclinicianname
+      mappings:
+      - field: consultantname
+      - field: consultantcode
+        priority: 2
+    - column: referringcliniciancode
+      mappings:
+      - field: consultantcode
+  YML
+
+  cross_populate_replace_mapping = YAML.load <<-YML
+    - column: referringclinicianname
+      mappings:
+      - field: consultantname
+      - field: consultantcode
+        priority: 2
+        replace:
+          ? !ruby/regexp /^BOB FOSSIL$/i
+          : "ROBERT FOSSIL"
+    - column: referringcliniciancode
+      mappings:
+      - field: consultantcode
+        priority: 1
+  YML
+
+  cross_populate_map_mapping = YAML.load <<-YML
+    - column: referringclinicianname
+      mappings:
+      - field: consultantname
+      - field: consultantcode
+        priority: 2
+        map:
+          "Bob Fossil": "C5678"
+    - column: referringcliniciancode
+      mappings:
+      - field: consultantcode
+        priority: 1
+  YML
+
+  cross_populate_order_mapping = YAML.load <<-YML
+    - column: referringclinicianname
+      mappings:
+      - field: consultantname
+      - field: consultantcode
+        priority: 2
+    - column: referringcliniciancode
+      mappings:
+      - field: consultantcode
+        priority: 1
+    - column: somecolumn
+      mappings:
+      - field: consultantcode
+        priority: 5
+    - column: anothercolumn
+      mappings:
+      - field: consultantcode
+        priority: 10
+  YML
+
+  cross_populate_no_priority = YAML.load <<-YML
+    - column: columnoneraw
+      mappings:
+      - field: columnone
+      - field: columntwo
+    - column: columntworaw
+      mappings:
+      - field: columntwo
+        priority: 5
+  YML
+
+  standard_mapping_without = YAML.load <<-YML
+    - column: surname
+      rawtext_name: surname
+      mappings:
+      - field: surname
+        clean: :name
+    - column: forename
+      rawtext_name: forenames
+      mappings:
+      - field: forenames
+        clean: :name
+    - column: sex
+      rawtext_name: sex
+      mappings:
+      - field: sex
+        clean: :sex
+    - column: nhs_no
+      rawtext_name: nhsnumber
+      mappings:
+      - field: nhsnumber
+        clean: :nhsnumber
+  YML
+
+  standard_mapping_with = YAML.load <<-YML
+    - standard_mapping: surname
+    - column: forename
+      standard_mapping: forenames
+    - standard_mapping: sex
+    - column: nhs_no
+      standard_mapping: nhsnumber
+  YML
+
+  standard_mapping_merge = YAML.load <<-YML
+    - column: surname
+      standard_mapping: surname
+      mappings:
+      - field: surname2
+  YML
+
+  standard_mapping_column = YAML.load <<-YML
+    - column: overriding_column_name
+      standard_mapping: test
+  YML
+
+  invalid_priorities = YAML.load <<-YML
+    - column: columnoneraw
+      mappings:
+      - field: columnone
+      - field: columntwo
+        priority: 5
+    - column: columntworaw
+      mappings:
+      - field: columntwo
+        priority: 5
+  YML
+
+  invalid_standard_mapping = YAML.load <<-YML
+    - column: surname
+      standard_mapping: surnames
+  YML
+
+  joined_mapping_blank_start = YAML.load <<-YML
+    - column: addressoneraw
+      mappings:
+      - field: address
+        join: ","
+        order: 1
+    - column: postcode
+      mappings:
+      - field: address
+        order: 2
+  YML
+
+  joined_mapping_blank_start_uncompacted = YAML.load <<-YML
+    - column: addressoneraw
+      mappings:
+      - field: address
+        join: ","
+        order: 1
+        compact: false
+    - column: postcode
+      mappings:
+      - field: address
+        order: 2
+  YML
+
+  date_mapping = YAML.load <<-YML
+    - column: birth_date
+      rawtext_name: dateofbirth
+      mappings:
+      - field: dateofbirth
+        format: dd/mm/yyyy
+    - column: received_date
+      rawtext_name: receiveddate
+      mappings:
+      - field: receiveddate
+        format: yyyymmdd
+    - column: american_date
+      rawtext_name: americandate
+      mappings:
+      - field: americandate
+        format: mm/dd/yyyy
+    - column: short_date
+      rawtext_name: shortdate
+      mappings:
+      - field: shortdate
+        format: dd/mm/yy
+    - column: funky_date
+      rawtext_name: funkydate
+      mappings:
+      - field: funkydate
+        format: dd/mmm/yy
+  YML
+
+  do_not_capture_column = YAML.load <<-YML
+    - column: ignore_me
+      do_not_capture: true
+  YML
+
+  base64_mapping = YAML.load <<-YML
+    - column: base64
+      decode:
+      - :base64
+      - :word_doc
+  YML
+
+  invalid_decode_mapping = YAML.load <<-YML
+    - column: column_name
+      decode:
+      - :invalid_encoding
+  YML
+
+  test 'map should return a number' do
+    assert_equal '1', TestMapper.new.mapped_value('A', map_mapping)
+  end
+
+  test 'map should return nil' do
+    assert_nil TestMapper.new.mapped_value('B', map_mapping)
+  end
+
+  test 'map should return correct date format' do
+    assert_equal Date.new(2011, 1, 25), TestMapper.new.mapped_value('25/01/2011', format_mapping)
+    assert_equal Date.new(2011, 1, 25),
+                 TestMapper.new.mapped_value('20110125', format_mapping_yyyymmdd)
+  end
+
+  test 'map should return incorrect date format' do
+    assert_not_equal Date.new(2011, 3, 4),
+                     TestMapper.new.mapped_value('03/04/2011', format_mapping)
+  end
+
+  test 'map should return nil date format' do
+    assert_nil TestMapper.new.mapped_value('03/25/2011', format_mapping)
+  end
+
+  test 'map should replace value' do
+    value = '2.0'
+    TestMapper.new.replace_before_mapping(value, replace_mapping)
+    assert_equal '2', value
+  end
+
+  test 'map should not alter value' do
+    value = '2.1'
+    TestMapper.new.replace_before_mapping(value, replace_mapping)
+    assert_equal '2.1', value
+  end
+
+  test 'map should clean name' do
+    assert_equal 'ANNABELLE SMITH',
+                 TestMapper.new.mapped_value('anna.belle,smith', clean_name_mapping)
+  end
+
+  test 'map should clean ethenic category' do
+    assert_equal 'M', TestMapper.new.mapped_value('1', clean_ethniccategory_mapping)
+    assert_equal 'X', TestMapper.new.mapped_value('99', clean_ethniccategory_mapping)
+    assert_equal 'A', TestMapper.new.mapped_value('A', clean_ethniccategory_mapping)
+    assert_equal 'INVALID', TestMapper.new.mapped_value('InValiD', clean_ethniccategory_mapping)
+  end
+
+  test 'map should clean icd code' do
+    assert_equal 'C34.3 R93.2 Z51.5',
+                 TestMapper.new.mapped_value('C34.3,R93.2,Z51.5', clean_icd_mapping)
+  end
+
+  test 'map should clean opcs code' do
+    assert_equal 'U212 Y973', TestMapper.new.mapped_value('U212,Y973,X1', clean_opcs_mapping)
+    assert_equal '', TestMapper.new.mapped_value('98', clean_opcs_mapping)
+    assert_equal '', TestMapper.new.mapped_value('TooLong', clean_opcs_mapping)
+    assert_equal nil, TestMapper.new.mapped_value('', clean_opcs_mapping)
+    assert_equal 'ABCD', TestMapper.new.mapped_value('AbcD', clean_opcs_mapping)
+    assert_equal '1234', TestMapper.new.mapped_value('1234', clean_opcs_mapping)
+  end
+
+  test 'should return correct date format for date fields with daysafter' do
+    assert_equal Date.new(2012, 5, 18), TestMapper.new.mapped_value(2, daysafter_mapping)
+    assert_equal Date.new(2012, 5, 18), TestMapper.new.mapped_value('2', daysafter_mapping)
+    assert_equal Date.new(2012, 5, 14), TestMapper.new.mapped_value(-2, daysafter_mapping)
+    assert_equal Date.new(2012, 5, 14), TestMapper.new.mapped_value('-2', daysafter_mapping)
+    assert_equal Date.new(2012, 5, 16), TestMapper.new.mapped_value(0, daysafter_mapping)
+    assert_equal 'String', TestMapper.new.mapped_value('String', daysafter_mapping)
+    assert_equal '', TestMapper.new.mapped_value('', daysafter_mapping)
+    assert_nil TestMapper.new.mapped_value(nil, daysafter_mapping)
+    assert_equal Date.new(2057, 8, 23), TestMapper.new.mapped_value(16_535, daysafter_mapping)
+    # Answer independently checked http://www.wolframalpha.com/input/?i=2012-05-16+%2B+9379+days
+    assert_equal Date.new(2038, 1, 19), TestMapper.new.mapped_value(9379, daysafter_mapping)
+    assert_equal Date.new(1946, 5, 11),
+                 TestMapper.new.mapped_value(16_900, 'daysafter' => '1900-02-01')
+    assert_equal Date.new(2014, 4, 8),
+                 TestMapper.new.mapped_value(16_900, 'daysafter' => '1967-12-31')
+    assert_equal Date.new(2046, 4, 9),
+                 TestMapper.new.mapped_value(16_900, 'daysafter' => '2000-01-01')
+  end
+
+  test 'line mapping should create valid hash' do
+    line_hash = TestMapper.new.mapped_line(['1 test road, testtown'], simple_mapping)
+    assert_equal '1 test road, testtown', line_hash['address']
+    assert_equal '1 test road, testtown', line_hash[:rawtext]['patient address']
+  end
+
+  test 'line mapping should create valid hash with blank cleaned value' do
+    assert_equal '', TestMapper.new.mapped_value('98', clean_opcs_mapping)
+    line_hash = TestMapper.new.mapped_line(['98'], simple_mapping_with_clean_opcs)
+    assert_equal nil, line_hash['primaryprocedures']
+    assert_equal '98', line_hash[:rawtext]['primaryprocedures']
+  end
+
+  test 'line mapping should create valid hash with join' do
+    line_hash = TestMapper.new.mapped_line(%w(Catherine Elizabeth), join_mapping)
+    assert_equal 'Catherine Elizabeth', line_hash['forenames']
+    assert_equal 'Catherine', line_hash[:rawtext]['forename1']
+    assert_equal 'Elizabeth', line_hash[:rawtext]['forename2']
+  end
+
+  test 'line mapping should create valid hash with rawtext only' do
+    line_hash = TestMapper.new.mapped_line(['otherinfo'], unused_mapping)
+    assert_equal 1, line_hash.length
+    assert_equal 'otherinfo', line_hash[:rawtext]['extra']
+  end
+
+  test 'should create valid hash with unused cross populate' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', 'C1234'], cross_populate_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'C1234', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'C1234', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', ''], cross_populate_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'Bob Fossil', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with unused cross populate replace' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', 'C1234'], cross_populate_replace_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'C1234', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'C1234', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate with replace' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', ''], cross_populate_replace_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'ROBERT FOSSIL', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate without replace' do
+    line_hash = TestMapper.new.mapped_line(['Bob Smith', ''], cross_populate_replace_mapping)
+    assert_equal 'Bob Smith', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Smith', line_hash['consultantname']
+    assert_equal 'Bob Smith', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with unused cross populate map' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', 'C1234'], cross_populate_map_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal 'C1234', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'C1234', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate with map' do
+    line_hash = TestMapper.new.mapped_line(['Bob Fossil', ''], cross_populate_map_mapping)
+    assert_equal 'Bob Fossil', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Fossil', line_hash['consultantname']
+    assert_equal 'C5678', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate without map' do
+    line_hash = TestMapper.new.mapped_line(['Bob Smith', ''], cross_populate_map_mapping)
+    assert_equal 'Bob Smith', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Bob Smith', line_hash['consultantname']
+    assert_nil line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate without map and priorities' do
+    line_hash = TestMapper.new.mapped_line(['Pass', '', 'Fail', 'Large Fail'],
+                                           cross_populate_order_mapping)
+    assert_equal 'Pass', line_hash[:rawtext]['referringclinicianname']
+    assert_equal '', line_hash[:rawtext]['referringcliniciancode']
+
+    assert_equal 'Pass', line_hash['consultantname']
+    assert_equal 'Pass', line_hash['consultantcode']
+  end
+
+  test 'should create valid hash with used cross populate without priority' do
+    line_hash = TestMapper.new.mapped_line(%w(Exists Not), cross_populate_no_priority)
+    assert_equal 'Exists', line_hash[:rawtext]['columnoneraw']
+    assert_equal 'Not', line_hash[:rawtext]['columntworaw']
+
+    assert_equal 'Exists', line_hash['columnone']
+    assert_equal 'Exists', line_hash['columntwo']
+  end
+
+  test 'should create equal hashes with standard mapping' do
+    line_hash_without = TestMapper.new.mapped_line(
+      ['Smith', 'John F', 'male', '01234567'], standard_mapping_without
+    )
+    line_hash_with = TestMapper.new.mapped_line(
+      ['Smith', 'John F', 'male', '01234567'], standard_mapping_with
+    )
+    assert_equal line_hash_without, line_hash_with
+  end
+
+  test 'should merge standard mapping and normal mapping' do
+    line_hash = TestMapper.new.mapped_line(['Smith'], standard_mapping_merge)
+    assert_equal 'SMITH', line_hash['surname']
+    assert_equal 'Smith', line_hash['surname2']
+  end
+
+  test 'should merge standard mapping in correct order' do
+    line_hash = TestMapper.new.mapped_line(['Smith'], standard_mapping_column)
+    assert_equal 'Smith', line_hash[:rawtext]['overriding_column_name']
+    refute line_hash[:rawtext].include?('standard_mapping_column_name')
+  end
+
+  test 'should raise duplicate priority exception' do
+    assert_raise(RuntimeError) do
+      TestMapper.new.mapped_line(%w(A B), invalid_priorities)
+    end
+  end
+
+  test 'should raise nonexistent standard mapping exception' do
+    assert_raise(RuntimeError) do
+      TestMapper.new.mapped_line(['A'], invalid_standard_mapping)
+    end
+  end
+
+  test 'should join blank first field with compacting' do
+    line_hash = TestMapper.new.mapped_line(['', 'CB3 0DS'], joined_mapping_blank_start)
+    assert_equal 'CB3 0DS', line_hash['address']
+  end
+
+  test 'should join blank first field without compacting' do
+    line_hash = TestMapper.new.mapped_line(['', 'CB3 0DS'], joined_mapping_blank_start_uncompacted)
+    assert_equal ',CB3 0DS', line_hash['address']
+  end
+
+  test 'line mapping should map date formats correctly' do
+    real_date = Date.new(1927, 7, 6)
+    incomings = %w( 06/07/1927  19270706     07/06/1927   06/07/27  06/JUL/27 )
+    columns   = %w( dateofbirth receiveddate americandate shortdate funkydate )
+    line_hash = TestMapper.new.mapped_line(incomings, date_mapping)
+
+    columns.each do |column_name|
+      assert_equal real_date, line_hash[column_name].to_date
+    end
+  end
+
+  test 'should ignore columns marked do not capture' do
+    line_hash = TestMapper.new.mapped_line(['rubbish'], do_not_capture_column)
+    refute line_hash[:rawtext].include?('ignore_me')
+  end
+
+  test 'should decode base64 encoded word document' do
+    test_file = @permanent_test_files.join('hello_world.doc')
+    encoded_content = Base64.encode64(File.binread(test_file))
+    line_hash = TestMapper.new.mapped_line([encoded_content], base64_mapping)
+    assert_equal 'Hello world, this is a word document', line_hash[:rawtext]['base64']
+  end
+
+  test 'should decode word.doc' do
+    test_file = @permanent_test_files.join('hello_world.doc')
+    file_content = File.binread(test_file)
+    text_content = TestMapper.new.send(:decode_raw_value, file_content, :word_doc)
+    assert_equal 'Hello world, this is a word document', text_content
+  end
+
+  test 'should read word.doc stream' do
+    test_file = @permanent_test_files.join('hello_world.doc')
+    file_content = TestMapper.new.send(:read_word_stream, File.open(test_file, 'r'))
+    assert_equal 'Hello world, this is a word document', file_content
+  end
+
+  test 'should raise unknown encoding exception' do
+    assert_raise(RuntimeError) do
+      TestMapper.new.mapped_line(['A'], invalid_decode_mapping)
+    end
+  end
+end