ndr_import 3.0.0

data/lib/ndr_import/standard_mappings.rb

@@ -0,0 +1,23 @@
+module NdrImport
+  # NdrImport::StandardMappings stores the standard mappings hash
+  class StandardMappings
+    # mappings are stored as a class level instance variable
+    class << self
+      # Gets the standard mappings
+      def mappings
+        if defined?(@standard_mappings)
+          @standard_mappings
+        else
+          fail 'NdrImport::StandardMappings not configured!'
+        end
+      end
+
+      # Sets the standard mappings
+      def mappings=(hash)
+        fail ArgumentError unless hash.is_a?(Hash)
+
+        @standard_mappings = hash
+      end
+    end
+  end
+end

data/lib/ndr_import/table.rb

@@ -0,0 +1,179 @@
+require 'ndr_import/mapper'
+require 'active_support/core_ext/hash'
+
+module NdrImport
+  # This class maintains the state of a table mapping and encapsulates the logic
+  # required to transform a table of data into "records". Particular attention
+  # has been made to use enumerables throughout to help with the transformation
+  # of large quantities of data.
+  class Table
+    include NdrImport::Mapper
+
+    def self.all_valid_options
+      %w(canonical_name filename_pattern tablename_pattern header_lines footer_lines format klass columns)
+    end
+
+    def all_valid_options
+      self.class.all_valid_options
+    end
+
+    attr_reader(*all_valid_options)
+    attr_writer :notifier
+
+    def initialize(options = {})
+      options.stringify_keys! if options.is_a?(Hash)
+      validate_options(options)
+
+      all_valid_options.each do |key|
+        # This pattern is used to only set attributes if option specified,
+        # which makes for more concise YAML serialization.
+        options[key] && instance_variable_set("@#{key}", options[key])
+      end
+
+      @row_index = 0
+    end
+
+    def match(filename, tablename)
+      ::File.basename(filename) =~ (filename_pattern || /\A.*\z/) &&
+        (tablename.nil? || tablename =~ (tablename_pattern || /\A.*\z/))
+    end
+
+    # This method transforms a table of data, given a line array/enumerator and yields
+    # klass, fields and index (input row number) for each record that it would create
+    # as a result of the transformation process.
+    def transform(lines, &block)
+      return enum_for(:transform, lines) unless block
+
+      @row_index = 0
+      @header_valid = false
+      @notifier.try(:started)
+
+      skip_footer_lines(lines, footer_lines).each do |line|
+        process_line(line, &block)
+      end
+
+      @notifier.try(:finished)
+    end
+
+    # This method process a line of data, If it is a header line it validates it, otherwise
+    # transforms it. It also increments and row index and notifies the amount of lines processed.
+    def process_line(line, &block)
+      return enum_for(:process_line, line) unless block
+
+      if @row_index < header_lines
+        validate_header(line, @columns)
+      else
+        fail 'Header is not valid' if header_lines > 0 && !header_valid?
+        transform_line(line, @row_index, &block)
+      end
+
+      @row_index += 1
+      @notifier.try(:processed, @row_index)
+    end
+
+    # This method transforms an incoming line of data by applying each of the klass masked
+    # mappings to the line and yielding the klass and fields for each mapped klass.
+    def transform_line(line, index)
+      return enum_for(:transform_line, line, index) unless block_given?
+
+      masked_mappings.each do |klass, klass_mappings|
+        fields = mapped_line(line, klass_mappings)
+        next if fields[:skip].to_s == 'true'
+        yield(klass, fields, index)
+      end
+    end
+
+    def header_valid?
+      @header_valid == true
+    end
+
+    private
+
+    # This method uses a buffer to not yield the last <buffer_size> iterations of an enumerable.
+    # We use it to skip footer lines (without having to convert the enumerable to an array).
+    def skip_footer_lines(lines, buffer_size)
+      return enum_for(:skip_footer_lines, lines, buffer_size) unless block_given?
+
+      buffer = []
+      lines.each do |line|
+        buffer.unshift(line)
+
+        yield buffer.pop if buffer.length > buffer_size
+      end
+    end
+
+    # This method memoizes the klass masked mappings. Where a table level
+    # klass is defined it is used with the whole mapping, otherwise the masks are generated.
+    def masked_mappings
+      @masked_mappings ||= begin
+        if @klass
+          { @klass => @columns }
+        else
+          column_level_klass_masked_mappings
+        end
+      end
+    end
+
+    # This method generates a hash of klass based mappings, one for each defined klass
+    # where the whole line mapping is masked to just the data items of that klass.
+    def column_level_klass_masked_mappings
+      ensure_mappings_define_klass
+
+      # Loop through each klass
+      masked_mappings = {}
+      @columns.map { |mapping| mapping['klass'] }.flatten.compact.uniq.each do |klass|
+        # Duplicate the column mappings and do not capture fields that relate to other klasses
+        masked_mappings[klass] = mask_mappings_by_klass(klass)
+      end
+      masked_mappings
+    end
+
+    # This method ensures that every column mapping defines a klass (unless it is a column that
+    # we do not capture). It is only used where a table level klass is not defined.
+    def ensure_mappings_define_klass
+      klassless_mappings = @columns.
+                           select { |mapping| mapping.nil? || mapping['klass'].nil? }.
+                           reject { |mapping| mapping['do_not_capture'] }.
+                           map { |mapping| mapping['column'] || mapping['standard_mapping'] }
+
+      return if klassless_mappings.empty?
+
+      # All column mappings for the single item file require a klass definition.
+      fail "Missing klass for column(s): #{klassless_mappings.to_sentence}"
+    end
+
+    # This method duplicates the mappings and applies a do_not_capture mask to those that do not
+    # relate to this klass, returning the masked mappings
+    def mask_mappings_by_klass(klass)
+      @columns.dup.map do |mapping|
+        if Array(mapping['klass']).include?(klass)
+          mapping
+        else
+          { 'do_not_capture' => true }
+        end
+      end
+    end
+
+    def validate_options(hash)
+      fail ArgumentError unless hash.is_a?(Hash)
+
+      unrecognised_options = hash.keys - all_valid_options
+      return if unrecognised_options.empty?
+      fail ArgumentError, "Unrecognised options: #{unrecognised_options.inspect}"
+    end
+
+    # if there is a header, then check the column headings are as expected in the correct order
+    def validate_header(line, column_mappings)
+      columns = column_names(column_mappings)
+      fail 'Number of columns does not match' if line.length != columns.length
+
+      return unless line.map(&:downcase) == columns
+      @header_valid = true
+    end
+
+    # returns the column names as we expect to receive them
+    def column_names(column_mappings)
+      column_mappings.map { |c| (c['column'] || c['standard_mapping']).downcase }
+    end
+  end # class Table
+end

data/lib/ndr_import/version.rb

@@ -0,0 +1,4 @@
+# This stores the current version of the NdrImport gem
+module NdrImport
+  VERSION = '3.0.0'
+end

data/ndr_import.gemspec

@@ -0,0 +1,44 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ndr_import/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ndr_import'
+  spec.version       = NdrImport::VERSION
+  spec.authors       = ['NCRS Development Team']
+  spec.email         = []
+  spec.summary       = 'NDR Import'
+  spec.description   = 'NDR ETL Importer'
+  spec.homepage      = 'https://github.com/PublicHealthEngland/ndr_import'
+  spec.license       = 'MIT'
+
+  # Exclude older versions of this gem from the package.
+  spec.files         = `git ls-files -z`.split("\x0").reject { |s| s =~ %r{^pkg/} }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.0.0'
+  spec.add_dependency 'ndr_support', '~> 3.0'
+
+  spec.add_dependency 'rubyzip', '~> 1.1'
+  spec.add_dependency 'roo', '~> 2.0'
+  # roo requires nokogiri >=1.5, but nokogiri (1.6.1) requires Ruby version >= 1.9.2.
+  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency 'roo-xls'
+  spec.add_dependency 'spreadsheet', '1.0.3'           # Aligning with encore
+  spec.add_dependency 'pdf-reader', '1.2.0'
+  spec.add_dependency 'msworddoc-extractor', '0.2.0'
+
+  spec.required_ruby_version = '>= 1.9.3'
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-rubocop'
+  spec.add_development_dependency 'guard-test'
+  spec.add_development_dependency 'terminal-notifier-guard' if RUBY_PLATFORM =~ /darwin/
+  spec.add_development_dependency 'simplecov'
+end

data/test/file/base_test.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+require 'ndr_import/file/registry'
+
+module NdrImport
+  module File
+    # Base file handler tests
+    class BaseTest < ActiveSupport::TestCase
+      # Handles a single table file, but for test purposes,
+      # I could be bothered to implement it fully
+      class SingleTableLazyDeveloper < ::NdrImport::File::Base
+      end
+
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should fail on not implementing rows' do
+        begin
+          Registry.register(SingleTableLazyDeveloper, 'lazy_dev')
+
+          exception = assert_raises(RuntimeError) do
+            file_path = @permanent_test_files.join('normal.csv')
+            handler = SingleTableLazyDeveloper.new(file_path, 'lazy_dev')
+
+            handler.tables.each do |tablename, sheet|
+              assert_nil tablename
+              assert_instance_of Enumerator, sheet
+              sheet.to_a
+            end
+          end
+
+          msg = 'Implement NdrImport::File::BaseTest::SingleTableLazyDeveloper#rows'
+          assert_equal msg, exception.message
+        ensure
+          Registry.unregister('lazy_dev')
+        end
+      end
+
+      test 'should not fail when set up with an readable safepath' do
+        assert Base.new(SafePath.new('test_space_r'), nil)
+        assert Base.new(SafePath.new('test_space_rw'), nil)
+      end
+
+      test 'should fail when set up with an unreadable safepath' do
+        assert_raises(SecurityError) { Base.new(SafePath.new('test_space_w'), nil) }
+      end
+
+      test 'should fail when set up with a non-safepath' do
+        exception = assert_raises(ArgumentError) { Base.new(NdrImport.root, nil) }
+        assert exception.message =~ /file_name should be of type SafePath, but it is String/
+      end
+    end
+  end
+end

data/test/file/delimited_test.rb

@@ -0,0 +1,143 @@
+require 'test_helper'
+require 'ndr_import/file/delimited'
+
+module NdrImport
+  module File
+    # Delimited file handler tests
+    class DelimitedTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read csv correctly' do
+        file_path = @permanent_test_files.join('normal.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          sheet = sheet.to_a
+          assert_equal(('A'..'Z').to_a, sheet[0])
+          assert_equal ['1'] * 26, sheet[1]
+          assert_equal ['2'] * 26, sheet[2]
+        end
+      end
+
+      test 'should read pipe correctly' do
+        file_path = @permanent_test_files.join('normal_pipe.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'pipe', 'col_sep' => nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          sheet = sheet.to_a
+          assert_equal(('A'..'Z').to_a, sheet[0])
+          assert_equal ['1'] * 26, sheet[1]
+          assert_equal ['2'] * 26, sheet[2]
+        end
+      end
+
+      test 'should read thorn correctly' do
+        file_path = @permanent_test_files.join('normal_thorn.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'thorn', 'col_sep' => nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          sheet = sheet.to_a
+          assert_equal(('A'..'Z').to_a, sheet[0])
+          assert_equal ['1'] * 26, sheet[1]
+          assert_equal ['2'] * 26, sheet[2]
+        end
+      end
+
+      test 'should read csv with a BOM' do
+        file_path = @permanent_test_files.join('bomd.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet = sheet.to_a
+          assert_equal(('A'..'Z').to_a, sheet[0])
+          assert_equal ['1'] * 26, sheet[1]
+          assert_equal ['2'] * 26, sheet[2]
+        end
+      end
+
+      test 'should read windows-1252 csv' do
+        file_path = @permanent_test_files.join('windows.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet = sheet.to_a
+          assert_equal 1, sheet.length
+        end
+      end
+
+      test 'should read acsii-delimited csv' do
+        file_path = @permanent_test_files.join('high_ascii_delimited.txt')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => "\xfe")
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet = sheet.to_a
+          assert_equal 2, sheet.length
+          assert_equal '1234567890', sheet[0][1]
+          assert_equal '1234567890', sheet[1][1]
+        end
+      end
+
+      test 'should read line-by-line' do
+        rows = []
+        file_path = @permanent_test_files.join('normal.csv')
+        handler = NdrImport::File::Delimited.new(file_path, 'csv')
+
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          sheet.each do |row|
+            rows << row
+          end
+        end
+
+        assert_equal(('A'..'Z').to_a, rows[0])
+        assert_equal ['1'] * 26, rows[1]
+        assert_equal ['2'] * 26, rows[2]
+      end
+
+      test 'should report addition details upon failure to slurp csv' do
+        exception = assert_raises(CSVLibrary::MalformedCSVError) do
+          file_path = @permanent_test_files.join('broken.csv')
+          handler = NdrImport::File::Delimited.new(file_path, 'csv', 'col_sep' => nil)
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+
+        msg = 'Invalid CSV format on row 2 of broken.csv. ' \
+              'Original: Missing or stray quote in line 2'
+        assert_equal msg, exception.message
+      end
+
+      test 'should report addition details upon failure to read csv line-by-line' do
+        rows_yielded = []
+        exception    = assert_raises(CSVLibrary::MalformedCSVError) do
+          file_path = @permanent_test_files.join('broken.csv')
+          handler = NdrImport::File::Delimited.new(file_path, 'csv')
+
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.each do |row|
+              rows_yielded << row
+            end
+          end
+        end
+
+        assert rows_yielded.empty?, 'no rows should have been yielded'
+
+        msg = 'Invalid CSV format on row 2 of broken.csv. ' \
+              'Original: Missing or stray quote in line 2'
+        assert_equal msg, exception.message
+      end
+    end
+  end
+end