ndr_import 3.0.0

data/lib/ndr_import/helpers/file/word.rb

@@ -0,0 +1,27 @@
+require 'ndr_support/safe_file'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds Word document functionality to unified importers.
+      # It provides a file reader method.
+      # currently only works on .doc (97-2003), not.docx
+      module Word
+        private
+
+        def read_word_file(path)
+          require 'msworddoc-extractor'
+          lines = []
+          begin
+            doc = MSWordDoc::Extractor.load(SafeFile.safepath_to_string(path))
+
+            lines.concat doc.whole_contents.split("\n")
+          rescue => e
+            raise("#{SafeFile.basename(path)} [#{e.class}: #{e.message}]")
+          end
+          lines
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/helpers/file/xml.rb

@@ -0,0 +1,45 @@
+require 'ndr_support/safe_file'
+require 'ndr_support/utf8_encoding'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds XML functionality to unified importers.
+      module Xml
+        include UTF8Encoding
+
+        private
+
+        def read_xml_file(path)
+          file_data = SafeFile.new(path).read
+
+          require 'nokogiri'
+
+          Nokogiri::XML(ensure_utf8! file_data).tap do |doc|
+            doc.encoding = 'UTF-8'
+            emulate_strict_mode_fatal_check!(doc)
+          end
+        end
+
+        # Nokogiri can use give a `STRICT` parse option to libxml, but our friendly
+        # handling of muddled encodings causes XML explicitly declared as something
+        # other than UTF-8 to fail (because it has been recoded to UTF-8 by the
+        # time it is given to Nokogiri / libxml).
+        # This raises a SyntaxError if strict mode would have found any other
+        # (fatal) issues with the document.
+        def emulate_strict_mode_fatal_check!(document)
+          # We let slide any warnings about xml declared as one of our
+          # auto encodings, but parsed as UTF-8:
+          encoding_pattern = AUTO_ENCODINGS.map { |name| Regexp.escape(name) }.join('|')
+          encoding_warning = /\ADocument labelled (#{encoding_pattern}) but has UTF-8 content\z/
+          fatal_errors     = document.errors.select do |error|
+            error.fatal? && (encoding_warning !~ error.message)
+          end
+
+          return unless fatal_errors.any?
+          fail Nokogiri::XML::SyntaxError, "The file had #{fatal_errors.length} fatal error(s)!"
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/helpers/file/zip.rb

@@ -0,0 +1,44 @@
+require 'ndr_support/safe_file'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds Zip functionality to unified importers.
+      module Zip
+        private
+
+        # Unzip the file, creating the destination directory if necessary.
+        # A pattern can be provided to only extract required files.
+        def unzip_file(source, destination, pattern = //)
+          # SECURE TVB Mon Aug 13 14:41:05 BST 2012 : SafePath will raise exception if insecure
+          # path is constructed
+          # SafeFile.safepath_to_string will make sure that the arguments are from type SafePath
+
+          # SECURE: BNS 2010-09-21 (for external access)
+          fail 'Not allowed in external environment' if defined?(::Rails) && ::Rails.env.external?
+
+          require 'zip'
+          # TODO: Abort if destination directory already exists...
+          FileUtils.mkdir_p(SafeFile.safepath_to_string(destination))
+
+          ::Zip::File.open(SafeFile.safepath_to_string(source)) do |zipfile|
+            zipfile.entries.each do |entry|
+              # SECURE: TPG 2010-11-1: The path is stripped from the zipfile entry when extracted
+              basename = ::File.basename(entry.name)
+              zipfile.extract(entry, destination.join(basename)) if entry.file? && basename.match(pattern)
+            end
+          end
+
+        rescue ::Zip::ZipDestinationFileExistsError
+          # I'm going to ignore this and just overwrite the files.
+        rescue SecurityError => ex
+          raise ex
+        rescue ArgumentError => ex
+          raise ex
+        rescue => ex
+          puts ex
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/mapper.rb

@@ -0,0 +1,220 @@
+require 'ndr_support/string/cleaning'
+require 'ndr_support/string/conversions'
+require 'ndr_import/standard_mappings'
+require 'base64'
+require 'msworddoc-extractor'
+
+# This module provides helper logic for mapping unified sources for import into the system
+module NdrImport::Mapper
+  private
+
+  # uses the mappings for this line to unpack the fixed width string
+  # returning an array of the resulting columns
+  def fixed_width_columns(line, line_mappings)
+    unpack_patterns = line_mappings.map { |c| c['unpack_pattern'] }.join
+    line.unpack(unpack_patterns)
+  end
+
+  # the replace option can be used before any other mapping option
+  def replace_before_mapping(original_value, field_mapping)
+    return unless field_mapping.include?('replace') && original_value
+
+    [field_mapping['replace']].flatten.each do |field_replacement|
+      field_replacement.each do |pattern, replacement|
+        original_value.gsub!(pattern, replacement)
+      end
+    end
+  end
+
+  # Returns the standard_mapping hash specified
+  # Assumes mappping exists
+  def standard_mapping(mapping_name, column_mapping)
+    mapping = NdrImport::StandardMappings.mappings[mapping_name]
+    return nil if mapping.nil?
+    if column_mapping['mappings']
+      mapping['mappings'] = mapping['mappings'] + column_mapping.delete('mappings')
+    end
+    mapping.merge(column_mapping)
+  end
+
+  # This takes an array of raw values and their associated mappings and returns an attribute hash
+  # It accepts a block to alter the raw value that is stored in the raw text (if necessary),
+  # enabling it to work for different sources
+  def mapped_line(line, line_mappings)
+    attributes = {}
+    rawtext = {}
+    validate_line_mappings(line_mappings)
+
+    line.each_with_index do |raw_value, col|
+      column_mapping = line_mappings[col]
+      if column_mapping.nil?
+        fail ArgumentError,
+             "Line has too many columns (expected #{line_mappings.size} but got #{line.size})"
+      end
+
+      next if column_mapping['do_not_capture']
+
+      if column_mapping['standard_mapping']
+        column_mapping = standard_mapping(column_mapping['standard_mapping'], column_mapping)
+      end
+      field_mappings = column_mapping['mappings'] || []
+
+      # Establish the rawtext column name we are to use for this column
+      rawtext_column_name = (column_mapping['rawtext_name'] || column_mapping['column']).downcase
+
+      # Replace raw_value with decoded raw_value
+      Array(column_mapping['decode']).each do |encoding|
+        raw_value = decode_raw_value(raw_value, encoding)
+      end
+
+      # raw value casting can vary between sources, so we allow the caller to apply it here
+      if respond_to?(:cast_raw_value)
+        raw_value = cast_raw_value(rawtext_column_name, raw_value, column_mapping)
+      end
+
+      # Store the raw column value
+      rawtext[rawtext_column_name] = raw_value
+
+      field_mappings.each do |field_mapping|
+        # create a duplicate of the raw value we can manipulate
+        original_value = raw_value ? raw_value.dup : nil
+
+        replace_before_mapping(original_value, field_mapping)
+        value = mapped_value(original_value, field_mapping)
+
+        field = field_mapping['field']
+
+        # Assumes join is specified in first joined field
+        joined = field_mapping['join'] ? true : false
+
+        # Currently assuming already validated YAML, s.t. no fields have the
+        # same priorities
+        #
+        # This has become really messy...
+        unless value.blank? && !joined
+          attributes[field] = {} unless attributes[field]
+          attributes[field][:priority] = {} unless attributes[field][:priority]
+          if field_mapping['order']
+            attributes[field][field_mapping['order']] = value
+            attributes[field][:join] = field_mapping['join'] if field_mapping['join']
+            attributes[field][:compact] = field_mapping['compact'] if field_mapping.include?('compact')
+          elsif field_mapping['priority']
+            attributes[field][:priority][field_mapping['priority']] = value
+          else
+            # Check if already a mapped-to field, and assign default low
+            # priority
+            attributes[field][:priority][1] = value
+            attributes[field][:value] = value
+          end
+        end
+      end
+    end
+
+    # tidy up many to one field mappings
+    # and one to many, for cross-populating
+    attributes.each do |field, value|
+      if value.include?(:join)
+        join_string = value.delete(:join) || ','
+        value.delete(:value)
+        value.delete(:priority)
+        if value.include?(:compact)
+          compact = value.delete(:compact)
+        else
+          compact = true
+        end
+        t = value.sort.map do |_part_order, part_value|
+          part_value.blank? ? nil : part_value
+        end
+        if compact
+          attributes[field] = t.compact.join(join_string)
+        else
+          attributes[field] = t.join(join_string)
+        end
+      else
+        attributes[field][:priority].reject! { |_k, v| v.blank? }
+        attributes[field] = attributes[field][:priority].sort.first[1]
+      end
+    end
+
+    attributes[:rawtext] = rawtext
+    attributes
+  end
+
+  def mapped_value(original_value, field_mapping)
+    if field_mapping.include?('format')
+      begin
+        return original_value.blank? ? nil : original_value.to_date(field_mapping['format'])
+      rescue ArgumentError => e
+        e2 = ArgumentError.new("#{e} value #{original_value.inspect}")
+        e2.set_backtrace(e.backtrace)
+        raise e2
+      end
+    elsif field_mapping.include?('clean')
+      return original_value.blank? ? nil : original_value.clean(field_mapping['clean'])
+    elsif field_mapping.include?('map')
+      return field_mapping['map'] ? field_mapping['map'][original_value] : nil
+    elsif field_mapping.include?('match')
+      # WARNING:TVB Thu Aug  9 17:09:25 BST 2012 field_mapping['match'] regexp
+      # may need to be escaped
+      matches = Regexp.new(field_mapping['match']).match(original_value)
+      return matches[1].strip if matches && matches.size > 0
+    elsif field_mapping.include?('daysafter')
+      return original_value unless original_value.to_i.to_s == original_value.to_s
+      return original_value.to_i.days.since(field_mapping['daysafter'].to_time).to_date
+    else
+      return nil if original_value.blank?
+      return original_value.is_a?(String) ? original_value.strip : original_value
+    end
+  end
+
+  # Check for duplicate priorities, check for nonexistent standard_mappings
+  def validate_line_mappings(line_mappings)
+    priority = {}
+    line_mappings.each do |column_mapping|
+      if column_mapping['standard_mapping']
+        if standard_mapping(column_mapping['standard_mapping'], column_mapping).nil?
+          fail "Standard mapping \"#{column_mapping['standard_mapping']}\" does not exist"
+        end
+      end
+      field_mappings = column_mapping['mappings'] || []
+      field_mappings.each do |field_mapping|
+        field = field_mapping['field']
+        if field_mapping['priority']
+          fail 'Cannot have duplicate priorities' if priority[field] == field_mapping['priority']
+          priority[field] = field_mapping['priority']
+        else
+          priority[field] = 1
+        end
+      end
+    end
+    true
+  end
+
+  # Decode raw_value using specified encoding
+  # E.g. adding decode to a column:
+  #
+  # - column: base64
+  #   decode:
+  #   - :base64
+  #   - :word_doc
+  #
+  # would base64 decode a word document and then 'decode' the word document into plain text
+  def decode_raw_value(raw_value, encoding)
+    case encoding
+    when :base64
+      Base64.decode64(raw_value)
+    when :word_doc
+      read_word_stream(StringIO.new(raw_value, 'r'))
+    else
+      fail "Cannot decode: #{encoding}"
+    end
+  end
+
+  # Given an IO stream representing a .doc word document,
+  # this method will extract the text for the document in the same way
+  # as NdrImport::Helpers::File::Word#read_word_file
+  def read_word_stream(stream)
+    # whole_contents adds "\n" to end of stream, we remove it
+    MSWordDoc::Extractor.load(stream).whole_contents.sub(/\n\z/, '')
+  end
+end

data/lib/ndr_import/mapping_error.rb

@@ -0,0 +1,5 @@
+# Raised if there is a problem with an import mapping.
+module NdrImport
+  class MappingError < StandardError
+  end
+end

data/lib/ndr_import/non_tabular/column_mapping.rb

@@ -0,0 +1,73 @@
+# encoding: UTF-8
+
+module NdrImport
+  module NonTabular
+    # This class stores the mapping for an individual non-tabular column, encapsulating
+    # the logic associated with finding matching lines of source data and subsequently
+    # capturing arrays of values within them.
+    class ColumnMapping
+      attr_accessor :name, :cell_mapping, :lines, :capture, :join
+
+      def initialize(column_mapping)
+        @name         = column_mapping['rawtext_name'] ||
+                        column_mapping['column'] ||
+                        column_mapping['standard_mapping']
+        @cell_mapping = column_mapping['non_tabular_cell']
+
+        validate_cell_mapping
+
+        @lines = @cell_mapping['lines']
+        @join  = @cell_mapping['join']
+      end
+
+      # This method returns the range of matching source data lines. If the range is a
+      # RegexpRange then it will calculate it for the text provided.
+      def matching_lines(text)
+        if @lines.is_a?(RegexpRange)
+          @lines.to_range(text)
+        else
+          @lines
+        end
+      end
+
+      # capture the required part of the line by replacing (recusively) the line,
+      # with the first captured regular expression group. This is hardcoded in an attempt
+      # to preserve the rawtext as much as possible
+      def capture_value(line)
+        value = line.dup
+        [@cell_mapping['capture']].flatten.each do |pattern|
+          if matchdata = value.to_s.match(pattern)
+            value = matchdata[1]
+          else
+            value = nil
+          end
+        end
+        value
+      end
+
+      def validate_cell_mapping
+        validate_presence_of_non_tabular_cell
+        validate_presence_of_non_tabular_cell_lines
+        validate_presence_of_non_tabular_cell_capture
+      end
+
+      def validate_presence_of_non_tabular_cell
+        return if @cell_mapping
+        fail NdrImport::MappingError,
+             I18n.t('mapping.errors.missing_non_tabular_cell', :name => @name)
+      end
+
+      def validate_presence_of_non_tabular_cell_lines
+        return if @cell_mapping['lines']
+        fail NdrImport::MappingError,
+             I18n.t('mapping.errors.missing_non_tabular_cell_lines', :name => @name)
+      end
+
+      def validate_presence_of_non_tabular_cell_capture
+        return if @cell_mapping['capture']
+        fail NdrImport::MappingError,
+             I18n.t('mapping.errors.missing_non_tabular_cell_capture', :name => @name)
+      end
+    end
+  end
+end

data/lib/ndr_import/non_tabular/line.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+module NdrImport
+  module NonTabular
+    # This class behaves like a string and is used instead of the each source line of text.
+    # It allows us to contain additional information relating to the use of the line e.g. is
+    # the line within a record or for which fields the line has been used to capture a value.
+    class Line
+      attr_accessor :absolute_line_number,
+                    :captured_fields,
+                    :captures_values,
+                    :in_a_record,
+                    :record_line_number,
+                    :removed
+
+      def initialize(line, absolute_line_number)
+        @line = line.rstrip
+        @absolute_line_number = absolute_line_number
+        @in_a_record = false
+        @removed = false
+        @captured_fields = []
+        @captures_values = []
+      end
+
+      def =~(other)
+        @line =~ other
+      end
+
+      def match(*args)
+        @line.match(*args)
+      end
+
+      def to_s
+        @line
+      end
+
+      def captured_for(field)
+        @captured_fields << field if field && !@captured_fields.include?(field)
+      end
+
+      def matches_for(field, value)
+        @captures_values << [field, value]
+      end
+    end
+  end
+end