ndr_import 3.0.0

data/lib/ndr_import/non_tabular/mapping.rb

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+require 'ndr_import/non_tabular/table'
+
+module NdrImport
+  module NonTabular
+    # This class stores the mapping used to break an incoming file into multiple rows/records
+    class Mapping < Table
+      def self.all_valid_options
+        super + %w(non_tabular_row)
+      end
+
+      def initialize(options)
+        non_tabular_mappings = options['non_tabular_row']
+        if non_tabular_mappings
+          initialize_non_tabular_mappings(non_tabular_mappings)
+        else
+          # validate presence of non_tabular_row
+          fail NdrImport::MappingError,
+               I18n.t('mapping.errors.missing_non_tabular_row')
+        end
+
+        super(options)
+      end
+
+      private
+
+      def initialize_non_tabular_mappings(non_tabular_mappings)
+        NON_TABULAR_OPTIONS.each do |key|
+          next unless non_tabular_mappings[key]
+          instance_variable_set("@#{key}", non_tabular_mappings[key])
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/non_tabular/record.rb

@@ -0,0 +1,99 @@
+# encoding: UTF-8
+
+module NdrImport
+  module NonTabular
+    # This class behaves like an array of NdrImport::NonTabular::Line elements
+    # that contains all the source lines of text that relate to a single record of data.
+    # It also encapsulates the logic that tabulates the data.
+    class Record
+      attr_reader :lines
+
+      def initialize
+        @lines = []
+      end
+
+      def <<(line)
+        return if line.removed
+        line.in_a_record = true
+        line.record_line_number = @lines.length
+        @lines << line
+      end
+
+      def empty?
+        @lines.empty?
+      end
+
+      # Call this if it turns out that this is not a record.
+      # All lines will be flagged accordingly.
+      def not_a_record!
+        @lines.each { |line| line.in_a_record = false }
+      end
+
+      # Returns an array of "cells" for a given array of lines of a file that represent
+      # a single "row" of data. Allowing the output to be mapped by the standard mapper.
+      #
+      # ==== Signature
+      #
+      #   tabulate(mappings)
+      #
+      # ==== Examples
+      #
+      #   If the YAML mapping is
+      #   ---
+      #   - standard_mapping: nhsnumber
+      #     non_tabular_cell:
+      #       lines: 0
+      #       capture:
+      #       - !ruby/regexp /^D\|([^|]*).*/
+      #   - column: fulltextreport
+      #     non_tabular_cell:
+      #       lines: !ruby/range
+      #         begin: 1
+      #         end: -1
+      #         excl: false
+      #       capture: !ruby/regexp /^(?:R|\d+)\|(.*)$/i
+      #       join: \n
+      #
+      #   lines = [
+      #     "D|1111111111|...",
+      #     "R|This is a",
+      #     "1|multiline report"
+      #   ]
+      #
+      #   tabulate(mappings)
+      #
+      #   # =>
+      #   [
+      #     "1111111111",
+      #     "This is a\nmultiline report"
+      #   ]
+      #
+      def tabulate(mappings)
+        cells = []
+        mappings.each do |column_mapping|
+          begin
+            matches = get_matches(column_mapping)
+            # Join the non-blank lines together and add to the array of cells
+            cells << matches.select { |value| !value.blank? }.join(column_mapping.join || '')
+          rescue RegexpRange::PatternMatchError
+            cells << nil
+          end
+        end
+        cells
+      end
+
+      # returns an array of matches from within the captured lines
+      def get_matches(column_mapping)
+        matching_lines = column_mapping.matching_lines(@lines)
+        # loop through the specified line (or lines)
+        matches = Array(@lines[matching_lines]).map do |line|
+          line.captured_for(column_mapping.name)
+          value = column_mapping.capture_value(line)
+          line.matches_for(column_mapping.name, value)
+          value
+        end
+        matches
+      end
+    end
+  end
+end

data/lib/ndr_import/non_tabular/table.rb

@@ -0,0 +1,193 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module NonTabular
+    # This class maintains the state of a non tabular table mapping and encapsulates
+    # the logic required to transform a table of data into "records". Particular
+    # attention has been made to use enumerables throughout to help with the
+    # transformation of large quantities of data.
+    class Table < ::NdrImport::Table
+      require 'i18n'
+      require 'ndr_support/regexp_range' # TODO: unneeded?
+      require 'ndr_support/utf8_encoding'
+      require 'ndr_import/non_tabular/column_mapping'
+      require 'ndr_import/non_tabular/record'
+      require 'ndr_import/non_tabular/line'
+
+      include UTF8Encoding
+
+      NON_TABULAR_OPTIONS = %w(capture_start_line start_line_pattern end_line_pattern remove_lines
+                               start_in_a_record end_in_a_record)
+
+      def self.all_valid_options
+        super - %w(tablename_pattern header_lines footer_lines) + NON_TABULAR_OPTIONS
+      end
+
+      attr_reader(*NON_TABULAR_OPTIONS)
+      attr_reader :non_tabular_lines
+
+      def header_lines
+        0
+      end
+
+      def footer_lines
+        0
+      end
+
+      def initialize(options = {})
+        super(options)
+
+        validate_presence_of_start_line_pattern
+      end
+
+      def tablename_pattern=(_value)
+        fail NdrImport::MappingError, 'Should not define tablename_pattern'
+      end
+
+      def validate_presence_of_start_line_pattern
+        return if @start_line_pattern
+        fail NdrImport::MappingError,
+             I18n.t('mapping.errors.missing_start_line_pattern')
+      end
+
+      # This method transforms a table of data, given a line array/enumerator and yields
+      # klass, fields and index (input row number) for each record that it would create
+      # as a result of the transformation process.
+      def transform(lines, &block)
+        return enum_for(:transform, lines) unless block
+
+        self.non_tabular_lines = ensure_utf8_enum!(lines)
+        remove_unwanted_lines
+
+        super(read_non_tabular_array, &block)
+      end
+
+      def validate_header(_line, _column_mappings)
+        @header_valid = true
+      end
+
+      protected
+
+      def ensure_utf8_enum!(lines)
+        return enum_for(:ensure_utf8_enum!, lines) unless block_given?
+
+        lines.each do |line|
+          # puts 'ensure_utf8_object!'
+          yield ensure_utf8_object!(line)
+        end
+      end
+
+      # This method flages unwanted lines, typically page headers and footers as removed
+      # preventing them from being captured in the non tabular record. Especially useful
+      # when there page headers and footers that are out of step with the start and end
+      # of each record and could therefore appear anywhere in an individual record if kept.
+      def remove_unwanted_lines
+        return unless @remove_lines.is_a?(Hash)
+        @non_tabular_lines.each_with_index do |_line, i|
+          @remove_lines.each do |_key, lines_to_remove|
+            comparable_lines = @non_tabular_lines[i, lines_to_remove.length]
+            next unless lines_equal(comparable_lines, lines_to_remove)
+            # All lines are equal, so flag them as removed
+            comparable_lines.each { |line| line.removed = true }
+          end
+        end
+      end
+
+      def read_non_tabular_array
+        @tabular_array = []
+        @in_a_record = @start_in_a_record
+        @non_tabular_record = NdrImport::NonTabular::Record.new
+
+        partition_and_process_non_tabular_lines
+        process_end_of_record
+
+        @tabular_array
+      end
+
+      # Reads the array of lines, looking to see if a line matches the start_line_pattern,
+      # identifying the start of a record. It then collects all the lines until a line
+      # matches the end_line_pattern (if defined, otherwise when it matches the next
+      # start_line_pattern) and sends these line to NdrImport::NonTabular::Record#tabulate.
+      #
+      # NOTE: Currently the end line is consumed and does not form part of the
+      # collected array.
+      def partition_and_process_non_tabular_lines
+        non_tabular_lines.each do |line|
+          if line =~ @start_line_pattern
+            # This is a start line
+            start_record(line)
+          elsif line =~ @end_line_pattern
+            # This is an end line
+            end_record
+          else
+            @non_tabular_record << line if @in_a_record
+          end
+        end
+      end
+
+      # Checks to see if we get the start of a new record before getting the end of the previous
+      # one and fails if so. Otherwise it tabulates the previous record
+      def start_record(line)
+        if @end_line_pattern
+          fail NdrImport::MappingError,
+               I18n.t('mapping.errors.start_pattern_before_end') if @in_a_record
+        else
+          # No endline mapping
+          @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+        end
+        @non_tabular_record = NdrImport::NonTabular::Record.new
+        @non_tabular_record << line if @capture_start_line
+        @in_a_record = true
+      end
+
+      # Tabulate the record (if in one), flagged it as no longer being in a record
+      # and set the record to be a new one.
+      def end_record
+        @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+        @in_a_record = false
+        @non_tabular_record = NdrImport::NonTabular::Record.new
+      end
+
+      # If the non-tabular data ends in a record (i.e. the last record is terminated by the EOF)
+      # then we need to process the last record manually or flag those lines as not being part
+      # of a record
+      def process_end_of_record
+        return if @non_tabular_record.empty?
+        if @end_in_a_record
+          @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+        else
+          @non_tabular_record.not_a_record!
+        end
+      end
+
+      # Store the source lines as instances of NdrImport::NonTabular::Line
+      def non_tabular_lines=(lines)
+        @non_tabular_lines = lines.map.with_index do |line, i|
+          NdrImport::NonTabular::Line.new(line, i)
+        end
+      end
+
+      # Create and memoize the column mappings
+      def column_mappings
+        @column_mappings ||= raw_column_mappings.map do |column_mapping|
+          NdrImport::NonTabular::ColumnMapping.new(column_mapping)
+        end
+      end
+
+      def raw_column_mappings
+        @columns || []
+      end
+
+      # This method compares two arrays, where the first must be an array of
+      # NdrImport::NonTabular::Line or string elements
+      # and the second can be a mix of strings and/or regular expressions
+      def lines_equal(lines, other_lines)
+        return false unless lines.length == other_lines.length
+        lines.each_with_index.map do |line, i|
+          other_line = other_lines[i]
+          other_line.is_a?(Regexp) ? line.to_s =~ other_line : line.to_s == other_line
+        end.all?
+      end
+    end
+  end
+end

data/lib/ndr_import/non_tabular_file_helper.rb

@@ -0,0 +1,160 @@
+# encoding: UTF-8
+
+module NdrImport
+  # This mixin adds (multiline) non-tabular file functionality to unified importers.
+  # It provides a file reader method and method to capture the rawtext value
+  # appropriately. These methods can be overridden or aliased as required.
+  #
+  # The YAML mapping must define the start_line_pattern which identifies the start
+  # of a multiline record (or "row") and can optionally define an end_line_pattern.
+  module NonTabularFileHelper
+    require 'i18n'
+    require 'ndr_support/regexp_range' # TODO: unneeded?
+    require 'ndr_support/utf8_encoding'
+    require 'ndr_import/non_tabular/column_mapping'
+    require 'ndr_import/non_tabular/record'
+    require 'ndr_import/non_tabular/line'
+    require 'ndr_import/non_tabular/mapping'
+
+    include UTF8Encoding
+
+    attr_reader :non_tabular_lines
+
+    protected
+
+    # Reads a non-tabular text file and returns an array of tabulated rows of data,
+    # where each row is an array of cells.
+    def read_non_tabular_file
+      self.non_tabular_lines = ensure_utf8_object! SafeFile.readlines(filename)
+      remove_unwanted_lines
+      read_non_tabular_array
+    end
+
+    # Reads a string and returns an array of tabulated data. Use only for prototyping.
+    def read_non_tabular_string(text)
+      self.non_tabular_lines = ensure_utf8_object!(text).split("\n")
+      remove_unwanted_lines
+      read_non_tabular_array
+    end
+
+    # This method flages unwanted lines, typically page headers and footers as removed
+    # preventing them from being captured in the non tabular record. Especially useful
+    # when there page headers and footers that are out of step with the start and end
+    # of each record and could therefore appear anywhere in an individual record if kept.
+    def remove_unwanted_lines
+      return unless row_mapping.remove_lines.is_a?(Hash)
+      @non_tabular_lines.each_with_index do |_line, i|
+        row_mapping.remove_lines.each do |_key, lines_to_remove|
+          comparable_lines = @non_tabular_lines[i, lines_to_remove.length]
+          next unless lines_equal(comparable_lines, lines_to_remove)
+          # All lines are equal, so flag them as removed
+          comparable_lines.each { |line| line.removed = true }
+        end
+      end
+    end
+
+    def read_non_tabular_array
+      @tabular_array = []
+      @in_a_record = row_mapping.start_in_a_record
+      @non_tabular_record = NdrImport::NonTabular::Record.new
+
+      partition_and_process_non_tabular_lines
+      process_end_of_record
+
+      # We change the mapping instance variable to only contain the column mappings.
+      # This enables the standard mapper to work unaltered.
+      @mappings = raw_column_mappings
+      @tabular_array
+    end
+
+    # Reads the array of lines, looking to see if a line matches the start_line_pattern,
+    # identifying the start of a record. It then collects all the lines until a line
+    # matches the end_line_pattern (if defined, otherwise when it matches the next
+    # start_line_pattern) and sends these line to NdrImport::NonTabular::Record#tabulate.
+    #
+    # NOTE: Currently the end line is consumed and does not form part of the
+    # collected array.
+    def partition_and_process_non_tabular_lines
+      non_tabular_lines.each do |line|
+        if line =~ row_mapping.start_line_pattern
+          # This is a start line
+          start_record(line)
+        elsif line =~ row_mapping.end_line_pattern
+          # This is an end line
+          end_record
+        else
+          @non_tabular_record << line if @in_a_record
+        end
+      end
+    end
+
+    # Checks to see if we get the start of a new record before getting the end of the previous
+    # one and fails if so. Otherwise it tabulates the previous record
+    def start_record(line)
+      if row_mapping.end_line_pattern
+        fail NdrImport::MappingError,
+             I18n.t('mapping.errors.start_pattern_before_end') if @in_a_record
+      else
+        # No endline mapping
+        @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+      end
+      @non_tabular_record = NdrImport::NonTabular::Record.new
+      @non_tabular_record << line if row_mapping.capture_start_line
+      @in_a_record = true
+    end
+
+    # Tabulate the record (if in one), flagged it as no longer being in a record
+    # and set the record to be a new one.
+    def end_record
+      @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+      @in_a_record = false
+      @non_tabular_record = NdrImport::NonTabular::Record.new
+    end
+
+    # If the non-tabular data ends in a record (i.e. the last record is terminated by the EOF)
+    # then we need to process the last record manually or flag those lines as not being part
+    # of a record
+    def process_end_of_record
+      return if @non_tabular_record.empty?
+      if row_mapping.end_in_a_record
+        @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
+      else
+        @non_tabular_record.not_a_record!
+      end
+    end
+
+    # Store the source lines as instances of NdrImport::NonTabular::Line
+    def non_tabular_lines=(lines)
+      @non_tabular_lines = lines.map.with_index do |line, i|
+        NdrImport::NonTabular::Line.new(line, i)
+      end
+    end
+
+    # Create and memoize the row mappings
+    def row_mapping
+      @row_mapping ||= NdrImport::NonTabular::Mapping.new(@mappings)
+    end
+
+    # Create and memoize the column mappings
+    def column_mappings
+      @column_mappings ||= raw_column_mappings.map do |column_mapping|
+        NdrImport::NonTabular::ColumnMapping.new(column_mapping)
+      end
+    end
+
+    def raw_column_mappings
+      @mappings['columns'] || []
+    end
+
+    # This method compares two arrays, where the first must be an array of
+    # NdrImport::NonTabular::Line or string elements
+    # and the second can be a mix of strings and/or regular expressions
+    def lines_equal(lines, other_lines)
+      return false unless lines.length == other_lines.length
+      lines.each_with_index.map do |line, i|
+        other_line = other_lines[i]
+        other_line.is_a?(Regexp) ? line.to_s =~ other_line : line.to_s == other_line
+      end.all?
+    end
+  end
+end