ndr_import 3.0.0

data/lib/ndr_import.rb

@@ -0,0 +1,13 @@
+require 'ndr_import/version'
+require 'ndr_import/csv_library'
+require 'ndr_import/mapping_error'
+require 'ndr_import/mapper'
+require 'ndr_import/non_tabular_file_helper'
+require 'ndr_import/table'
+require 'ndr_import/non_tabular/table'
+
+module NdrImport
+  def self.root
+    ::File.expand_path('../..', __FILE__)
+  end
+end

data/lib/ndr_import/csv_library.rb

@@ -0,0 +1,40 @@
+# This file allows us to choose the CSV library we want to use.
+
+require 'csv'
+# Using relevant core CSV library.
+CSVLibrary = CSV
+
+class << CSVLibrary
+  # Is the library we're using FasterCSV?
+  def fastercsv?
+    not self.const_defined?(:Reader)
+  end
+
+  def write_csv_to_string(data)
+    self.generate do |csv|
+      data.each { |line| csv << line }
+    end
+  end
+
+  def write_csv_to_file(data, filepath, mode = 'w')
+    self.open(filepath, mode) do |csv|
+      data.each { |line| csv << line }
+    end
+  end
+
+  def read_csv_from_file(filepath)
+    self.read(filepath)
+  end
+end
+
+# Forward port CSV::Cell, as it is sometimes
+# serialised in YAML. :-(
+class CSV::Cell < String
+  def initialize(data = '', is_null = false)
+    super(is_null ? '' : data)
+  end
+
+  def data
+    to_s
+  end
+end

data/lib/ndr_import/file/all.rb

@@ -0,0 +1,8 @@
+require_relative 'base'
+require_relative 'delimited'
+require_relative 'excel'
+require_relative 'pdf'
+require_relative 'text'
+require_relative 'word'
+# # require_relative 'xml'
+require_relative 'zip'

data/lib/ndr_import/file/base.rb

@@ -0,0 +1,76 @@
+require 'ndr_support/safe_file'
+require 'ndr_import/csv_library'
+require_relative 'registry'
+
+module NdrImport
+  # This is the base of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # All common base file handler logic is defined here.
+    class Base
+      def initialize(filename, format, options = {})
+        @filename = filename
+        @format = format
+        @options = options
+
+        validate_filename_is_safe_and_readable!
+      end
+
+      # This method iterates over the files in the given file and yields the filenames.
+      # For a zip file it will yield for every file in the zip file and for (currently)
+      # every other file it will yield its own filename.
+      #
+      # As the majority of files are not containers (of other files), the Base implementation
+      # is defined for these handlers. If your file contains more than one file, then
+      # override this method. If you do overide this method, then you will probably want
+      # to raise an exception if your tables method is called. E.g. a zip file handler would
+      # produce files, never tables.
+      def files
+        return enum_for(:files) unless block_given?
+
+        yield @filename
+      end
+
+      # This method iterates over the tables in the given file and yields with two arguments:
+      # a tablename and a row enumerator (for that table). For a spreadsheet it may yield for
+      # every worksheet in the file and for a CSV file it will only yield once (the entire
+      # file is one table).
+      #
+      # As single table files are in the majority, the Base implementation is defined for
+      # single table handlers and you will only need to implement the rows iterator. If your
+      # file contains more than one table, then override this method.
+      #
+      # NOTE: for single table handlers, the tablename argument should be nil.
+      def tables
+        return enum_for(:tables) unless block_given?
+
+        yield nil, rows
+      end
+
+      private
+
+      # If this is a single table file handler then this method must be implemented by
+      # the subclass. It iterates over each of the rows of the current table (the whole
+      # file in this case) and should work in both the block and non-block form, returning
+      # an Enumerator in the latter case. We recommend that you follow the following pattern:
+      #
+      # def rows
+      #   return enum_for(:rows) unless block_given?
+      #
+      #   ... your code goes here ...
+      # end
+      #
+      def rows
+        fail "Implement #{self.class}#rows"
+      end
+
+      def validate_filename_is_safe_and_readable!
+        SafeFile.safepath_to_string(@filename)
+
+        # Ensure that we're allowed to read from the safe path:
+        # (they can be configured to be write-only, for example)
+        SafeFile.verify_mode(@filename, 'r')
+      end
+    end
+  end
+end

data/lib/ndr_import/file/delimited.rb

@@ -0,0 +1,86 @@
+require 'ndr_support/safe_file'
+require 'ndr_import/csv_library'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a delimited file handler that returns a single table.
+    class Delimited < Base
+      DELIMITED_COL_SEP = {
+        'csv' => nil,
+        'pipe' => '|',
+        'thorn' => "\xfe" # high-ascii (lower case thorn) delimited files
+      }
+
+      def initialize(filename, format, options = {})
+        super
+
+        @options['col_sep'] ||= DELIMITED_COL_SEP[format]
+      end
+
+      private
+
+      # Iterate through the file line by line, yielding each one in turn.
+      def rows
+        return enum_for(:rows) unless block_given?
+
+        safe_path = SafeFile.safepath_to_string(@filename)
+        encodings = determine_encodings!(safe_path)
+
+        # By now, we know `encodings` should let us read the whole
+        # file succesfully; if there are problems, we should crash.
+        CSVLibrary.foreach(safe_path, encodings) do |line|
+          yield line.map(&:to_s) unless line.length <= 5
+        end
+      end
+
+      # Derive the source encoding by trying all supported encodings.
+      # Returns first set of working options, or raises if none could be found.
+      def determine_encodings!(safe_path)
+        # delimiter encoding => # FasterCSV encoding string
+        supported_encodings = {
+          'UTF-8'        => 'bom|utf-8',
+          'Windows-1252' => 'windows-1252:utf-8'
+        }
+
+        successful_options = nil
+        supported_encodings.each do |delimiter_encoding, csv_encoding|
+          begin
+            col_sep = @options['col_sep']
+            options = {
+              :col_sep  => (col_sep || ',').force_encoding(delimiter_encoding),
+              :encoding => csv_encoding
+            }
+
+            row_num = 0
+            # Iterate through the file; if we reach the end, this encoding worked:
+            CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
+          rescue ArgumentError => e
+            next if e.message =~ /invalid byte sequence/ # This encoding didn't work
+            raise(e)
+          rescue CSVLibrary::MalformedCSVError => e
+            description = (col_sep ? col_sep.inspect + ' delimited' : 'CSV')
+
+            raise(CSVLibrary::MalformedCSVError, "Invalid #{description} format " \
+              "on row #{row_num + 1} of #{::File.basename(safe_path)}. Original: #{e.message}")
+          end
+
+          # We got this far => encoding choice worked:
+          successful_options = options
+          break
+        end
+
+        # We tried them all, and none worked:
+        unless successful_options
+          fail "None of the encodings #{supported_encodings.values.inspect} were successful!"
+        end
+
+        successful_options
+      end
+    end
+
+    Registry.register(Delimited, 'csv', 'pipe', 'thorn')
+  end
+end

data/lib/ndr_import/file/excel.rb

@@ -0,0 +1,131 @@
+require 'roo'
+require 'roo-xls'
+require 'ole/storage'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is an excel file handler that returns tables (worksheets).
+    # It provides a file reader method and methods to cast raw values
+    # appropriately. These methods can be overridden or aliased as required.
+    #
+    class Excel < Base
+      # Iterate through the file table by table, yielding each one in turn.
+      def tables
+        return enum_for(:tables) unless block_given?
+
+        workbook = load_workbook(@filename)
+        workbook.each_with_pagename do |tablename, sheet|
+          yield tablename, excel_rows(workbook, sheet)
+        end
+      end
+
+      protected
+
+      def cast_excel_value(raw_value)
+        return raw_value if raw_value.nil?
+
+        if raw_value.is_a?(Date) || raw_value.is_a?(DateTime) || raw_value.is_a?(Time)
+          cast_excel_datetime_as_date(raw_value)
+        elsif raw_value.is_a?(Float)
+          if raw_value.to_f == raw_value.to_i
+            # Whole number
+            return raw_value.to_i.to_s
+          else
+            return raw_value.to_f.to_s
+          end
+        else
+          return raw_value.to_s.strip
+        end
+      end
+
+      def cast_excel_datetime_as_date(raw_value)
+        raw_value.to_s(:db)
+      end
+
+      private
+
+      # Iterate through the sheet line by line, yielding each one in turn.
+      def excel_rows(workbook, sheet, &block)
+        return enum_for(:excel_rows, workbook, sheet) unless block
+
+        if workbook.is_a?(Roo::Excelx)
+          # FIXME: xlsx_rows(sheet, &block) should produce the same output as xls_rows
+          xls_rows(sheet, &block)
+        else
+          xls_rows(sheet, &block)
+        end
+      end
+
+      # Iterate through an xls sheet line by line, yielding each one in turn.
+      def xls_rows(sheet)
+        return enum_for(:xls_rows, sheet) unless block_given?
+
+        sheet.first_row.upto(sheet.last_row) do |row|
+          line = []
+          sheet.first_column.upto(sheet.last_column) do |col|
+            line << cast_excel_value(sheet.cell(row, col))
+          end
+          yield line
+        end
+      end
+
+      # Iterate through an xlsx sheet line by line, yielding each one in turn.
+      # This method uses streaming https://github.com/roo-rb/roo#excel-xlsx-and-xlsm-support
+      def xlsx_rows(sheet)
+        return enum_for(:xlsx_rows, sheet) unless block_given?
+
+        sheet.each_row_streaming(:pad_cells => true) do |row|
+          yield row.map { |cell| cast_excel_value(cell.value) }
+        end
+      end
+
+      def load_workbook(path)
+        case SafeFile.extname(path).downcase
+        when '.xls'
+          Roo::Excel.new(SafeFile.safepath_to_string(path))
+        when '.xlsx'
+          Roo::Excelx.new(SafeFile.safepath_to_string(path))
+        else
+          fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
+        end
+      rescue Ole::Storage::FormatError => e
+        # TODO: Do we need to remove the new_file after using it?
+
+        # try to load the .xls file as an .xlsx file, useful for sources like USOM
+        # roo check file extensions in file_type_check (GenericSpreadsheet),
+        # so we create a duplicate file in xlsx extension
+        if /(.*)\.xls$/.match(path)
+          new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
+          new_file_path = SafeFile.dirname(path).join(new_file_name)
+          copy_file(path, new_file_path)
+
+          load_workbook(new_file_path)
+        else
+          raise e.message
+        end
+      rescue => e
+        raise ["Unable to read the file '#{path}'", e.message].join('; ')
+      end
+
+      # Note that this method can produce insecure calls. All callers must protect
+      # their arguments.
+      # Arguments:
+      #   * source - SafeFile
+      #   * dest - SafeFile
+      #
+      def copy_file(source, dest)
+        # SECURE: TVB Mon Aug 13 13:53:02 BST 2012 : Secure SafePath will do the security checks
+        # before it is converted to string.
+        # SafeFile will make sure that the arguments are actually SafePath
+        FileUtils.mkdir_p(SafeFile.safepath_to_string(SafeFile.dirname(dest)))
+        FileUtils.cp(SafeFile.safepath_to_string(source), SafeFile.safepath_to_string(dest))
+      end
+    end
+
+    Registry.register(Excel, 'xls', 'xlsx')
+  end
+end

data/lib/ndr_import/file/pdf.rb

@@ -0,0 +1,38 @@
+require 'pdf-reader'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a PDF file handler that returns a single table.
+    class Pdf < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        reader = PDF::Reader.new(SafeFile.safepath_to_string(@filename))
+
+        reader.pages.each do |page|
+          process_page(page, &block)
+        end
+
+      rescue NoMethodError
+        raise "Failed to read #{SafeFile.basename(@filename)} as a PDF"
+      end
+
+      def process_page(page, &block)
+        page.text.split("\n").each do |line|
+          block.call(line)
+        end
+      rescue => e
+        raise "Invalid format on page #{page.number} of #{SafeFile.basename(@filename)} " \
+              "[#{e.class}: #{e.message}]"
+      end
+    end
+
+    Registry.register(Pdf, 'pdf')
+  end
+end

data/lib/ndr_import/file/registry.rb

@@ -0,0 +1,50 @@
+module NdrImport
+  module File
+    # This mixin adds table enumeration functionality to importers.
+    module Registry
+      class <<self
+        attr_accessor :handlers
+
+        def register(klass, *formats)
+          @handlers ||= {}
+
+          formats.each do |format|
+            @handlers[format] = klass
+          end
+        end
+
+        def unregister(*formats)
+          formats.each do |format|
+            @handlers.delete(format)
+          end
+        end
+
+        def files(filename, options = {}, &block)
+          return enum_for(:files, filename, options) unless block
+
+          klass_factory(filename, nil, options).files(&block)
+        end
+
+        def tables(filename, format = nil, options = {}, &block)
+          return enum_for(:tables, filename, format, options) unless block
+
+          klass_factory(filename, format, options).tables(&block)
+        end
+
+        private
+
+        def klass_factory(filename, format, options)
+          format ||= SafeFile.extname(filename).delete('.').downcase
+          klass = Registry.handlers[format]
+          if klass
+            klass.new(filename, format, options)
+          else
+            fail "Error: Unknown file format #{format.inspect}"
+          end
+        end
+      end
+    end
+  end
+end
+
+require_relative 'all'