ndr_import 3.0.0

data/lib/ndr_import/file/text.rb

@@ -0,0 +1,52 @@
+require 'ndr_support/safe_file'
+require 'ndr_support/utf8_encoding'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a text file handler that returns a single table.
+    class Text < Base
+      include UTF8Encoding
+
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        # Encoding:
+        #   As we're going to be yielding the lines of the file as it is streamed
+        #   (rather than slurped in advance), we need to know which encoding / mode
+        #   is going to work in advance.
+        #
+        path = SafeFile.safepath_to_string(@filename)
+        mode = read_mode_for(path)
+
+        # SECURE: TG 13 Oct 2015 SafeFile.safepath_to_string ensures that the path is SafePath.
+        ::File.new(path, mode).each { |line| block.call ensure_utf8!(line).chomp }
+      rescue => e
+        raise "Failed to read #{SafeFile.basename(@filename)} as text [#{e.class}: #{e.message}]"
+      end
+
+      # TODO: In Ruby 2.0+, a mode of "rb:bom|utf-16:utf-8" seemed to fix all cases,
+      # but this doesn't work on Ruby 1.9.3, which we are currently still supporting.
+      # Therefore, we have to test multiple modes in advance, hence #read_mode_for.
+      def read_mode_for(trusted_path)
+        # These are the read modes we will try, in order:
+        modes = ['rb:utf-16:utf-8', 'r:utf-8']
+
+        begin
+          ::File.new(trusted_path, modes.first).each { |_line| }
+        rescue Encoding::InvalidByteSequenceError
+          modes.shift # That one didn't work...
+          retry if modes.any?
+        end
+
+        modes.first || fail('Unable to determine working stream encoding!')
+      end
+    end
+
+    Registry.register(Text, 'txt') # TODO: Add 'nontabular'?
+  end
+end

data/lib/ndr_import/file/word.rb

@@ -0,0 +1,30 @@
+require 'msworddoc-extractor'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a Word document file handler that returns a single table.
+    # currently only works on .doc (97-2003), not.docx
+    class Word < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        doc = MSWordDoc::Extractor.load(SafeFile.safepath_to_string(@filename))
+
+        doc.whole_contents.split("\n").each do |line|
+          block.call(line)
+        end
+
+      rescue => e
+        raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
+      end
+    end
+
+    Registry.register(Word, 'doc') # TODO: Add 'word'?
+  end
+end

data/lib/ndr_import/file/zip.rb

@@ -0,0 +1,67 @@
+require 'zip'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a zip file handler that returns tables from the extracted files.
+    class Zip < Base
+      def initialize(filename, format, options = {})
+        super
+        @pattern = options['pattern'] || //
+        @unzip_path = options['unzip_path']
+
+        validate_unzip_path_is_safe!
+      end
+
+      def files(&block)
+        fail 'Not allowed in external environment' if defined?(::Rails) && ::Rails.env.external?
+
+        return enum_for(:files) unless block
+
+        destination = @unzip_path.join(Time.current.strftime('%H%M%S%L'))
+        FileUtils.mkdir_p(SafeFile.safepath_to_string(destination))
+
+        ::Zip::File.open(SafeFile.safepath_to_string(@filename)) do |zipfile|
+          unzip_entries(zipfile, destination, &block)
+        end
+      end
+
+      # Zip files produce files, never tables.
+      def tables
+        fail 'Zip#tables should never be called'
+      end
+
+      private
+
+      # Unzip the zip file entry and enumerate over it
+      def unzip_entries(zipfile, destination, &block)
+        zipfile.entries.each do |entry|
+          # SECURE: TPG 2010-11-1: The path is stripped from the zipfile entry when extracted
+          basename = ::File.basename(entry.name)
+          next unless entry.file? && basename.match(@pattern)
+
+          unzipped_filename = destination.join(basename)
+          zipfile.extract(entry, unzipped_filename)
+
+          unzipped_files(unzipped_filename, &block)
+        end
+      end
+
+      # Enumerate over an unzipped file like any other
+      def unzipped_files(unzipped_filename, &block)
+        Registry.files(unzipped_filename, @options).each do |filename|
+          block.call(filename)
+        end
+      end
+
+      def validate_unzip_path_is_safe!
+        SafeFile.safepath_to_string(@unzip_path)
+      end
+    end
+
+    Registry.register(Zip, 'zip')
+  end
+end

data/lib/ndr_import/helpers/file/delimited.rb

@@ -0,0 +1,105 @@
+require 'ndr_support/safe_file'
+require 'ndr_import/csv_library'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds delimited file functionality to unified importers.
+      module Delimited
+        # Read a plain text CSV file, return an array of the content
+        def read_csv_file(path)
+          # Read the page below when encountering "CSV::IllegalFormatError" error caused by CSV
+          # file generated at MAC OS
+          # http://stackoverflow.com/questions/1549139/ruby-cannot-parse-excel-file-exported-as-csv-in-os-x
+
+          read_delimited_file(path)
+        end
+
+        # Slurp the entire file into an array of lines.
+        def read_delimited_file(path, col_sep = nil)
+          delimited_rows(path, col_sep).to_a
+        end
+
+        # Iterate through the file table by table, yielding each one in turn.
+        def delimited_tables(path, col_sep = nil)
+          return enum_for(:delimited_tables, path, col_sep) unless block_given?
+
+          yield nil, delimited_rows(path, col_sep)
+        end
+
+        # Deprecated method
+        def each_delimited_table(path, col_sep = nil, &block)
+          Kernel.warn '[warning] each_delimited_table will be deprecated,' \
+                      ' please use delimited_tables instead.'
+          delimited_tables(path, col_sep, &block)
+        end
+
+        # Iterate through the file line by line, yielding each one in turn.
+        def delimited_rows(path, col_sep = nil)
+          return enum_for(:delimited_rows, path, col_sep) unless block_given?
+
+          safe_path = SafeFile.safepath_to_string(path)
+          encodings = determine_encodings!(safe_path, col_sep)
+
+          # By now, we know `encodings` should let us read the whole
+          # file succesfully; if there are problems, we should crash.
+          CSVLibrary.foreach(safe_path, encodings) do |line|
+            yield line.map(&:to_s) unless line.length <= 5
+          end
+        end
+
+        # Deprecated method
+        def each_delimited_row(path, col_sep = nil, &block)
+          Kernel.warn '[warning] each_delimited_row will be deprecated,' \
+                      ' please use delimited_rows instead.'
+          delimited_rows(path, col_sep, &block)
+        end
+
+        private
+
+        # Derive the source encoding by trying all supported encodings.
+        # Returns first set of working options, or raises if none could be found.
+        def determine_encodings!(safe_path, col_sep = nil)
+          # delimiter encoding => # FasterCSV encoding string
+          supported_encodings = {
+            'UTF-8'        => 'bom|utf-8',
+            'Windows-1252' => 'windows-1252:utf-8'
+          }
+
+          successful_options = nil
+          supported_encodings.each do |delimiter_encoding, csv_encoding|
+            begin
+              options = {
+                :col_sep  => (col_sep || ',').force_encoding(delimiter_encoding),
+                :encoding => csv_encoding
+              }
+
+              row_num = 0
+              # Iterate through the file; if we reach the end, this encoding worked:
+              CSVLibrary.foreach(safe_path, options) { |_line| row_num += 1 }
+            rescue ArgumentError => e
+              next if e.message =~ /invalid byte sequence/ # This encoding didn't work
+              raise(e)
+            rescue CSVLibrary::MalformedCSVError => e
+              description = (col_sep ? col_sep.inspect + ' delimited' : 'CSV')
+
+              raise(CSVLibrary::MalformedCSVError, "Invalid #{description} format " \
+                "on row #{row_num + 1} of #{::File.basename(safe_path)}. Original: #{e.message}")
+            end
+
+            # We got this far => encoding choice worked:
+            successful_options = options
+            break
+          end
+
+          # We tried them all, and none worked:
+          unless successful_options
+            fail "None of the encodings #{supported_encodings.values.inspect} were successful!"
+          end
+
+          successful_options
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/helpers/file/excel.rb

@@ -0,0 +1,181 @@
+require 'ndr_support/safe_file'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds excel spreadsheet functionality to unified importers.
+      # It provides a file reader method and methods to cast raw values
+      # appropriately. These methods can be overridden or aliased as required.
+      #
+      module Excel
+        require 'roo'
+        require 'roo-xls'
+        require 'ole/storage'
+
+        protected
+
+        def cast_excel_value(raw_value)
+          return raw_value if raw_value.nil?
+
+          if raw_value.is_a?(Date) || raw_value.is_a?(DateTime) || raw_value.is_a?(Time)
+            cast_excel_datetime_as_date(raw_value)
+          elsif raw_value.is_a?(Float)
+            if raw_value.to_f == raw_value.to_i
+              # Whole number
+              return raw_value.to_i.to_s
+            else
+              return raw_value.to_f.to_s
+            end
+          else
+            return raw_value.to_s.strip
+          end
+        end
+
+        def cast_excel_datetime_as_date(raw_value)
+          raw_value.to_s(:db)
+        end
+
+        # Iterate through the file table by table, yielding each one in turn.
+        def excel_tables(path)
+          return enum_for(:excel_tables, path) unless block_given?
+
+          workbook = load_workbook(path)
+          workbook.each_with_pagename do |tablename, sheet|
+            yield tablename, excel_rows(workbook, sheet)
+          end
+        end
+
+        # Deprecated method
+        def each_excel_table(path, &block)
+          Kernel.warn '[warning] each_excel_table will be deprecated,' \
+                      ' please use excel_tables instead.'
+          excel_tables(path, &block)
+        end
+
+        private
+
+        def read_excel_file(path, selected_sheet = nil)
+          # SECURE: TVB Mon Aug 13 15:30:32 BST 2012 SafeFile.safepath_to_string makes sure that
+          # the path is SafePath.
+
+          # Load the workbook
+          workbook = load_workbook(path)
+
+          # Choose selected worksheet (if provided and exist) or the first worksheet
+          workbook.default_sheet =
+            if selected_sheet.nil? || !workbook.sheets.include?(selected_sheet.to_s)
+              workbook.sheets.first
+            else
+              selected_sheet.to_s
+            end
+
+          # Read the cells from working worksheet into a nested array
+          excel_rows(workbook, workbook).to_a
+        end
+
+        # Iterate through the sheet line by line, yielding each one in turn.
+        def excel_rows(workbook, sheet, &block)
+          return enum_for(:excel_rows, workbook, sheet) unless block
+
+          if workbook.is_a?(Roo::Excelx)
+            # FIXME: xlsx_rows(sheet, &block) should produce the same output as xls_rows
+            xls_rows(sheet, &block)
+          else
+            xls_rows(sheet, &block)
+          end
+        end
+
+        # Deprecated method
+        def each_excel_row(workbook, sheet, &block)
+          Kernel.warn '[warning] each_excel_row will be deprecated,' \
+                      ' please use excel_rows instead.'
+          excel_rows(workbook, sheet, &block)
+        end
+
+        # Iterate through an xls sheet line by line, yielding each one in turn.
+        def xls_rows(sheet)
+          return enum_for(:xls_rows, sheet) unless block_given?
+
+          sheet.first_row.upto(sheet.last_row) do |row|
+            line = []
+            sheet.first_column.upto(sheet.last_column) do |col|
+              line << cast_excel_value(sheet.cell(row, col))
+            end
+            yield line
+          end
+        end
+
+        # Deprecated method
+        def each_xls_row(sheet, &block)
+          Kernel.warn '[warning] each_xls_row will be deprecated,' \
+                      ' please use xls_rows instead.'
+          xls_rows(sheet, &block)
+        end
+
+        # Iterate through an xlsx sheet line by line, yielding each one in turn.
+        # This method uses streaming https://github.com/roo-rb/roo#excel-xlsx-and-xlsm-support
+        def xlsx_rows(sheet)
+          return enum_for(:xlsx_rows, sheet) unless block_given?
+
+          sheet.each_row_streaming(:pad_cells => true) do |row|
+            yield row.map { |cell| cast_excel_value(cell.value) }
+          end
+        end
+
+        # Deprecated method
+        def each_xlsx_row(sheet, &block)
+          Kernel.warn '[warning] each_xlsx_row will be deprecated,' \
+                      ' please use xlsx_rows instead.'
+          xlsx_rows(sheet, &block)
+        end
+
+        def get_excel_sheets_name(path)
+          workbook = load_workbook(path)
+          workbook.sheets
+        end
+
+        def load_workbook(path)
+          case SafeFile.extname(path).downcase
+          when '.xls'
+            Roo::Excel.new(SafeFile.safepath_to_string(path))
+          when '.xlsx'
+            Roo::Excelx.new(SafeFile.safepath_to_string(path))
+          else
+            fail "Received file path with unexpected extension #{SafeFile.extname(path)}"
+          end
+        rescue Ole::Storage::FormatError => e
+          # TODO: Do we need to remove the new_file after using it?
+
+          # try to load the .xls file as an .xlsx file, useful for sources like USOM
+          # roo check file extensions in file_type_check (GenericSpreadsheet),
+          # so we create a duplicate file in xlsx extension
+          if /(.*)\.xls$/.match(path)
+            new_file_name = SafeFile.basename(path).gsub(/(.*)\.xls$/, '\1_amend.xlsx')
+            new_file_path = SafeFile.dirname(path).join(new_file_name)
+            copy_file(path, new_file_path)
+
+            load_workbook(new_file_path)
+          else
+            raise e.message
+          end
+        rescue => e
+          raise ["Unable to read the file '#{path}'", e.message].join('; ')
+        end
+
+        # Note that this method can produce insecure calls. All callers must protect
+        # their arguments.
+        # Arguments:
+        #   * source - SafeFile
+        #   * dest - SafeFile
+        #
+        def copy_file(source, dest)
+          # SECURE: TVB Mon Aug 13 13:53:02 BST 2012 : Secure SafePath will do the security checks
+          # before it is converted to string.
+          # SafeFile will make sure that the arguments are actually SafePath
+          FileUtils.mkdir_p(SafeFile.safepath_to_string(SafeFile.dirname(dest)))
+          FileUtils.cp(SafeFile.safepath_to_string(source), SafeFile.safepath_to_string(dest))
+        end
+      end
+    end
+  end
+end

data/lib/ndr_import/helpers/file/pdf.rb

@@ -0,0 +1,29 @@
+require 'ndr_support/safe_file'
+
+module NdrImport
+  module Helpers
+    module File
+      # This mixin adds PDF functionality to unified importers. It provides a file reader method.
+      module Pdf
+        private
+
+        def read_pdf_file(path)
+          require 'pdf-reader'
+          lines = []
+          pagenum = 0
+          begin
+            reader = PDF::Reader.new(SafeFile.safepath_to_string(path))
+
+            reader.pages.each do |page|
+              lines.concat page.text.split("\n")
+              pagenum += 1
+            end
+          rescue => e
+            raise("Invalid format on page #{pagenum + 1} of #{SafeFile.basename(path)} [#{e.class}: #{e.message}]")
+          end
+          lines
+        end
+      end
+    end
+  end
+end