ndr_import 7.0.0 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a5dd34ac83682dce8b3bd5d53f3610c750e3db86defc7e6f808738d4c6a47d9
-  data.tar.gz: 942ca0f0d52d949c39d338938a6ab1d354f1c037675bcb8bae940c9357e672a0
+  metadata.gz: f74d60e93cf1f1d72baeebcbe931fc4225c3e74c11aaa13c324578b261561008
+  data.tar.gz: 70657ad1f113089bd2f719de5b59317310212c730ba34e762d99f9395da8fff2
 SHA512:
-  metadata.gz: 370e387ead245e596ac48c4d9669e86083754e43b3d9521b20501a13569286c3820e8a699b3492121410b4de2ceada0e9c9957d3e07469afc95820f42ab3ffaa
-  data.tar.gz: 1e74f1eb05e132fd611250730c5f720d10930488401f07774bd12afa853dde31f22e0d1e54e3fd0d24f93a77633b8136e852932a8fc634057fda4ccba7954113
+  metadata.gz: 91bced100d286a44df6fa193cad58a2ad85f6733d9b8cbbf1970e438ed8a3a7d9785a6d0e8b16396713511d045edd0a6aba00090a903c217e4c71b8e2fde7174
+  data.tar.gz: 4e2e968084f78f25fffebbc793806402fefd515605afc0ac9c81a5d535f0cb7f27a50af3c4dcb733ff5a9a8bc17311084c65593a2fb971ac35c5a4937bbfe777

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+## [Unreleased]
+*no unreleased changes*
+
+## 8.0.0 / 2018-11-26
+### Changed
+* Strip non tabular captured values by default (#28)
+
+### Added
+* Add validations to field mappings (#27)
+* control `ndr_table` serialise order with `encode_with` method (#30)
+
+## 7.0.0 / 2018-11-09
+### Changed
+* Breaking refactor of universal importer mixin (#25)
+* Update `pdf-reader` version to support recent Rubies (#16)
+
+## 6.4.1 / 2018-10-18
+### Fixed
+* bump nokogiri re: CVE-2018-14404
+
+## 6.4.0 / 2018-10-17
+### Added
+* Allow `decode: :word_doc` to read a .DOCX file (#26)
+
+## 6.3.0 / 2018-10-12
+### Added
+* Add XML file support (#22)
+* Add liberal CSV parsing option (#24)

data/README.md

@@ -2,7 +2,7 @@
 
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, XML and Zip files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, XML, 7-Zip and Zip files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation
@@ -48,7 +48,7 @@ files.each do |filename|
 end
 ```
 
-See test/readme_test.rb for a more complete working example.
+See `test/readme_test.rb` for a more complete working example.
 
 More information on the workings of the mapper are available in the [wiki](https://github.com/PublicHealthEngland/ndr_import/wiki).
 

data/code_safety.yml

@@ -16,6 +16,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+  CHANGELOG.md:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 952e45599240ab96b22d7ce223a7551853e50ae5
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -35,7 +39,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -59,7 +63,7 @@ file safety:
   lib/ndr_import.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: eca44583e9989159b45e90021dd1c65228447180
   lib/ndr_import/csv_library.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -67,7 +71,7 @@ file safety:
   lib/ndr_import/file/all.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 4ab72f84201c2d5f0147b7dfd041f488f6ff0422
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
   lib/ndr_import/file/base.rb:
     comments: 
     reviewed_by: timgentry
@@ -92,6 +96,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 2104514689a1a1286195fff18144a8cecb93048b
+  lib/ndr_import/file/seven_zip.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
   lib/ndr_import/file/text.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -143,15 +151,19 @@ file safety:
   lib/ndr_import/mapper.rb:
     comments: Writes to a Tempfile, but cleans up. Ruby will respect TMP_DIR
     reviewed_by: josh.pencheon
-    safe_revision: a41595349baf4428e63380c103b524ba2fa8d7df
+    safe_revision: eca44583e9989159b45e90021dd1c65228447180
   lib/ndr_import/mapping_error.rb:
     comments: 
     reviewed_by: timgentry
     safe_revision: 96ccc535bb9f933081dfc73bf0442b32b6bdce1d
+  lib/ndr_import/missing_field_error.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: eca44583e9989159b45e90021dd1c65228447180
   lib/ndr_import/non_tabular/column_mapping.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 96ccc535bb9f933081dfc73bf0442b32b6bdce1d
+    safe_revision: f216687d0bba7f2940f74a3353a32be3c900c194
   lib/ndr_import/non_tabular/line.rb:
     comments: 
     reviewed_by: timgentry
@@ -179,7 +191,7 @@ file safety:
   lib/ndr_import/table.rb:
     comments: uses File.basename
     reviewed_by: josh.pencheon
-    safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+    safe_revision: 2baccd482bcd875dc273f6a34a0e28823790e8d5
   lib/ndr_import/universal_importer_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -187,7 +199,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: 3f1743adb7a0ae612e480ffb9b39ba14bfb0493f
+    safe_revision: 952e45599240ab96b22d7ce223a7551853e50ae5
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -195,7 +207,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
+    safe_revision: eca44583e9989159b45e90021dd1c65228447180
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -219,7 +231,11 @@ file safety:
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: cb24ed3ea8116730d07f74546cd6fed0738b171d
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
+  test/file/seven_zip_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -267,7 +283,7 @@ file safety:
   test/mapper_test.rb:
     comments: exposes Mapper internals to test them
     reviewed_by: josh.pencheon
-    safe_revision: a41595349baf4428e63380c103b524ba2fa8d7df
+    safe_revision: eca44583e9989159b45e90021dd1c65228447180
   test/non_tabular/mapping_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -275,7 +291,7 @@ file safety:
   test/non_tabular/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 337bf56e39f0f08cf7593b03867bb2da48630663
+    safe_revision: f216687d0bba7f2940f74a3353a32be3c900c194
   test/non_tabular_file_helper_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -364,6 +380,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 71979e0a602ca5a0ce415c194f10add9959f0116
+  test/resources/normal.7z:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
   test/resources/normal.csv:
     comments: 
     reviewed_by: timgentry
@@ -463,7 +483,7 @@ file safety:
   test/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 14775e7262dbec3b6d0f69930ff526f856a94214
+    safe_revision: 2baccd482bcd875dc273f6a34a0e28823790e8d5
   test/test_helper.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import.rb

@@ -1,6 +1,7 @@
 require 'ndr_import/version'
 require 'ndr_import/csv_library'
 require 'ndr_import/mapping_error'
+require 'ndr_import/missing_field_error'
 require 'ndr_import/mapper'
 require 'ndr_import/non_tabular_file_helper'
 require 'ndr_import/table'

data/lib/ndr_import/file/all.rb

@@ -3,6 +3,7 @@ require_relative 'delimited'
 require_relative 'docx'
 require_relative 'excel'
 require_relative 'pdf'
+require_relative 'seven_zip'
 require_relative 'text'
 require_relative 'unregistered_filetype'
 require_relative 'word'

data/lib/ndr_import/file/seven_zip.rb

@@ -0,0 +1,70 @@
+require 'seven_zip_ruby'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a 7zip file handler that returns tables from the extracted files.
+    class SevenZip < Base
+      def initialize(filename, format, options = {})
+        super
+        @pattern = options['pattern'] || //
+        @unzip_path = options['unzip_path']
+        @password = options['password']
+
+        validate_unzip_path_is_safe!
+      end
+
+      def files(&block)
+        raise 'Not allowed in external environment' if defined?(::Rails) && ::Rails.env.external?
+
+        return enum_for(:files) unless block
+
+        destination = @unzip_path.join(Time.current.strftime('%H%M%S%L'))
+        FileUtils.mkdir_p(SafeFile.safepath_to_string(destination))
+
+        ::File.open(SafeFile.safepath_to_string(@filename), 'rb') do |zipfile|
+          unzip_entries(zipfile, destination, &block)
+        end
+      end
+
+      # 7zip files produce files, never tables.
+      def tables
+        raise 'SevenZip#tables should never be called'
+      end
+
+      private
+
+      # Unzip the 7zip file entry and enumerate over it
+      def unzip_entries(zipfile, destination, &block)
+        SevenZipRuby::Reader.open(zipfile, password: @password) do |szr|
+          szr.entries.each do |entry|
+            # SECURE: TPG 2018-11-21: The path is stripped from the zipfile entry when extracted
+            basename = ::File.basename(entry.path)
+            next unless entry.file? && basename.match(@pattern)
+
+            unzipped_filename = destination.join(basename)
+            szr.extract([entry], unzipped_filename)
+
+            unzipped_files(unzipped_filename, &block)
+          end
+        end
+      end
+
+      # Enumerate over an unzipped file like any other
+      def unzipped_files(unzipped_filename)
+        Registry.files(unzipped_filename, @options).each do |filename|
+          yield(filename)
+        end
+      end
+
+      def validate_unzip_path_is_safe!
+        SafeFile.safepath_to_string(@unzip_path)
+      end
+    end
+
+    Registry.register(SevenZip, '7z')
+  end
+end

data/lib/ndr_import/mapper.rb

@@ -23,11 +23,13 @@ module NdrImport::Mapper
     MAPPINGS         = 'mappings'.freeze
     MATCH            = 'match'.freeze
     ORDER            = 'order'.freeze
+    PRESENCE         = 'presence'.freeze
     PRIORITY         = 'priority'.freeze
     RAWTEXT_NAME     = 'rawtext_name'.freeze
     REPLACE          = 'replace'.freeze
     STANDARD_MAPPING = 'standard_mapping'.freeze
     UNPACK_PATTERN   = 'unpack_pattern'.freeze
+    VALIDATES        = 'validates'.freeze
   end
 
   private
@@ -89,8 +91,8 @@ module NdrImport::Mapper
     line.each_with_index do |raw_value, col|
       column_mapping = line_mappings[col]
       if column_mapping.nil?
-        fail ArgumentError,
-             "Line has too many columns (expected #{line_mappings.size} but got #{line.size})"
+        raise ArgumentError,
+              "Line has too many columns (expected #{line_mappings.size} but got #{line.size})"
       end
 
       next if column_mapping[Strings::DO_NOT_CAPTURE]
@@ -100,7 +102,8 @@ module NdrImport::Mapper
       end
 
       # Establish the rawtext column name we are to use for this column
-      rawtext_column_name = (column_mapping[Strings::RAWTEXT_NAME] || column_mapping[Strings::COLUMN]).downcase
+      rawtext_column_name = (column_mapping[Strings::RAWTEXT_NAME] ||
+                              column_mapping[Strings::COLUMN]).downcase
 
       # Replace raw_value with decoded raw_value
       Array(column_mapping[Strings::DECODE]).each do |encoding|
@@ -123,6 +126,9 @@ module NdrImport::Mapper
         replace_before_mapping(original_value, field_mapping)
         value = mapped_value(original_value, field_mapping)
 
+        validations = field_mapping[Strings::VALIDATES].presence
+        apply_validations_on(field_mapping[Strings::FIELD], value, validations) if validations
+
         # We don't care about blank values, unless we're mapping a :join
         # field (in which case, :compact may or may not be being used).
         next if value.blank? && !field_mapping[Strings::JOIN]
@@ -135,7 +141,9 @@ module NdrImport::Mapper
 
         if field_mapping[Strings::ORDER]
           data[field][:join] ||= field_mapping[Strings::JOIN]
-          data[field][:compact] = field_mapping[Strings::COMPACT] if field_mapping.key?(Strings::COMPACT)
+          if field_mapping.key?(Strings::COMPACT)
+            data[field][:compact] = field_mapping[Strings::COMPACT]
+          end
 
           data[field][:values][field_mapping[Strings::ORDER] - 1] = value
         elsif field_mapping[Strings::PRIORITY]
@@ -156,7 +164,7 @@ module NdrImport::Mapper
       attributes[field] =
         if field_data.key?(:join)
           # Map "blank" values to nil:
-          values = values.map { |value| value if value.present? }
+          values = values.map(&:presence)
           values.compact! if field_data[:compact]
           values.join(field_data[:join])
         else
@@ -222,6 +230,20 @@ module NdrImport::Mapper
     true
   end
 
+  # Apply ActiveRecord-like validations specified in field mappings, e.g.
+  # - column: column_one
+  #   mappings:
+  #   - field: field_one
+  #     validates:
+  #       presence: true
+  def apply_validations_on(field, value, validations)
+    presence_validation_on(field, value) if validations[Strings::PRESENCE]
+  end
+
+  def presence_validation_on(field, value)
+    raise NdrImport::MissingFieldError, field if value.blank?
+  end
+
   # Decode raw_value using specified encoding
   # E.g. adding decode to a column:
   #

data/lib/ndr_import/missing_field_error.rb

@@ -0,0 +1,14 @@
+require 'active_model' # Source I18n translations
+
+module NdrImport
+  # Raised if a mandatory field is blank.
+  class MissingFieldError < StandardError
+    attr_reader :field
+
+    def initialize(field)
+      @field = field
+      message = "#{field} #{I18n.t('errors.messages.blank')}"
+      super(message)
+    end
+  end
+end

data/lib/ndr_import/non_tabular/column_mapping.rb

@@ -32,7 +32,8 @@ module NdrImport
 
       # capture the required part of the line by replacing (recusively) the line,
       # with the first captured regular expression group. This is hardcoded in an attempt
-      # to preserve the rawtext as much as possible
+      # to preserve the rawtext as much as possible.
+      # The captured value is `String#strip`ed by default.
       def capture_value(line)
         value = line.dup
         [@cell_mapping['capture']].flatten.each do |pattern|
@@ -42,7 +43,7 @@ module NdrImport
             value = nil
           end
         end
-        value
+        value.nil? ? value : value.strip
       end
 
       def validate_cell_mapping

data/lib/ndr_import/table.rb

@@ -92,6 +92,16 @@ module NdrImport
       @header_valid == true
     end
 
+    # For readability, we should serialise the columns last
+    def encode_with(coder)
+      options = self.class.all_valid_options - ['columns']
+      options.each do |option|
+        value = send(option)
+        coder[option] = value if value
+      end
+      coder['columns'] = @columns
+    end
+
     private
 
     # This method uses a buffer to not yield the last <buffer_size> iterations of an enumerable.

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '7.0.0'.freeze
+  VERSION = '8.0.0'.freeze
 end

data/ndr_import.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'activemodel'
   spec.add_dependency 'activesupport', '>= 3.2.18', '< 5.3'
   spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
@@ -31,6 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.5'
   spec.add_dependency 'pdf-reader', '~> 2.1'
   spec.add_dependency 'roo-xls'
+  spec.add_dependency 'seven_zip_ruby', '~> 1.2'
   spec.add_dependency 'spreadsheet', '1.0.3'
 
   spec.required_ruby_version = '>= 2.4'

data/test/file/registry_test.rb

@@ -12,7 +12,7 @@ module NdrImport
 
       test 'Registry.handlers' do
         assert_instance_of Hash, NdrImport::File::Registry.handlers
-        assert_equal %w[csv delimited doc docx nontabular pdf text txt xls xlsx xml_table zip],
+        assert_equal %w[7z csv delimited doc docx nontabular pdf text txt xls xlsx xml_table zip],
                      NdrImport::File::Registry.handlers.keys.sort
       end
 

data/test/file/seven_zip_test.rb

@@ -0,0 +1,59 @@
+require 'test_helper'
+require 'ndr_import/file/seven_zip'
+
+module NdrImport
+  module File
+    # 7zip file handler tests
+    class SevenZipTestTest < ActiveSupport::TestCase
+      def setup
+        @home = SafePath.new('test_space_rw')
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should reject non SafePath arguments' do
+        file_path = @home.join('imaginary.7z')
+
+        assert_raises ArgumentError do
+          NdrImport::File::SevenZip.new(file_path.to_s, nil, 'unzip_path' => @home.to_s)
+        end
+
+        assert_raises ArgumentError do
+          NdrImport::File::SevenZip.new(file_path.to_s, nil, 'unzip_path' => @home)
+        end
+
+        assert_raises ArgumentError do
+          NdrImport::File::SevenZip.new(file_path, nil, 'unzip_path' => @home.to_s)
+        end
+      end
+
+      test 'should read 7zip file with correct password' do
+        options = { 'password' => 'FortuneCookie', 'unzip_path' => @home }
+        file_path = @permanent_test_files.join('normal.7z')
+
+        handler = NdrImport::File::SevenZip.new(file_path, nil, options)
+        handler.files.all? do |filename|
+          assert_instance_of SafePath, filename
+        end
+        files = handler.files.to_a
+        assert_equal 'normal_pipe.csv', ::File.basename(files[0])
+        assert_equal 'normal_thorn.csv', ::File.basename(files[1])
+
+        exception = assert_raises RuntimeError do
+          handler.tables
+        end
+        assert_equal 'SevenZip#tables should never be called', exception.message
+      end
+
+      test 'should not read 7zip file with incorrect password' do
+        options = { 'password' => 'WrongPassword', 'unzip_path' => @home }
+        file_path = @permanent_test_files.join('normal.7z')
+
+        handler = NdrImport::File::SevenZip.new(file_path, nil, options)
+
+        assert_raises SevenZipRuby::InvalidArchive do
+          handler.files.to_a
+        end
+      end
+    end
+  end
+end

data/test/mapper_test.rb

@@ -293,6 +293,17 @@ class MapperTest < ActiveSupport::TestCase
           : 'RGT01'
   YML
 
+  validates_presence_mapping = YAML.safe_load <<-YML
+    - column: column_one
+      mappings:
+      - field: field_one
+        validates:
+          presence: true
+    - column: column_two
+      mappings:
+      - field: field_two
+  YML
+
   test 'map should return a number' do
     assert_equal '1', TestMapper.new.mapped_value('A', map_mapping)
   end
@@ -365,6 +376,13 @@ class MapperTest < ActiveSupport::TestCase
     assert_equal %w(RGT01 RGT01), mapped_value['hospital']
   end
 
+  test 'should raise an error on blank mandatory field' do
+    exception = assert_raise(NdrImport::MissingFieldError) do
+      TestMapper.new.mapped_line(['', 'RGT01'], validates_presence_mapping)
+    end
+    assert_equal "field_one can't be blank", exception.message
+  end
+
   test 'should return correct date format for date fields with daysafter' do
     assert_equal Date.new(2012, 5, 18), TestMapper.new.mapped_value(2, daysafter_mapping)
     assert_equal Date.new(2012, 5, 18), TestMapper.new.mapped_value('2', daysafter_mapping)

data/test/non_tabular/table_test.rb

@@ -542,4 +542,48 @@ STR
       table.transform(junk).to_a
     end
   end
+
+  def test_should_strip_captured_rawtext
+    unwanted_white_space = <<-STR.each_line
+111
+Trailing whitespace        end_of_line
+------
+111
+        Leading whitespaceend_of_line
+------
+111
+        Leading and trailing whitespace        end_of_line
+------
+111
+Should not match this
+------
+STR
+
+    table = YAML.load <<-YML.strip_heredoc
+      --- !ruby/object:NdrImport::NonTabular::Table
+      start_line_pattern: !ruby/regexp /^111$/
+      end_in_a_record: true
+      klass: SomeTestKlass
+      columns:
+      - column: one
+        non_tabular_cell:
+          lines: 0
+          capture: !ruby/regexp /^(.*)end_of_line$/i
+          trim_rawtext: left
+    YML
+
+    enum = table.transform(unwanted_white_space)
+    assert_instance_of Enumerator, enum
+
+    output = []
+    enum.each do |klass, fields, index|
+      output << [klass, fields, index]
+    end
+
+    expected_rawtext_ouput = [{ 'one' => 'Trailing whitespace' },
+                              { 'one' => 'Leading whitespace' },
+                              { 'one' => 'Leading and trailing whitespace' },
+                              { 'one' => '' }]
+    assert_equal expected_rawtext_ouput, (output.map { |row| row[1][:rawtext] })
+  end
 end

data/test/table_test.rb

@@ -1,5 +1,9 @@
 require 'test_helper'
 
+class TestNoCoderTable < NdrImport::NonTabular::Table
+  undef_method :encode_with
+end
+
 # This tests the NdrImport::Table mapping class
 class TableTest < ActiveSupport::TestCase
   def test_deserialize_table
@@ -137,7 +141,67 @@ class TableTest < ActiveSupport::TestCase
   end
 
   def test_encode_with
-    # encode_with(coder)
+    table = NdrImport::Table.new
+    assert table.instance_variables.include?(:@row_index)
+    refute table.class.all_valid_options.include?('row_index')
+    assert_nil table.columns
+
+    coder = {}
+    table.encode_with(coder)
+    assert coder.key?('columns')
+
+    yaml_output = table.to_yaml
+    assert yaml_output.include?('columns')
+    refute yaml_output.include?('row_index')
+    assert YAML.load(yaml_output).is_a?(NdrImport::Table)
+  end
+
+  def test_encode_with_compare
+    table_options = {
+      columns: %w[a b],
+      klass: 'SomeKlass',
+      start_line_pattern: 'TODO',
+      end_line_pattern: 'TODO'
+    }
+    no_coder_table = TestNoCoderTable.new(table_options)
+    ndr_table = NdrImport::NonTabular::Table.new(table_options)
+
+    assert no_coder_table.is_a?(NdrImport::Table)
+    assert ndr_table.is_a?(NdrImport::Table)
+    assert no_coder_table.is_a?(NdrImport::NonTabular::Table)
+    assert ndr_table.is_a?(NdrImport::NonTabular::Table)
+
+    refute no_coder_table.respond_to?(:encode_with)
+    assert ndr_table.respond_to?(:encode_with)
+
+    no_coder_table_yaml_order = get_yaml_mapping_order(no_coder_table.to_yaml)
+    ndr_table_yaml_order = get_yaml_mapping_order(ndr_table.to_yaml)
+
+    # no_coder_table_yaml_order => ["klass", "columns", "start_line_pattern", "end_line_pattern", "row_index"]
+    # ndr_table_yaml_order => ["klass", "start_line_pattern", "end_line_pattern", "columns"]
+
+    assert no_coder_table_yaml_order.include?('row_index')
+    refute ndr_table_yaml_order.include?('row_index')
+
+    refute no_coder_table_yaml_order.last == 'columns'
+    assert ndr_table_yaml_order.last == 'columns'
+
+    # test objects deserialized from yaml mappings
+    deserialized_no_coder_table_yaml = YAML.load(no_coder_table.to_yaml)
+    deserialized_ndr_table_yaml = YAML.load(ndr_table.to_yaml)
+
+    assert deserialized_no_coder_table_yaml.is_a?(NdrImport::NonTabular::Table)
+    assert deserialized_ndr_table_yaml.is_a?(NdrImport::NonTabular::Table)
+
+    assert_nil deserialized_no_coder_table_yaml.filename_pattern
+    assert_equal deserialized_no_coder_table_yaml.klass, no_coder_table.klass
+    assert_equal deserialized_no_coder_table_yaml.start_line_pattern, no_coder_table.start_line_pattern
+    assert_equal deserialized_no_coder_table_yaml.columns, no_coder_table.columns
+
+    assert_nil deserialized_ndr_table_yaml.filename_pattern
+    assert_equal deserialized_ndr_table_yaml.klass, ndr_table.klass
+    assert_equal deserialized_ndr_table_yaml.start_line_pattern, ndr_table.start_line_pattern
+    assert_equal deserialized_ndr_table_yaml.columns, ndr_table.columns
   end
 
   def test_skip_footer_lines
@@ -407,4 +471,10 @@ YML
       ]
     }
   end
+
+  def get_yaml_mapping_order(yaml_mapping)
+    yaml_mapping.split("\n").
+      delete_if { |line| /-+/.match(line) }.
+      map { |line| /(.*):/.match(line)[1].to_s }
+  end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 8.0.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-09 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -160,6 +174,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: seven_zip_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
@@ -329,6 +357,7 @@ files:
 - ".gitignore"
 - ".hound.yml"
 - ".rubocop.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Guardfile
@@ -349,6 +378,7 @@ files:
 - lib/ndr_import/file/excel.rb
 - lib/ndr_import/file/pdf.rb
 - lib/ndr_import/file/registry.rb
+- lib/ndr_import/file/seven_zip.rb
 - lib/ndr_import/file/text.rb
 - lib/ndr_import/file/unregistered_filetype.rb
 - lib/ndr_import/file/word.rb
@@ -363,6 +393,7 @@ files:
 - lib/ndr_import/helpers/file/zip.rb
 - lib/ndr_import/mapper.rb
 - lib/ndr_import/mapping_error.rb
+- lib/ndr_import/missing_field_error.rb
 - lib/ndr_import/non_tabular/column_mapping.rb
 - lib/ndr_import/non_tabular/line.rb
 - lib/ndr_import/non_tabular/mapping.rb
@@ -381,6 +412,7 @@ files:
 - test/file/excel_test.rb
 - test/file/pdf_test.rb
 - test/file/registry_test.rb
+- test/file/seven_zip_test.rb
 - test/file/text_test.rb
 - test/file/word_test.rb
 - test/file/xml_test.rb
@@ -417,6 +449,7 @@ files:
 - test/resources/malformed.csv
 - test/resources/malformed.xml
 - test/resources/malformed_pipe.csv
+- test/resources/normal.7z
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
 - test/resources/normal_pipe.csv
@@ -476,6 +509,7 @@ test_files:
 - test/file/excel_test.rb
 - test/file/pdf_test.rb
 - test/file/registry_test.rb
+- test/file/seven_zip_test.rb
 - test/file/text_test.rb
 - test/file/word_test.rb
 - test/file/xml_test.rb
@@ -512,6 +546,7 @@ test_files:
 - test/resources/malformed.csv
 - test/resources/malformed.xml
 - test/resources/malformed_pipe.csv
+- test/resources/normal.7z
 - test/resources/normal.csv
 - test/resources/normal.csv.zip
 - test/resources/normal_pipe.csv