topographer 0.0.2 → 0.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjI0NDZhMDEyMDhkNzA0NzA4MTczMjBjYzY3NWZkMzQxODBmYzBjNg==
+    NDcwNmMxNWRlMjkzMDgzMzg0MTMxNjM1YjMzODhmMDRhZmYwNzFlMg==
   data.tar.gz: !binary |-
-    OTI1MzZhMWU2NTZhOTllOTE2Nzg4ZjMyZTVmMmU5MDQzMWU3ZmI1Ng==
+    Mjg4YzFlYWNiZmY0YzFkOTNlNGQ0ZDE5YWFkYmE1Zjk3YTRhOGVkYw==
 SHA512:
   metadata.gz: !binary |-
-    YzczMzllN2M3YTc3ZTc2M2E3YTY2NmJjMjU0YmNkODdhN2ViZDY5MGEzYjg3
-    MTU1YTVkZmIwYjUwM2UyNWNhMjAzMTdiMjc2ZmIwZTZiMWE5OWU1MGU1OTk5
-    ZDYyNWUyNTNjMjYzNzhkYzM4N2FiNjU4OWYyMWY0NzViNGMzNjM=
+    MDQ1YzFkYTI2MmI2ZTA2YzY3MzYyOWY1YzAyYTUwY2VhYmU1ZWVjMzllN2Y3
+    ODRmODMxNTc2ZmUxMzI1NWFhMDg4NGFmYTZkNWE5YmIyMTRkNTUxNjQ4N2Y3
+    NGRkMDlhMGNjZTM5YmQ1MDczZGVkMzM5ODNmNmEwYzQ4ZTYxZDA=
   data.tar.gz: !binary |-
-    MjExOTg5NGM5OWYzZDhkYzM1NzBlM2RmMTY5N2JiNDRmNmJkNDUyZDhhOTll
-    YTcwMWYyYmVkMjg5M2RkZTkwN2IwMjg5OWQ5N2I4MzhiNTEwZDQ3ZWVmNTEw
-    ODViNDYxMjA4NDFhODIyMTI3OWY0YzAyYTFmY2I1YjE2YmVlZDI=
+    MGZkODRiZGI0OWY0MTlkMjFhNGM2MTQ3YjA0YjYwNDZiZGU1YmExMmVkY2Ux
+    Y2IxZGM1ZjIyODA0ODU4YmZmMGQxNDFjNTM2YzJmYzUzZTEyM2Q4YjFlZjcx
+    MDJjZWMyZjlkZmQzMDlmMTdkNTc4ZTUyNjYyODk5N2IyYzVhYTk=

data/.gitignore

@@ -3,6 +3,7 @@
 .bundle
 .config
 .yardoc
+.flooignore
 Gemfile.lock
 InstalledFiles
 _yardoc

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "1.9.3"
+  - "2.0.0"
+script: bundle exec rspec spec

data/README.md

@@ -1,4 +1,4 @@
-# Topographer
+# Topographer [![Gem Version](https://badge.fury.io/rb/topographer.png)](http://badge.fury.io/rb/topographer) [![Build Status](https://travis-ci.org/SciMed/topographer.png?branch=master)](https://travis-ci.org/SciMed/topographer)
 
 Topographer is a gem that provides functionality to conveniently import from various data sources into
 ActiveRecord or other ORM systems.  This accomplished by defining an input wrapper, a mapping from input data to
@@ -32,7 +32,4 @@ Check back soon for detailed usage instructions
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
-=======
-Topographer
-===========
 

data/lib/Topographer/importer/input/source_data.rb

@@ -5,4 +5,8 @@ class Topographer::Importer::Input::SourceData
     @source_identifier = source_identifier
     @data = data
   end
+
+  def empty?
+    @data.empty?
+  end
 end

data/lib/Topographer/importer/mapper/mapper_builder.rb

@@ -0,0 +1,46 @@
+class Topographer::Importer::Mapper::MapperBuilder
+  include Topographer::Importer::Mapper::MappingColumns
+  include Topographer::Importer::Mapper::MappingValidator
+
+  attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
+              :validation_mappings, :default_values, :key_fields
+
+  def initialize
+    @required_mappings = {}
+    @optional_mappings = {}
+    @ignored_mappings = {}
+    @validation_mappings = {}
+    @default_values = {}
+    @key_fields = []
+  end
+
+  def required_mapping(input_columns, output_field, &mapping_behavior)
+    validate_unique_mapping(input_columns, output_field)
+    @required_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
+  end
+
+  def optional_mapping(input_columns, output_field, &mapping_behavior)
+    validate_unique_mapping(input_columns, output_field)
+    @optional_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
+  end
+
+  def validation_field(name, input_columns, &mapping_behavior)
+    validate_unique_validation_name(name)
+    @validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
+  end
+
+  def default_value(output_field, &mapping_behavior)
+    validate_unique_mapping([], output_field)
+    @default_values[output_field] = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
+  end
+
+  def key_field(output_field)
+    validate_key_field(output_field)
+    @key_fields << output_field
+  end
+
+  def ignored_column(input_column)
+    validate_unique_column_mapping_type(input_column, ignored: true)
+    @ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
+  end
+end

data/lib/Topographer/importer/mapper/mapping_columns.rb

@@ -0,0 +1,46 @@
+module Topographer::Importer::Mapper::MappingColumns
+  def output_fields
+    (required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
+  end
+
+  def required_input_columns
+    (required_mapping_columns + validation_mapping_columns).uniq
+  end
+
+  def input_columns
+    (required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
+  end
+
+  def default_fields
+    default_values.keys
+  end
+
+  def validation_mapping_columns
+    validation_mappings.values.flat_map(&:input_columns)
+  end
+
+  def ignored_mapping_columns
+    ignored_mappings.values.flat_map(&:input_columns)
+  end
+
+  def optional_mapping_columns
+    optional_mappings.values.flat_map(&:input_columns)
+  end
+
+  def required_mapping_columns
+    required_mappings.values.flat_map(&:input_columns)
+  end
+
+  private
+    def mapped_input_columns
+      required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
+    end
+
+    def mappings
+      required_mappings.merge(optional_mappings).merge(ignored_mappings).merge(default_values)
+    end
+
+    def non_ignored_columns
+      required_mappings.merge(optional_mappings)
+    end
+end

data/lib/Topographer/importer/mapper/mapping_validator.rb

@@ -0,0 +1,40 @@
+module Topographer::Importer::Mapper::MappingValidator
+
+  def validate_unique_validation_name(name)
+    raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
+  end
+
+  def validate_unique_output_mapping(output_field)
+    if output_fields.include?(output_field)
+      raise Topographer::InvalidMappingError, 'Output column already mapped.'
+    end
+  end
+
+  def validate_unique_column_mapping_type(mapping_input_columns, options = {})
+    ignored = options.fetch(:ignored, false)
+    mapping_input_columns = Array(mapping_input_columns)
+    mapping_input_columns.each do |col|
+      if ignored && ((input_columns + ignored_mapping_columns).include?(col))
+        raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
+      elsif (ignored_mapping_columns.include?(col))
+        raise Topographer::InvalidMappingError, 'Input column already ignored.'
+      end
+    end
+  end
+
+  def validate_unique_mapping(mapping_input_columns, output_field)
+    if (output_field.is_a?(Array))
+      raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
+    end
+    validate_unique_column_mapping_type(mapping_input_columns)
+    validate_unique_output_mapping(output_field)
+  end
+
+  def validate_key_field(field)
+    if field.is_a?(Array)
+      raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
+    elsif key_fields.include?(field)
+      raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
+    end
+  end
+end

data/lib/Topographer/importer/mapper.rb

@@ -1,161 +1,68 @@
 class Topographer::Importer::Mapper
+  require_relative 'mapper/mapping_validator'
+  require_relative 'mapper/mapping_columns'
+  require_relative 'mapper/mapper_builder'
   require_relative 'mapper/field_mapping'
   require_relative 'mapper/ignored_field_mapping'
   require_relative 'mapper/validation_field_mapping'
   require_relative 'mapper/default_field_mapping'
   require_relative 'mapper/result'
 
-  attr_reader :bad_columns, :missing_columns, :model_class, :key_fields
+  include Topographer::Importer::Mapper::MappingColumns
+
+  attr_reader :required_mappings, :optional_mappings, :ignored_mappings, :validation_mappings,
+              :default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
 
   def self.build_mapper(model_class)
-    mapper = self.new(model_class)
-    yield mapper
+    mapper_builder = MapperBuilder.new()
+    yield mapper_builder
 
-    mapper
+    new(mapper_builder, model_class)
   end
 
-  def initialize(model_class)
-    @required_mappings = {}
-    @optional_mappings = {}
-    @ignored_mappings = {}
-    @validation_mappings = {}
-    @default_values = {}
-    @key_fields = []
+  def initialize(mapper_builder, model_class)
+    @required_mappings = mapper_builder.required_mappings
+    @optional_mappings = mapper_builder.optional_mappings
+    @ignored_mappings = mapper_builder.ignored_mappings
+    @validation_mappings = mapper_builder.validation_mappings
+    @default_values = mapper_builder.default_values
+    @key_fields = mapper_builder.key_fields
     @model_class = model_class
   end
 
-  def required_columns
-    @required_mappings.values.flat_map(&:input_columns)
-  end
-
-  def optional_columns
-    @optional_mappings.values.flat_map(&:input_columns)
-  end
-
-  def ignored_columns
-    @ignored_mappings.values.flat_map(&:input_columns)
-  end
-
-  def validation_columns
-    @validation_mappings.values.flat_map(&:input_columns)
-  end
-
-  def default_fields
-    @default_values.keys
-  end
-
-  def input_columns
-    required_columns + optional_columns + validation_columns
-  end
-
-  def required_input_columns
-    required_columns + validation_columns
-  end
-
-  def output_fields
-    (@required_mappings.merge(@optional_mappings).merge(@default_values)).values.map(&:output_field)
-  end
-
-  def required_mapping(input_columns, output_field, &mapping_behavior)
-    validate_unique_mapping(input_columns, output_field)
-    @required_mappings[output_field] = FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
-  end
-
-  def optional_mapping(input_columns, output_field, &mapping_behavior)
-    validate_unique_mapping(input_columns, output_field)
-    @optional_mappings[output_field] = FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
-  end
-
-  def validation_field(name, input_columns, &mapping_behavior)
-    validate_unique_validation_name(name)
-    @validation_mappings[name] = ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
-  end
-
-  def default_value(output_field, &mapping_behavior)
-    validate_unique_mapping([], output_field)
-    @default_values[output_field] = DefaultFieldMapping.new(output_field, &mapping_behavior)
-  end
-
-  def key_field(output_field)
-    validate_key_field(output_field)
-    @key_fields << output_field
-  end
-
-  def ignored_column(input_column)
-    validate_unique_column_mapping_type(input_column, ignored: true)
-    @ignored_mappings[input_column] = IgnoredFieldMapping.new(input_column)
-  end
-
-  def input_structure_valid?(input_columns)
+  def input_structure_valid?(input_columns, options={})
+    ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
     @bad_columns ||= input_columns - mapped_input_columns
     @missing_columns ||= required_input_columns - input_columns
-    @bad_columns.empty? && @missing_columns.empty?
+
+    if ignore_unmapped_columns
+      @missing_columns.empty?
+    else
+      @bad_columns.empty? && @missing_columns.empty?
+    end
   end
 
   def map_input(source_data)
     mapping_result = Result.new(source_data.source_identifier)
 
-    @validation_mappings.values.each do |validation_field_mapping|
-      validation_field_mapping.process_input(source_data.data, mapping_result)
-    end
+    if source_data.empty?
+      handle_no_data(mapping_result)
+    else
+      @validation_mappings.values.each do |validation_field_mapping|
+        validation_field_mapping.process_input(source_data.data, mapping_result)
+      end
 
-    output_fields.each do |output_field|
-      field_mapping = mappings[output_field]
-      field_mapping.process_input(source_data.data, mapping_result)
+      output_fields.each do |output_field|
+        field_mapping = mappings[output_field]
+        field_mapping.process_input(source_data.data, mapping_result)
+      end
     end
 
     mapping_result
   end
 
   private
-    def mapped_input_columns
-      required_columns + optional_columns + ignored_columns + validation_columns
-    end
-
-    def mappings
-      @required_mappings.merge(@optional_mappings).merge(@ignored_mappings).merge(@default_values)
-    end
-
-    def non_ignored_columns
-      @required_mappings.merge(@optional_mappings)
-    end
-
-    def validate_key_field(field)
-      if field.is_a?(Array)
-        raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
-      elsif @key_fields.include?(field)
-        raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
-      end
-    end
-
-    def validate_unique_mapping(input_columns, output_field)
-      if(output_field.is_a?(Array))
-        raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
-      end
-      validate_unique_column_mapping_type(input_columns)
-      validate_unique_output_mapping(output_field)
-    end
-
-    def validate_unique_column_mapping_type(mapping_input_columns, options = {})
-      ignored = options.fetch(:ignored, false)
-      mapping_input_columns = Array(mapping_input_columns)
-      mapping_input_columns.each do |col|
-        if ignored && ((input_columns + ignored_columns).include?(col))
-          raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
-        elsif(ignored_columns.include?(col))
-          raise Topographer::InvalidMappingError, 'Input column already ignored.'
-        end
-      end
+    def handle_no_data(mapping_result)
+      mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
     end
-
-    def validate_unique_output_mapping(output_field)
-      if output_fields.include?(output_field)
-        raise Topographer::InvalidMappingError, 'Output column already mapped.'
-      end
-    end
-
-    def validate_unique_validation_name(name)
-      raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if @validation_mappings.has_key?(name)
-    end
-
 end

data/lib/Topographer/importer.rb

@@ -13,15 +13,18 @@ class Topographer::Importer
   end
 
   def self.import_data(input, import_class, strategy_class, logger, options = {})
-    dry_run = options.fetch(:dry_run, false)
-    importer = new(input, import_class, strategy_class, logger, dry_run)
+    importer = new(input, import_class, strategy_class, logger, options)
     importer.logger
   end
 
-  def initialize(input, import_class, strategy_class, logger, dry_run)
+  def initialize(input, import_class, strategy_class, logger, options = {})
     @logger = logger
+
+    dry_run = options.fetch(:dry_run, false)
+    ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
+
     mapper = import_class.get_mapper(strategy_class)
-    valid_header = mapper.input_structure_valid?(input.get_header)
+    valid_header = mapper.input_structure_valid?(input.get_header, ignore_unmapped_columns: ignore_unmapped_columns)
 
     if valid_header
       strategy = strategy_class.new(mapper)

data/lib/Topographer/version.rb

@@ -1,3 +1,3 @@
 module Topographer
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/{Cartographer → Topographer}/importer/importer_spec.rb

@@ -115,6 +115,17 @@ describe Topographer::Importer do
       expect(import_log.fatal_errors.first.message).
         to match(/Invalid Input Header.+Missing Columns:\s+Field2.+Invalid Columns:\s+BadCol1.+BadCol2/)
     end
+
+    it 'does import data with umapped columns when ignoring unmapped columns' do
+      extra_column_input = input
+      extra_column_input.stub(:get_header) { %w(Field1 Field2 Field3 UnknownField1) }
+
+      import_log = Topographer::Importer.import_data(extra_column_input, model_class, strategy_class, simple_logger, ignore_unmapped_columns: true)
+
+      expect(import_log.fatal_error?).to be_false
+      expect(import_log.total_imports).to be 4
+      expect(import_log.successful_imports).to be 3
+    end
   end
   describe '.build_mapper' do
     it 'returns a mapper with the defined mappings' do
@@ -124,9 +135,9 @@ describe Topographer::Importer do
           mapping.optional_mapping 'Field3', 'field_3'
           mapping.ignored_column 'IgnoredField'
       end
-      expect(mapper.required_columns).to eql(['Field1', 'Field2'])
-      expect(mapper.optional_columns).to eql(['Field3'])
-      expect(mapper.ignored_columns).to eql(['IgnoredField'])
+      expect(mapper.required_mapping_columns).to eql(['Field1', 'Field2'])
+      expect(mapper.optional_mapping_columns).to eql(['Field3'])
+      expect(mapper.ignored_mapping_columns).to eql(['IgnoredField'])
     end
   end
 end

data/spec/Topographer/importer/input/source_data_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Topographer::Importer::Input::SourceData do
+
+  let(:with_no_data) do
+    Topographer::Importer::Input::SourceData.new("foo", {})
+  end
+
+  let(:with_data) do
+    Topographer::Importer::Input::SourceData.new("foo", {bar: 'baz'})
+  end
+
+  describe '#empty?' do
+    it 'should return true if there is no data' do
+      expect(with_no_data.empty?).to be_true
+    end
+    it 'should return false if there is data' do
+      expect(with_data.empty?).to be_false
+    end
+  end
+
+end

data/spec/Topographer/importer/mapper/mapper_builder_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+
+describe Topographer::Importer::Mapper::MapperBuilder do
+
+  let(:builder) { Topographer::Importer::Mapper::MapperBuilder.new }
+
+  describe '#required_mapping' do
+    it 'should add a new simple required mapping' do
+      builder.required_mapping('input_column1', 'output_field1')
+      expect(builder.required_mappings.count).to eql(1)
+      expect(builder.required_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
+    end
+    it 'should add a new complex required mapping' do
+      builder.required_mapping 'input_column1', 'output_field1' do |inputs|
+        'foo'
+      end
+      expect(builder.required_mappings.count).to eql(1)
+      expect(builder.required_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
+    end
+    it 'should raise an error if the output field has already been mapped' do
+      builder.required_mapping('input_column1', 'output_field1')
+      expect{
+        builder.required_mapping('input_column2', 'output_field1')
+      }.to raise_error
+    end
+  end
+
+  describe '#optional_mapping' do
+    it 'should add a new simple optional mapping' do
+      builder.optional_mapping('input_column1', 'output_field1')
+      expect(builder.optional_mappings.count).to eql(1)
+      expect(builder.optional_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
+    end
+    it 'should add a new complex optional mapping' do
+      builder.optional_mapping 'input_column1', 'output_field1' do |inputs|
+        'foo'
+      end
+      expect(builder.optional_mappings.count).to eql(1)
+      expect(builder.optional_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
+    end
+    it 'should raise an error if the output field has already been mapped' do
+      builder.optional_mapping('input_column1', 'output_field1')
+      expect{
+        builder.optional_mapping('input_column2', 'output_field1')
+      }.to raise_error
+    end
+  end
+
+  describe '#validation_field' do
+    it 'should add a new validation field mapping' do
+      builder.validation_field('validation_1', 'input_column1') do
+        return true
+      end
+      expect(builder.validation_mappings.count).to eql(1)
+      expect(builder.validation_mappings['validation_1']).to be_an_instance_of(Topographer::Importer::Mapper::ValidationFieldMapping)
+    end
+    it 'should raise an error if there is no behavior block provided' do
+      expect {
+        builder.validation_field('validation_1', 'input_column1')
+      }.to raise_error
+    end
+    it 'should raise an error if a validation mapping already exists with a given name' do
+      builder.validation_field('validation_1', 'input_column1') do
+        return true
+      end
+      expect {
+        builder.validation_field('validation_1', 'input_column1') do
+          return true
+        end
+      }.to raise_error
+    end
+  end
+
+  describe '#default_value' do
+    it 'should add a new default value mapping' do
+      builder.default_value('output_field1') do
+        return true
+      end
+      expect(builder.default_values.count).to eql(1)
+      expect(builder.default_values['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::DefaultFieldMapping)
+    end
+    it 'should raise an error if there is no behavior block provided' do
+      expect {
+        builder.default_value('output_field1')
+      }.to raise_error
+    end
+    it 'should raise an error if a default value mapping already exists for a column' do
+      builder.default_value('output_field1') do
+        return true
+      end
+      expect {
+        builder.default_value('output_field1') do
+          return true
+        end
+      }.to raise_error
+    end
+  end
+
+  describe '#key_field' do
+    it 'should add a new key field mapping' do
+      builder.key_field('output_field1')
+      expect(builder.key_fields.count).to eql(1)
+      expect(builder.key_fields.first).to eql('output_field1')
+    end
+    it 'should raise an error if a default value mapping already exists for a column' do
+      builder.key_field('output_field1')
+      expect {
+
+        builder.key_field('output_field1')
+      }.to raise_error
+    end
+  end
+
+  describe '#ignored_column' do
+    it 'should add a new ignored column' do
+      builder.ignored_column('column1')
+      expect(builder.ignored_mappings.count).to eql(1)
+      expect(builder.ignored_mappings['column1']).to be_an_instance_of(Topographer::Importer::Mapper::IgnoredFieldMapping)
+    end
+    it 'should raise an error if the column has already been ignored or added as an input' do
+      builder.ignored_column('column1')
+      builder.required_mapping('column2', 'field1')
+      expect {
+        builder.ignored_column('column1')
+      }.to raise_error
+      expect {
+        builder.ignored_column('column2')
+      }.to raise_error
+    end
+  end
+end

data/spec/Topographer/importer/mapper/mapping_columns_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+require 'ostruct'
+
+class MockMapping
+  include Topographer::Importer::Mapper::MappingColumns
+
+  attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
+              :validation_mappings, :default_values, :key_fields
+
+
+  def initialize
+    @required_mappings = {
+      'field1' => OpenStruct.new(output_field: 'field1',
+                                 input_columns: ['column1', 'column2']
+      ),
+      'field2' => OpenStruct.new(output_field: 'field2',
+                                 input_columns: ['column3']
+      )
+    }
+    @optional_mappings = {
+      'field3' => OpenStruct.new(output_field: 'field3',
+        input_columns: ['column4', 'column5']
+      ),
+      'field4' => OpenStruct.new(output_field: 'field4',
+        input_columns: ['column8']
+      )
+    }
+    @default_values = {
+      'field5' => OpenStruct.new(output_field: 'field5')
+    }
+    @validation_mappings = {
+      'validation1' => OpenStruct.new(input_columns: ['column1', 'column2']),
+      'validation2' => OpenStruct.new(input_columns: 'column6')
+    }
+    @ignored_mappings = {
+      'column4' => OpenStruct.new(input_columns: 'column4'),
+      'column5' => OpenStruct.new(input_columns: 'column5')
+    }
+  end
+end
+
+describe Topographer::Importer::Mapper::MappingColumns do
+
+  let(:mapping) { MockMapping.new }
+
+  describe '#output_fields' do
+    it 'should return an array of all fields that are required, optional, or have default values' do
+      expect(mapping.output_fields).to eql(['field1', 'field2', 'field3', 'field4', 'field5'])
+    end
+  end
+  describe '#required_mapping_columns' do
+    it 'returns an array of the required input column names' do
+      expect(mapping.required_mapping_columns).to eql(['column1', 'column2', 'column3'])
+    end
+  end
+  describe '#optional_mapping_columns' do
+    it 'returns an array of the optional input column names' do
+      expect(mapping.optional_mapping_columns).to eql(['column4', 'column5', 'column8'])
+    end
+  end
+  describe '#ignored_mapping_columns' do
+    it 'returns an array of the ignored input column names' do
+      expect(mapping.ignored_mapping_columns).to eql(['column4', 'column5'])
+    end
+  end
+  describe '#validation_mapping_columns' do
+    it 'returns an array of the validation mapping input columns' do
+      expect(mapping.validation_mapping_columns).to eql(['column1', 'column2', 'column6'])
+    end
+  end
+  describe '#required_input_columns' do
+    it 'should return an array of the required and validation columns' do
+      expect(mapping.required_input_columns).to eql(['column1', 'column2', 'column3', 'column6'])
+    end
+  end
+  describe '#input_columns' do
+    it 'should return an array of the required, validation, and optional columns' do
+      expect(mapping.input_columns).to eql(['column1', 'column2', 'column3', 'column4', 'column5', 'column8', 'column6'])
+    end
+  end
+  describe '#default_fields' do
+    it 'should return all the fields given a default value' do
+      expect(mapping.default_fields).to eql(['field5'])
+    end
+  end
+
+end

data/spec/Topographer/importer/mapper/mapping_validator_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+class MockValidator
+  include Topographer::Importer::Mapper::MappingValidator
+
+  attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
+              :validation_mappings, :default_values, :key_fields,
+              :output_fields, :input_columns, :ignored_mapping_columns
+
+  def initialize
+    @validation_mappings = {
+      'Field1' => {},
+      'Field2' => {}
+    }
+    @output_fields = %w(output_field1 output_field2)
+    @input_columns = %w(input_column1 input_column2)
+    @ignored_mapping_columns = %w(ignored_column1 ignored_column2)
+    @key_fields = %w(key_field1)
+  end
+
+end
+
+describe Topographer::Importer::Mapper::MappingValidator do
+
+  let(:validator) do
+    MockValidator.new
+  end
+
+  describe '#validate_unique_validation_name' do
+    it 'should not raise an error if a name is not in the validation list' do
+      expect {
+        validator.validate_unique_validation_name('unique_field_name_37')
+      }.not_to raise_error
+    end
+    it 'should raise an error if a name is in the validation list' do
+      expect {
+        validator.validate_unique_validation_name('Field1')
+      }.to raise_error
+    end
+  end
+
+  describe '#validate_unique_output_mapping' do
+    it 'should not raise an error if a field is not already an output field' do
+      expect {
+        validator.validate_unique_output_mapping('unique_field_37')
+      }.not_to raise_error
+    end
+    it 'should raise an error if a field is already an output field' do
+      expect {
+        validator.validate_unique_output_mapping('output_field2')
+      }.to raise_error
+    end
+  end
+
+  describe '#validate_unique_column_mapping_type' do
+    it 'should not raise an error if a field is not already mapped' do
+      expect {
+        validator.validate_unique_column_mapping_type('input_column3')
+      }.not_to raise_error
+    end
+    it 'should not raise an error if a field is already mapped but is not ignored' do
+      expect {
+        validator.validate_unique_column_mapping_type('input_column2')
+      }.not_to raise_error
+    end
+    it 'should raise an error if a field is already ignored' do
+      expect {
+        validator.validate_unique_column_mapping_type('ignored_column1')
+      }.to raise_error
+    end
+    it 'should raise an error when validating an ignored column that has already been mapped' do
+      expect {
+        validator.validate_unique_column_mapping_type('input_column1', ignored: true)
+      }.to raise_error
+      expect {
+        validator.validate_unique_column_mapping_type('ignored_column1', ignored: true)
+      }.to raise_error
+    end
+  end
+
+  describe '#validate_unique_mapping' do
+    it 'should not raise an error if a new mapping has a unique output field' do
+      expect {
+        validator.validate_unique_mapping('input_column2', 'output_field3')
+      }.not_to raise_error
+    end
+    it 'should raise an error if a new mapping does not have a unique output field' do
+      expect {
+        validator.validate_unique_mapping('input_column2', 'output_field2')
+      }.to raise_error
+    end
+    it 'should raise an error if a new mapping includes an already ignored column' do
+      expect {
+        validator.validate_unique_mapping('ignored_column2', 'output_field2')
+      }.to raise_error
+    end
+  end
+  it 'should raise an error if a new mapping includes many output fields' do
+    expect {
+      validator.validate_unique_mapping('ignored_column2', %w(output_field2 output_field3))
+    }.to raise_error
+  end
+
+  describe '#validate_key_field' do
+    it 'should raise an error if a new key field includes multiple fields' do
+      expect {
+        validator.validate_key_field(%w(output_field2 output_field3))
+      }.to raise_error
+    end
+    it 'should raise an error if a key field is duplicated' do
+      expect {
+        validator.validate_key_field('key_field1')
+      }.to raise_error
+    end
+    it 'should not raise an error if a key field is not already mapped' do
+      expect {
+        validator.validate_key_field('key_field2')
+      }.not_to raise_error
+    end
+  end
+end

data/spec/{Cartographer → Topographer}/importer/mapper_spec.rb

@@ -7,14 +7,14 @@ describe Topographer::Importer::Mapper do
         mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
           m.required_mapping 'Field1', 'field_1'
         end
-        mapper.required_columns.should include("Field1")
+        mapper.required_mapping_columns.should include("Field1")
         mapper.output_fields.should include('field_1')
       end
       it 'can require a many to one field mapping' do
         mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
           m.required_mapping ['Field1', 'Field2'], 'field_1'
         end
-        mapper.required_columns.should include("Field1", "Field2")
+        mapper.required_mapping_columns.should include("Field1", "Field2")
         mapper.output_fields.should include('field_1')
       end
       it 'cannot require a one to many field mapping' do
@@ -29,14 +29,14 @@ describe Topographer::Importer::Mapper do
         mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
           m.optional_mapping 'Field1', 'field_1'
         end
-        mapper.optional_columns.should include("Field1")
+        mapper.optional_mapping_columns.should include("Field1")
         mapper.output_fields.should include('field_1')
       end
       it 'can create an optional many to one field mapping' do
         mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
           m.optional_mapping ['Field1', 'Field2'], 'field_1'
         end
-        mapper.optional_columns.should include("Field1", "Field2")
+        mapper.optional_mapping_columns.should include("Field1", "Field2")
         mapper.output_fields.should include('field_1')
       end
       it 'cannot create an optional one to many field mapping' do
@@ -51,7 +51,7 @@ describe Topographer::Importer::Mapper do
         mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
           m.ignored_column 'Field1'
         end
-        mapper.ignored_columns.should include('Field1')
+        mapper.ignored_mapping_columns.should include('Field1')
         mapper.output_fields.should be_empty
       end
 
@@ -84,7 +84,7 @@ describe Topographer::Importer::Mapper do
             raise 'No Input' unless input
           end
         end
-        expect(mapper.validation_columns).to include('Field1')
+        expect(mapper.validation_mapping_columns).to include('Field1')
         expect(mapper.output_fields.empty?).to be_true
       end
       it 'can create a multicolumn validation' do
@@ -93,7 +93,7 @@ describe Topographer::Importer::Mapper do
             raise 'No Input' unless input
           end
         end
-        expect(mapper.validation_columns).to eql(['Field1', 'Field2'])
+        expect(mapper.validation_mapping_columns).to eql(['Field1', 'Field2'])
         expect(mapper.output_fields.empty?).to be_true
       end
       it 'raises an error if a validation name is repeated' do
@@ -133,6 +133,32 @@ describe Topographer::Importer::Mapper do
           end
         end }.to raise_error(Topographer::InvalidMappingError)
       end
+
+
+    end
+
+    describe 'key field mappings' do
+      it 'should add a key field to the list of key fields' do
+        mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
+          m.key_field 'Field1'
+        end
+        expect(mapper.key_fields).to eql(['Field1'])
+      end
+      it 'should not allow multiple key fields at one time' do
+        expect {
+          mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
+            m.key_field ['Field1', 'Field2']
+          end
+        }.to raise_error(Topographer::InvalidMappingError)
+      end
+      it 'should not allow the same key field more than once' do
+        expect {
+          mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
+            m.key_field 'Field1'
+            m.key_field 'Field1'
+          end
+        }.to raise_error(Topographer::InvalidMappingError)
+      end
     end
 
     it 'associates the model class with the mapper instance' do
@@ -144,25 +170,6 @@ describe Topographer::Importer::Mapper do
     end
   end
 
-  describe '#key_field' do
-    let(:mapper) { Topographer::Importer::Mapper.new(Object) }
-    it 'should add a key field to the list of key fields' do
-      mapper.key_field 'Field1'
-      expect(mapper.key_fields).to eql(['Field1'])
-    end
-    it 'should not allow multiple key fields at one time' do
-      expect {
-        mapper.key_field ['Field1', 'Field2']
-      }.to raise_error(Topographer::InvalidMappingError)
-    end
-    it 'should not allow the same key field more than once' do
-      expect {
-        mapper.key_field 'Field1'
-        mapper.key_field 'Field1'
-      }.to raise_error(Topographer::InvalidMappingError)
-    end
-  end
-
   describe '#input_structure_valid?' do
     let(:mapper) do
       Topographer::Importer::Mapper.build_mapper(Object) do |m|
@@ -178,7 +185,8 @@ describe Topographer::Importer::Mapper do
     let(:valid_input_structure_without_options) { ['Field1', 'Field2', 'Field4'] }
     let(:missing_required_column_structure) { ['Field1', 'Field3'] }
     let(:missing_validation_column_structure) { ['Field1', 'Field2', 'Field3'] }
-    let(:bad_column_structure) { ['Field1', 'UnknownField', 'Field2', 'Field4'] }
+    let(:bad_column_structure) { ['Field1', 'UnknownField', 'Field4'] }
+    let(:unmapped_column_structure) {['Field1', 'Field2', 'Field3', 'Field4', 'UnmappedField'] }
 
     it 'returns false if required fields are missing' do
       expect(mapper.input_structure_valid?(missing_required_column_structure)).to be_false
@@ -196,8 +204,18 @@ describe Topographer::Importer::Mapper do
       expect(mapper.input_structure_valid?(valid_input_structure_with_options)).to be_true
       expect(mapper.input_structure_valid?(valid_input_structure_with_options+['Field5'])).to be_true
     end
-    it 'returns false if there are any extra fields that have not been ignored' do
-      mapper.input_structure_valid?(bad_column_structure).should be_false
+    context 'not ignoring unmapped columns' do
+      it 'returns false if there are any extra fields that have not been ignored' do
+        expect(mapper.input_structure_valid?(bad_column_structure)).to be_false
+      end
+    end
+    context 'ignoring unmapped columns' do
+      it 'returns false if there are any extra fields that have not been ignored and required fields are missing' do
+        expect(mapper.input_structure_valid?(bad_column_structure, ignore_unmapped_columns: true)).to be_false
+      end
+      it 'returns true if there are any extra fields that have not been ignored but all required fields are present' do
+        expect(mapper.input_structure_valid?(unmapped_column_structure, ignore_unmapped_columns: true)).to be_true
+      end
     end
   end
 
@@ -255,7 +273,8 @@ describe Topographer::Importer::Mapper do
              source_identifier: 'row1',
              data: {'Field1' => 'datum1',
                     'Field2' => 'datum2',
-                    'Field3' => 6}
+                    'Field3' => 6},
+             empty?: false
     end
 
     let(:invalid_data_input) do
@@ -263,14 +282,23 @@ describe Topographer::Importer::Mapper do
              source_identifier: 'row1',
              data: {'Field1' => 'datum1',
                     'Field2' => 'bad_field2_data',
-                    'Field3' => 6}
+                    'Field3' => 6},
+             empty?: false
     end
 
     let(:missing_field_input) do
       double 'SourceData',
              source_identifier: 'row1',
              data: {'Field1' => 'datum1',
-                    'Field2' => 'datum2'}
+                    'Field2' => 'datum2'},
+             empty?: false
+    end
+
+    let(:empty_input) do
+      double 'SourceData',
+             source_identifier: 'row1',
+             data: {},
+             empty?: true
     end
 
     let(:result) do
@@ -314,5 +342,11 @@ describe Topographer::Importer::Mapper do
     it 'does not return an error if an optional field is missing in the input data' do
       expect(result.errors?).to be_false
     end
+
+    it 'returns a `blank row` error if an entire row is blank' do
+      empty_result = mapper.map_input(empty_input)
+      expect(empty_result.errors?).to be_true
+      expect(empty_result.errors['EmptyRow']).to include('empty row')
+    end
   end
 end

data/spec/{Cartographer → Topographer}/importer/strategy/create_or_update_record_spec.rb

@@ -9,12 +9,14 @@ describe Topographer::Importer::Strategy::CreateOrUpdateRecord do
     double 'Data',
            source_identifier: 'record',
            data: {'Field1' => 'datum1',
-                  'Field2' => 'datum2'}
+                  'Field2' => 'datum2'},
+           empty?: false
   end
   let(:invalid_input) do
     double 'Data',
            source_identifier: 'bad record',
-           data: {'Field1' => 'datum1'}
+           data: {'Field1' => 'datum1'},
+           empty?: false
   end
 
 

data/spec/{Cartographer → Topographer}/importer/strategy/import_new_records_spec.rb

@@ -27,7 +27,8 @@ describe Topographer::Importer::Strategy::ImportNewRecord do
     double 'Data',
            source_identifier: 'record',
            data: {'Field1' => 'datum1',
-                  'Field2' => 'datum2'}
+                  'Field2' => 'datum2'},
+           empty?: false
   end
 
   describe '#initialize' do

data/spec/{Cartographer → Topographer}/importer/strategy/update_record_spec.rb

@@ -9,12 +9,14 @@ describe Topographer::Importer::Strategy::UpdateRecord do
     double 'Data',
            source_identifier: 'record',
            data: {'Field1' => 'datum1',
-                  'Field2' => 'datum2'}
+                  'Field2' => 'datum2'},
+           empty?: false
   end
   let(:invalid_input) do
     double 'Data',
            source_identifier: 'bad record',
-           data: {'Field1' => 'datum1'}
+           data: {'Field1' => 'datum1'},
+           empty?: false
   end
 
   describe '#import_record' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: topographer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Mike Simpson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-11 00:00:00.000000000 Z
+date: 2014-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -76,6 +76,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE
 - LICENSE.txt
@@ -100,6 +101,9 @@ files:
 - lib/Topographer/importer/mapper/default_field_mapping.rb
 - lib/Topographer/importer/mapper/field_mapping.rb
 - lib/Topographer/importer/mapper/ignored_field_mapping.rb
+- lib/Topographer/importer/mapper/mapper_builder.rb
+- lib/Topographer/importer/mapper/mapping_columns.rb
+- lib/Topographer/importer/mapper/mapping_validator.rb
 - lib/Topographer/importer/mapper/result.rb
 - lib/Topographer/importer/mapper/validation_field_mapping.rb
 - lib/Topographer/importer/strategy.rb
@@ -110,23 +114,27 @@ files:
 - lib/Topographer/importer/strategy/update_record.rb
 - lib/Topographer/version.rb
 - lib/topographer.rb
-- spec/Cartographer/importer/helpers/write_log_to_csv_spec.rb
-- spec/Cartographer/importer/helpers_spec.rb
-- spec/Cartographer/importer/importable_spec.rb
-- spec/Cartographer/importer/importer_spec.rb
-- spec/Cartographer/importer/logger/base_spec.rb
-- spec/Cartographer/importer/logger/fatal_error_entry_spec.rb
-- spec/Cartographer/importer/logger/simple_spec.rb
-- spec/Cartographer/importer/mapper/default_field_mapping_spec.rb
-- spec/Cartographer/importer/mapper/field_mapping_spec.rb
-- spec/Cartographer/importer/mapper/validation_field_mapping_spec.rb
-- spec/Cartographer/importer/mapper_spec.rb
-- spec/Cartographer/importer/strategy/base_spec.rb
-- spec/Cartographer/importer/strategy/create_or_update_record_spec.rb
-- spec/Cartographer/importer/strategy/import_new_records_spec.rb
-- spec/Cartographer/importer/strategy/import_status_spec.rb
-- spec/Cartographer/importer/strategy/mapped_model.rb
-- spec/Cartographer/importer/strategy/update_record_spec.rb
+- spec/Topographer/importer/helpers/write_log_to_csv_spec.rb
+- spec/Topographer/importer/helpers_spec.rb
+- spec/Topographer/importer/importable_spec.rb
+- spec/Topographer/importer/importer_spec.rb
+- spec/Topographer/importer/input/source_data_spec.rb
+- spec/Topographer/importer/logger/base_spec.rb
+- spec/Topographer/importer/logger/fatal_error_entry_spec.rb
+- spec/Topographer/importer/logger/simple_spec.rb
+- spec/Topographer/importer/mapper/default_field_mapping_spec.rb
+- spec/Topographer/importer/mapper/field_mapping_spec.rb
+- spec/Topographer/importer/mapper/mapper_builder_spec.rb
+- spec/Topographer/importer/mapper/mapping_columns_spec.rb
+- spec/Topographer/importer/mapper/mapping_validator_spec.rb
+- spec/Topographer/importer/mapper/validation_field_mapping_spec.rb
+- spec/Topographer/importer/mapper_spec.rb
+- spec/Topographer/importer/strategy/base_spec.rb
+- spec/Topographer/importer/strategy/create_or_update_record_spec.rb
+- spec/Topographer/importer/strategy/import_new_records_spec.rb
+- spec/Topographer/importer/strategy/import_status_spec.rb
+- spec/Topographer/importer/strategy/mapped_model.rb
+- spec/Topographer/importer/strategy/update_record_spec.rb
 - spec/spec_helper.rb
 - topographer.gemspec
 homepage: ''
@@ -155,22 +163,26 @@ specification_version: 4
 summary: Topographer allows the mapping of columnar input data to fields for active
   record models. This facilitates importing from a variety of sources.
 test_files:
-- spec/Cartographer/importer/helpers/write_log_to_csv_spec.rb
-- spec/Cartographer/importer/helpers_spec.rb
-- spec/Cartographer/importer/importable_spec.rb
-- spec/Cartographer/importer/importer_spec.rb
-- spec/Cartographer/importer/logger/base_spec.rb
-- spec/Cartographer/importer/logger/fatal_error_entry_spec.rb
-- spec/Cartographer/importer/logger/simple_spec.rb
-- spec/Cartographer/importer/mapper/default_field_mapping_spec.rb
-- spec/Cartographer/importer/mapper/field_mapping_spec.rb
-- spec/Cartographer/importer/mapper/validation_field_mapping_spec.rb
-- spec/Cartographer/importer/mapper_spec.rb
-- spec/Cartographer/importer/strategy/base_spec.rb
-- spec/Cartographer/importer/strategy/create_or_update_record_spec.rb
-- spec/Cartographer/importer/strategy/import_new_records_spec.rb
-- spec/Cartographer/importer/strategy/import_status_spec.rb
-- spec/Cartographer/importer/strategy/mapped_model.rb
-- spec/Cartographer/importer/strategy/update_record_spec.rb
+- spec/Topographer/importer/helpers/write_log_to_csv_spec.rb
+- spec/Topographer/importer/helpers_spec.rb
+- spec/Topographer/importer/importable_spec.rb
+- spec/Topographer/importer/importer_spec.rb
+- spec/Topographer/importer/input/source_data_spec.rb
+- spec/Topographer/importer/logger/base_spec.rb
+- spec/Topographer/importer/logger/fatal_error_entry_spec.rb
+- spec/Topographer/importer/logger/simple_spec.rb
+- spec/Topographer/importer/mapper/default_field_mapping_spec.rb
+- spec/Topographer/importer/mapper/field_mapping_spec.rb
+- spec/Topographer/importer/mapper/mapper_builder_spec.rb
+- spec/Topographer/importer/mapper/mapping_columns_spec.rb
+- spec/Topographer/importer/mapper/mapping_validator_spec.rb
+- spec/Topographer/importer/mapper/validation_field_mapping_spec.rb
+- spec/Topographer/importer/mapper_spec.rb
+- spec/Topographer/importer/strategy/base_spec.rb
+- spec/Topographer/importer/strategy/create_or_update_record_spec.rb
+- spec/Topographer/importer/strategy/import_new_records_spec.rb
+- spec/Topographer/importer/strategy/import_status_spec.rb
+- spec/Topographer/importer/strategy/mapped_model.rb
+- spec/Topographer/importer/strategy/update_record_spec.rb
 - spec/spec_helper.rb
 has_rdoc: