topographer 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/.gitignore +1 -0
- data/.travis.yml +5 -0
- data/README.md +1 -4
- data/lib/Topographer/importer/input/source_data.rb +4 -0
- data/lib/Topographer/importer/mapper/mapper_builder.rb +46 -0
- data/lib/Topographer/importer/mapper/mapping_columns.rb +46 -0
- data/lib/Topographer/importer/mapper/mapping_validator.rb +40 -0
- data/lib/Topographer/importer/mapper.rb +37 -130
- data/lib/Topographer/importer.rb +7 -4
- data/lib/Topographer/version.rb +1 -1
- data/spec/{Cartographer → Topographer}/importer/importer_spec.rb +14 -3
- data/spec/Topographer/importer/input/source_data_spec.rb +22 -0
- data/spec/Topographer/importer/mapper/mapper_builder_spec.rb +131 -0
- data/spec/Topographer/importer/mapper/mapping_columns_spec.rb +87 -0
- data/spec/Topographer/importer/mapper/mapping_validator_spec.rb +121 -0
- data/spec/{Cartographer → Topographer}/importer/mapper_spec.rb +66 -32
- data/spec/{Cartographer → Topographer}/importer/strategy/create_or_update_record_spec.rb +4 -2
- data/spec/{Cartographer → Topographer}/importer/strategy/import_new_records_spec.rb +2 -1
- data/spec/{Cartographer → Topographer}/importer/strategy/update_record_spec.rb +4 -2
- metadata +48 -36
- /data/spec/{Cartographer → Topographer}/importer/helpers/write_log_to_csv_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/helpers_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/importable_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/logger/base_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/logger/fatal_error_entry_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/logger/simple_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/mapper/default_field_mapping_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/mapper/field_mapping_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/mapper/validation_field_mapping_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/strategy/base_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/strategy/import_status_spec.rb +0 -0
- /data/spec/{Cartographer → Topographer}/importer/strategy/mapped_model.rb +0 -0
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NDcwNmMxNWRlMjkzMDgzMzg0MTMxNjM1YjMzODhmMDRhZmYwNzFlMg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Mjg4YzFlYWNiZmY0YzFkOTNlNGQ0ZDE5YWFkYmE1Zjk3YTRhOGVkYw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MDQ1YzFkYTI2MmI2ZTA2YzY3MzYyOWY1YzAyYTUwY2VhYmU1ZWVjMzllN2Y3
|
10
|
+
ODRmODMxNTc2ZmUxMzI1NWFhMDg4NGFmYTZkNWE5YmIyMTRkNTUxNjQ4N2Y3
|
11
|
+
NGRkMDlhMGNjZTM5YmQ1MDczZGVkMzM5ODNmNmEwYzQ4ZTYxZDA=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MGZkODRiZGI0OWY0MTlkMjFhNGM2MTQ3YjA0YjYwNDZiZGU1YmExMmVkY2Ux
|
14
|
+
Y2IxZGM1ZjIyODA0ODU4YmZmMGQxNDFjNTM2YzJmYzUzZTEyM2Q4YjFlZjcx
|
15
|
+
MDJjZWMyZjlkZmQzMDlmMTdkNTc4ZTUyNjYyODk5N2IyYzVhYTk=
|
data/.gitignore
CHANGED
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Topographer
|
1
|
+
# Topographer [![Gem Version](https://badge.fury.io/rb/topographer.png)](http://badge.fury.io/rb/topographer) [![Build Status](https://travis-ci.org/SciMed/topographer.png?branch=master)](https://travis-ci.org/SciMed/topographer)
|
2
2
|
|
3
3
|
Topographer is a gem that provides functionality to conveniently import from various data sources into
|
4
4
|
ActiveRecord or other ORM systems. This accomplished by defining an input wrapper, a mapping from input data to
|
@@ -32,7 +32,4 @@ Check back soon for detailed usage instructions
|
|
32
32
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
33
33
|
4. Push to the branch (`git push origin my-new-feature`)
|
34
34
|
5. Create new Pull Request
|
35
|
-
=======
|
36
|
-
Topographer
|
37
|
-
===========
|
38
35
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
class Topographer::Importer::Mapper::MapperBuilder
|
2
|
+
include Topographer::Importer::Mapper::MappingColumns
|
3
|
+
include Topographer::Importer::Mapper::MappingValidator
|
4
|
+
|
5
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
|
6
|
+
:validation_mappings, :default_values, :key_fields
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@required_mappings = {}
|
10
|
+
@optional_mappings = {}
|
11
|
+
@ignored_mappings = {}
|
12
|
+
@validation_mappings = {}
|
13
|
+
@default_values = {}
|
14
|
+
@key_fields = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def required_mapping(input_columns, output_field, &mapping_behavior)
|
18
|
+
validate_unique_mapping(input_columns, output_field)
|
19
|
+
@required_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
|
20
|
+
end
|
21
|
+
|
22
|
+
def optional_mapping(input_columns, output_field, &mapping_behavior)
|
23
|
+
validate_unique_mapping(input_columns, output_field)
|
24
|
+
@optional_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
|
25
|
+
end
|
26
|
+
|
27
|
+
def validation_field(name, input_columns, &mapping_behavior)
|
28
|
+
validate_unique_validation_name(name)
|
29
|
+
@validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
|
30
|
+
end
|
31
|
+
|
32
|
+
def default_value(output_field, &mapping_behavior)
|
33
|
+
validate_unique_mapping([], output_field)
|
34
|
+
@default_values[output_field] = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
|
35
|
+
end
|
36
|
+
|
37
|
+
def key_field(output_field)
|
38
|
+
validate_key_field(output_field)
|
39
|
+
@key_fields << output_field
|
40
|
+
end
|
41
|
+
|
42
|
+
def ignored_column(input_column)
|
43
|
+
validate_unique_column_mapping_type(input_column, ignored: true)
|
44
|
+
@ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Topographer::Importer::Mapper::MappingColumns
|
2
|
+
def output_fields
|
3
|
+
(required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
|
4
|
+
end
|
5
|
+
|
6
|
+
def required_input_columns
|
7
|
+
(required_mapping_columns + validation_mapping_columns).uniq
|
8
|
+
end
|
9
|
+
|
10
|
+
def input_columns
|
11
|
+
(required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
|
12
|
+
end
|
13
|
+
|
14
|
+
def default_fields
|
15
|
+
default_values.keys
|
16
|
+
end
|
17
|
+
|
18
|
+
def validation_mapping_columns
|
19
|
+
validation_mappings.values.flat_map(&:input_columns)
|
20
|
+
end
|
21
|
+
|
22
|
+
def ignored_mapping_columns
|
23
|
+
ignored_mappings.values.flat_map(&:input_columns)
|
24
|
+
end
|
25
|
+
|
26
|
+
def optional_mapping_columns
|
27
|
+
optional_mappings.values.flat_map(&:input_columns)
|
28
|
+
end
|
29
|
+
|
30
|
+
def required_mapping_columns
|
31
|
+
required_mappings.values.flat_map(&:input_columns)
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
def mapped_input_columns
|
36
|
+
required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
|
37
|
+
end
|
38
|
+
|
39
|
+
def mappings
|
40
|
+
required_mappings.merge(optional_mappings).merge(ignored_mappings).merge(default_values)
|
41
|
+
end
|
42
|
+
|
43
|
+
def non_ignored_columns
|
44
|
+
required_mappings.merge(optional_mappings)
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Topographer::Importer::Mapper::MappingValidator
|
2
|
+
|
3
|
+
def validate_unique_validation_name(name)
|
4
|
+
raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
|
5
|
+
end
|
6
|
+
|
7
|
+
def validate_unique_output_mapping(output_field)
|
8
|
+
if output_fields.include?(output_field)
|
9
|
+
raise Topographer::InvalidMappingError, 'Output column already mapped.'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def validate_unique_column_mapping_type(mapping_input_columns, options = {})
|
14
|
+
ignored = options.fetch(:ignored, false)
|
15
|
+
mapping_input_columns = Array(mapping_input_columns)
|
16
|
+
mapping_input_columns.each do |col|
|
17
|
+
if ignored && ((input_columns + ignored_mapping_columns).include?(col))
|
18
|
+
raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
|
19
|
+
elsif (ignored_mapping_columns.include?(col))
|
20
|
+
raise Topographer::InvalidMappingError, 'Input column already ignored.'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate_unique_mapping(mapping_input_columns, output_field)
|
26
|
+
if (output_field.is_a?(Array))
|
27
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
28
|
+
end
|
29
|
+
validate_unique_column_mapping_type(mapping_input_columns)
|
30
|
+
validate_unique_output_mapping(output_field)
|
31
|
+
end
|
32
|
+
|
33
|
+
def validate_key_field(field)
|
34
|
+
if field.is_a?(Array)
|
35
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
36
|
+
elsif key_fields.include?(field)
|
37
|
+
raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -1,161 +1,68 @@
|
|
1
1
|
class Topographer::Importer::Mapper
|
2
|
+
require_relative 'mapper/mapping_validator'
|
3
|
+
require_relative 'mapper/mapping_columns'
|
4
|
+
require_relative 'mapper/mapper_builder'
|
2
5
|
require_relative 'mapper/field_mapping'
|
3
6
|
require_relative 'mapper/ignored_field_mapping'
|
4
7
|
require_relative 'mapper/validation_field_mapping'
|
5
8
|
require_relative 'mapper/default_field_mapping'
|
6
9
|
require_relative 'mapper/result'
|
7
10
|
|
8
|
-
|
11
|
+
include Topographer::Importer::Mapper::MappingColumns
|
12
|
+
|
13
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings, :validation_mappings,
|
14
|
+
:default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
|
9
15
|
|
10
16
|
def self.build_mapper(model_class)
|
11
|
-
|
12
|
-
yield
|
17
|
+
mapper_builder = MapperBuilder.new()
|
18
|
+
yield mapper_builder
|
13
19
|
|
14
|
-
|
20
|
+
new(mapper_builder, model_class)
|
15
21
|
end
|
16
22
|
|
17
|
-
def initialize(model_class)
|
18
|
-
@required_mappings =
|
19
|
-
@optional_mappings =
|
20
|
-
@ignored_mappings =
|
21
|
-
@validation_mappings =
|
22
|
-
@default_values =
|
23
|
-
@key_fields =
|
23
|
+
def initialize(mapper_builder, model_class)
|
24
|
+
@required_mappings = mapper_builder.required_mappings
|
25
|
+
@optional_mappings = mapper_builder.optional_mappings
|
26
|
+
@ignored_mappings = mapper_builder.ignored_mappings
|
27
|
+
@validation_mappings = mapper_builder.validation_mappings
|
28
|
+
@default_values = mapper_builder.default_values
|
29
|
+
@key_fields = mapper_builder.key_fields
|
24
30
|
@model_class = model_class
|
25
31
|
end
|
26
32
|
|
27
|
-
def
|
28
|
-
|
29
|
-
end
|
30
|
-
|
31
|
-
def optional_columns
|
32
|
-
@optional_mappings.values.flat_map(&:input_columns)
|
33
|
-
end
|
34
|
-
|
35
|
-
def ignored_columns
|
36
|
-
@ignored_mappings.values.flat_map(&:input_columns)
|
37
|
-
end
|
38
|
-
|
39
|
-
def validation_columns
|
40
|
-
@validation_mappings.values.flat_map(&:input_columns)
|
41
|
-
end
|
42
|
-
|
43
|
-
def default_fields
|
44
|
-
@default_values.keys
|
45
|
-
end
|
46
|
-
|
47
|
-
def input_columns
|
48
|
-
required_columns + optional_columns + validation_columns
|
49
|
-
end
|
50
|
-
|
51
|
-
def required_input_columns
|
52
|
-
required_columns + validation_columns
|
53
|
-
end
|
54
|
-
|
55
|
-
def output_fields
|
56
|
-
(@required_mappings.merge(@optional_mappings).merge(@default_values)).values.map(&:output_field)
|
57
|
-
end
|
58
|
-
|
59
|
-
def required_mapping(input_columns, output_field, &mapping_behavior)
|
60
|
-
validate_unique_mapping(input_columns, output_field)
|
61
|
-
@required_mappings[output_field] = FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
|
62
|
-
end
|
63
|
-
|
64
|
-
def optional_mapping(input_columns, output_field, &mapping_behavior)
|
65
|
-
validate_unique_mapping(input_columns, output_field)
|
66
|
-
@optional_mappings[output_field] = FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
|
67
|
-
end
|
68
|
-
|
69
|
-
def validation_field(name, input_columns, &mapping_behavior)
|
70
|
-
validate_unique_validation_name(name)
|
71
|
-
@validation_mappings[name] = ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
|
72
|
-
end
|
73
|
-
|
74
|
-
def default_value(output_field, &mapping_behavior)
|
75
|
-
validate_unique_mapping([], output_field)
|
76
|
-
@default_values[output_field] = DefaultFieldMapping.new(output_field, &mapping_behavior)
|
77
|
-
end
|
78
|
-
|
79
|
-
def key_field(output_field)
|
80
|
-
validate_key_field(output_field)
|
81
|
-
@key_fields << output_field
|
82
|
-
end
|
83
|
-
|
84
|
-
def ignored_column(input_column)
|
85
|
-
validate_unique_column_mapping_type(input_column, ignored: true)
|
86
|
-
@ignored_mappings[input_column] = IgnoredFieldMapping.new(input_column)
|
87
|
-
end
|
88
|
-
|
89
|
-
def input_structure_valid?(input_columns)
|
33
|
+
def input_structure_valid?(input_columns, options={})
|
34
|
+
ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
|
90
35
|
@bad_columns ||= input_columns - mapped_input_columns
|
91
36
|
@missing_columns ||= required_input_columns - input_columns
|
92
|
-
|
37
|
+
|
38
|
+
if ignore_unmapped_columns
|
39
|
+
@missing_columns.empty?
|
40
|
+
else
|
41
|
+
@bad_columns.empty? && @missing_columns.empty?
|
42
|
+
end
|
93
43
|
end
|
94
44
|
|
95
45
|
def map_input(source_data)
|
96
46
|
mapping_result = Result.new(source_data.source_identifier)
|
97
47
|
|
98
|
-
|
99
|
-
|
100
|
-
|
48
|
+
if source_data.empty?
|
49
|
+
handle_no_data(mapping_result)
|
50
|
+
else
|
51
|
+
@validation_mappings.values.each do |validation_field_mapping|
|
52
|
+
validation_field_mapping.process_input(source_data.data, mapping_result)
|
53
|
+
end
|
101
54
|
|
102
|
-
|
103
|
-
|
104
|
-
|
55
|
+
output_fields.each do |output_field|
|
56
|
+
field_mapping = mappings[output_field]
|
57
|
+
field_mapping.process_input(source_data.data, mapping_result)
|
58
|
+
end
|
105
59
|
end
|
106
60
|
|
107
61
|
mapping_result
|
108
62
|
end
|
109
63
|
|
110
64
|
private
|
111
|
-
def
|
112
|
-
|
113
|
-
end
|
114
|
-
|
115
|
-
def mappings
|
116
|
-
@required_mappings.merge(@optional_mappings).merge(@ignored_mappings).merge(@default_values)
|
117
|
-
end
|
118
|
-
|
119
|
-
def non_ignored_columns
|
120
|
-
@required_mappings.merge(@optional_mappings)
|
121
|
-
end
|
122
|
-
|
123
|
-
def validate_key_field(field)
|
124
|
-
if field.is_a?(Array)
|
125
|
-
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
126
|
-
elsif @key_fields.include?(field)
|
127
|
-
raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
def validate_unique_mapping(input_columns, output_field)
|
132
|
-
if(output_field.is_a?(Array))
|
133
|
-
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
134
|
-
end
|
135
|
-
validate_unique_column_mapping_type(input_columns)
|
136
|
-
validate_unique_output_mapping(output_field)
|
137
|
-
end
|
138
|
-
|
139
|
-
def validate_unique_column_mapping_type(mapping_input_columns, options = {})
|
140
|
-
ignored = options.fetch(:ignored, false)
|
141
|
-
mapping_input_columns = Array(mapping_input_columns)
|
142
|
-
mapping_input_columns.each do |col|
|
143
|
-
if ignored && ((input_columns + ignored_columns).include?(col))
|
144
|
-
raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
|
145
|
-
elsif(ignored_columns.include?(col))
|
146
|
-
raise Topographer::InvalidMappingError, 'Input column already ignored.'
|
147
|
-
end
|
148
|
-
end
|
65
|
+
def handle_no_data(mapping_result)
|
66
|
+
mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
|
149
67
|
end
|
150
|
-
|
151
|
-
def validate_unique_output_mapping(output_field)
|
152
|
-
if output_fields.include?(output_field)
|
153
|
-
raise Topographer::InvalidMappingError, 'Output column already mapped.'
|
154
|
-
end
|
155
|
-
end
|
156
|
-
|
157
|
-
def validate_unique_validation_name(name)
|
158
|
-
raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if @validation_mappings.has_key?(name)
|
159
|
-
end
|
160
|
-
|
161
68
|
end
|
data/lib/Topographer/importer.rb
CHANGED
@@ -13,15 +13,18 @@ class Topographer::Importer
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def self.import_data(input, import_class, strategy_class, logger, options = {})
|
16
|
-
|
17
|
-
importer = new(input, import_class, strategy_class, logger, dry_run)
|
16
|
+
importer = new(input, import_class, strategy_class, logger, options)
|
18
17
|
importer.logger
|
19
18
|
end
|
20
19
|
|
21
|
-
def initialize(input, import_class, strategy_class, logger,
|
20
|
+
def initialize(input, import_class, strategy_class, logger, options = {})
|
22
21
|
@logger = logger
|
22
|
+
|
23
|
+
dry_run = options.fetch(:dry_run, false)
|
24
|
+
ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
|
25
|
+
|
23
26
|
mapper = import_class.get_mapper(strategy_class)
|
24
|
-
valid_header = mapper.input_structure_valid?(input.get_header)
|
27
|
+
valid_header = mapper.input_structure_valid?(input.get_header, ignore_unmapped_columns: ignore_unmapped_columns)
|
25
28
|
|
26
29
|
if valid_header
|
27
30
|
strategy = strategy_class.new(mapper)
|
data/lib/Topographer/version.rb
CHANGED
@@ -115,6 +115,17 @@ describe Topographer::Importer do
|
|
115
115
|
expect(import_log.fatal_errors.first.message).
|
116
116
|
to match(/Invalid Input Header.+Missing Columns:\s+Field2.+Invalid Columns:\s+BadCol1.+BadCol2/)
|
117
117
|
end
|
118
|
+
|
119
|
+
it 'does import data with umapped columns when ignoring unmapped columns' do
|
120
|
+
extra_column_input = input
|
121
|
+
extra_column_input.stub(:get_header) { %w(Field1 Field2 Field3 UnknownField1) }
|
122
|
+
|
123
|
+
import_log = Topographer::Importer.import_data(extra_column_input, model_class, strategy_class, simple_logger, ignore_unmapped_columns: true)
|
124
|
+
|
125
|
+
expect(import_log.fatal_error?).to be_false
|
126
|
+
expect(import_log.total_imports).to be 4
|
127
|
+
expect(import_log.successful_imports).to be 3
|
128
|
+
end
|
118
129
|
end
|
119
130
|
describe '.build_mapper' do
|
120
131
|
it 'returns a mapper with the defined mappings' do
|
@@ -124,9 +135,9 @@ describe Topographer::Importer do
|
|
124
135
|
mapping.optional_mapping 'Field3', 'field_3'
|
125
136
|
mapping.ignored_column 'IgnoredField'
|
126
137
|
end
|
127
|
-
expect(mapper.
|
128
|
-
expect(mapper.
|
129
|
-
expect(mapper.
|
138
|
+
expect(mapper.required_mapping_columns).to eql(['Field1', 'Field2'])
|
139
|
+
expect(mapper.optional_mapping_columns).to eql(['Field3'])
|
140
|
+
expect(mapper.ignored_mapping_columns).to eql(['IgnoredField'])
|
130
141
|
end
|
131
142
|
end
|
132
143
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Topographer::Importer::Input::SourceData do
|
4
|
+
|
5
|
+
let(:with_no_data) do
|
6
|
+
Topographer::Importer::Input::SourceData.new("foo", {})
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:with_data) do
|
10
|
+
Topographer::Importer::Input::SourceData.new("foo", {bar: 'baz'})
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#empty?' do
|
14
|
+
it 'should return true if there is no data' do
|
15
|
+
expect(with_no_data.empty?).to be_true
|
16
|
+
end
|
17
|
+
it 'should return false if there is data' do
|
18
|
+
expect(with_data.empty?).to be_false
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Topographer::Importer::Mapper::MapperBuilder do
|
4
|
+
|
5
|
+
let(:builder) { Topographer::Importer::Mapper::MapperBuilder.new }
|
6
|
+
|
7
|
+
describe '#required_mapping' do
|
8
|
+
it 'should add a new simple required mapping' do
|
9
|
+
builder.required_mapping('input_column1', 'output_field1')
|
10
|
+
expect(builder.required_mappings.count).to eql(1)
|
11
|
+
expect(builder.required_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
|
12
|
+
end
|
13
|
+
it 'should add a new complex required mapping' do
|
14
|
+
builder.required_mapping 'input_column1', 'output_field1' do |inputs|
|
15
|
+
'foo'
|
16
|
+
end
|
17
|
+
expect(builder.required_mappings.count).to eql(1)
|
18
|
+
expect(builder.required_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
|
19
|
+
end
|
20
|
+
it 'should raise an error if the output field has already been mapped' do
|
21
|
+
builder.required_mapping('input_column1', 'output_field1')
|
22
|
+
expect{
|
23
|
+
builder.required_mapping('input_column2', 'output_field1')
|
24
|
+
}.to raise_error
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
describe '#optional_mapping' do
|
29
|
+
it 'should add a new simple optional mapping' do
|
30
|
+
builder.optional_mapping('input_column1', 'output_field1')
|
31
|
+
expect(builder.optional_mappings.count).to eql(1)
|
32
|
+
expect(builder.optional_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
|
33
|
+
end
|
34
|
+
it 'should add a new complex optional mapping' do
|
35
|
+
builder.optional_mapping 'input_column1', 'output_field1' do |inputs|
|
36
|
+
'foo'
|
37
|
+
end
|
38
|
+
expect(builder.optional_mappings.count).to eql(1)
|
39
|
+
expect(builder.optional_mappings['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::FieldMapping)
|
40
|
+
end
|
41
|
+
it 'should raise an error if the output field has already been mapped' do
|
42
|
+
builder.optional_mapping('input_column1', 'output_field1')
|
43
|
+
expect{
|
44
|
+
builder.optional_mapping('input_column2', 'output_field1')
|
45
|
+
}.to raise_error
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe '#validation_field' do
|
50
|
+
it 'should add a new validation field mapping' do
|
51
|
+
builder.validation_field('validation_1', 'input_column1') do
|
52
|
+
return true
|
53
|
+
end
|
54
|
+
expect(builder.validation_mappings.count).to eql(1)
|
55
|
+
expect(builder.validation_mappings['validation_1']).to be_an_instance_of(Topographer::Importer::Mapper::ValidationFieldMapping)
|
56
|
+
end
|
57
|
+
it 'should raise an error if there is no behavior block provided' do
|
58
|
+
expect {
|
59
|
+
builder.validation_field('validation_1', 'input_column1')
|
60
|
+
}.to raise_error
|
61
|
+
end
|
62
|
+
it 'should raise an error if a validation mapping already exists with a given name' do
|
63
|
+
builder.validation_field('validation_1', 'input_column1') do
|
64
|
+
return true
|
65
|
+
end
|
66
|
+
expect {
|
67
|
+
builder.validation_field('validation_1', 'input_column1') do
|
68
|
+
return true
|
69
|
+
end
|
70
|
+
}.to raise_error
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
describe '#default_value' do
|
75
|
+
it 'should add a new default value mapping' do
|
76
|
+
builder.default_value('output_field1') do
|
77
|
+
return true
|
78
|
+
end
|
79
|
+
expect(builder.default_values.count).to eql(1)
|
80
|
+
expect(builder.default_values['output_field1']).to be_an_instance_of(Topographer::Importer::Mapper::DefaultFieldMapping)
|
81
|
+
end
|
82
|
+
it 'should raise an error if there is no behavior block provided' do
|
83
|
+
expect {
|
84
|
+
builder.default_value('output_field1')
|
85
|
+
}.to raise_error
|
86
|
+
end
|
87
|
+
it 'should raise an error if a default value mapping already exists for a column' do
|
88
|
+
builder.default_value('output_field1') do
|
89
|
+
return true
|
90
|
+
end
|
91
|
+
expect {
|
92
|
+
builder.default_value('output_field1') do
|
93
|
+
return true
|
94
|
+
end
|
95
|
+
}.to raise_error
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
describe '#key_field' do
|
100
|
+
it 'should add a new key field mapping' do
|
101
|
+
builder.key_field('output_field1')
|
102
|
+
expect(builder.key_fields.count).to eql(1)
|
103
|
+
expect(builder.key_fields.first).to eql('output_field1')
|
104
|
+
end
|
105
|
+
it 'should raise an error if a default value mapping already exists for a column' do
|
106
|
+
builder.key_field('output_field1')
|
107
|
+
expect {
|
108
|
+
|
109
|
+
builder.key_field('output_field1')
|
110
|
+
}.to raise_error
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
describe '#ignored_column' do
|
115
|
+
it 'should add a new ignored column' do
|
116
|
+
builder.ignored_column('column1')
|
117
|
+
expect(builder.ignored_mappings.count).to eql(1)
|
118
|
+
expect(builder.ignored_mappings['column1']).to be_an_instance_of(Topographer::Importer::Mapper::IgnoredFieldMapping)
|
119
|
+
end
|
120
|
+
it 'should raise an error if the column has already been ignored or added as an input' do
|
121
|
+
builder.ignored_column('column1')
|
122
|
+
builder.required_mapping('column2', 'field1')
|
123
|
+
expect {
|
124
|
+
builder.ignored_column('column1')
|
125
|
+
}.to raise_error
|
126
|
+
expect {
|
127
|
+
builder.ignored_column('column2')
|
128
|
+
}.to raise_error
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ostruct'
|
3
|
+
|
4
|
+
class MockMapping
|
5
|
+
include Topographer::Importer::Mapper::MappingColumns
|
6
|
+
|
7
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
|
8
|
+
:validation_mappings, :default_values, :key_fields
|
9
|
+
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@required_mappings = {
|
13
|
+
'field1' => OpenStruct.new(output_field: 'field1',
|
14
|
+
input_columns: ['column1', 'column2']
|
15
|
+
),
|
16
|
+
'field2' => OpenStruct.new(output_field: 'field2',
|
17
|
+
input_columns: ['column3']
|
18
|
+
)
|
19
|
+
}
|
20
|
+
@optional_mappings = {
|
21
|
+
'field3' => OpenStruct.new(output_field: 'field3',
|
22
|
+
input_columns: ['column4', 'column5']
|
23
|
+
),
|
24
|
+
'field4' => OpenStruct.new(output_field: 'field4',
|
25
|
+
input_columns: ['column8']
|
26
|
+
)
|
27
|
+
}
|
28
|
+
@default_values = {
|
29
|
+
'field5' => OpenStruct.new(output_field: 'field5')
|
30
|
+
}
|
31
|
+
@validation_mappings = {
|
32
|
+
'validation1' => OpenStruct.new(input_columns: ['column1', 'column2']),
|
33
|
+
'validation2' => OpenStruct.new(input_columns: 'column6')
|
34
|
+
}
|
35
|
+
@ignored_mappings = {
|
36
|
+
'column4' => OpenStruct.new(input_columns: 'column4'),
|
37
|
+
'column5' => OpenStruct.new(input_columns: 'column5')
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe Topographer::Importer::Mapper::MappingColumns do
|
43
|
+
|
44
|
+
let(:mapping) { MockMapping.new }
|
45
|
+
|
46
|
+
describe '#output_fields' do
|
47
|
+
it 'should return an array of all fields that are required, optional, or have default values' do
|
48
|
+
expect(mapping.output_fields).to eql(['field1', 'field2', 'field3', 'field4', 'field5'])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
describe '#required_mapping_columns' do
|
52
|
+
it 'returns an array of the required input column names' do
|
53
|
+
expect(mapping.required_mapping_columns).to eql(['column1', 'column2', 'column3'])
|
54
|
+
end
|
55
|
+
end
|
56
|
+
describe '#optional_mapping_columns' do
|
57
|
+
it 'returns an array of the optional input column names' do
|
58
|
+
expect(mapping.optional_mapping_columns).to eql(['column4', 'column5', 'column8'])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
describe '#ignored_mapping_columns' do
|
62
|
+
it 'returns an array of the ignored input column names' do
|
63
|
+
expect(mapping.ignored_mapping_columns).to eql(['column4', 'column5'])
|
64
|
+
end
|
65
|
+
end
|
66
|
+
describe '#validation_mapping_columns' do
|
67
|
+
it 'returns an array of the validation mapping input columns' do
|
68
|
+
expect(mapping.validation_mapping_columns).to eql(['column1', 'column2', 'column6'])
|
69
|
+
end
|
70
|
+
end
|
71
|
+
describe '#required_input_columns' do
|
72
|
+
it 'should return an array of the required and validation columns' do
|
73
|
+
expect(mapping.required_input_columns).to eql(['column1', 'column2', 'column3', 'column6'])
|
74
|
+
end
|
75
|
+
end
|
76
|
+
describe '#input_columns' do
|
77
|
+
it 'should return an array of the required, validation, and optional columns' do
|
78
|
+
expect(mapping.input_columns).to eql(['column1', 'column2', 'column3', 'column4', 'column5', 'column8', 'column6'])
|
79
|
+
end
|
80
|
+
end
|
81
|
+
describe '#default_fields' do
|
82
|
+
it 'should return all the fields given a default value' do
|
83
|
+
expect(mapping.default_fields).to eql(['field5'])
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class MockValidator
|
4
|
+
include Topographer::Importer::Mapper::MappingValidator
|
5
|
+
|
6
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
|
7
|
+
:validation_mappings, :default_values, :key_fields,
|
8
|
+
:output_fields, :input_columns, :ignored_mapping_columns
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@validation_mappings = {
|
12
|
+
'Field1' => {},
|
13
|
+
'Field2' => {}
|
14
|
+
}
|
15
|
+
@output_fields = %w(output_field1 output_field2)
|
16
|
+
@input_columns = %w(input_column1 input_column2)
|
17
|
+
@ignored_mapping_columns = %w(ignored_column1 ignored_column2)
|
18
|
+
@key_fields = %w(key_field1)
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
describe Topographer::Importer::Mapper::MappingValidator do
|
24
|
+
|
25
|
+
let(:validator) do
|
26
|
+
MockValidator.new
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '#validate_unique_validation_name' do
|
30
|
+
it 'should not raise an error if a name is not in the validation list' do
|
31
|
+
expect {
|
32
|
+
validator.validate_unique_validation_name('unique_field_name_37')
|
33
|
+
}.not_to raise_error
|
34
|
+
end
|
35
|
+
it 'should raise an error if a name is in the validation list' do
|
36
|
+
expect {
|
37
|
+
validator.validate_unique_validation_name('Field1')
|
38
|
+
}.to raise_error
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe '#validate_unique_output_mapping' do
|
43
|
+
it 'should not raise an error if a field is not already an output field' do
|
44
|
+
expect {
|
45
|
+
validator.validate_unique_output_mapping('unique_field_37')
|
46
|
+
}.not_to raise_error
|
47
|
+
end
|
48
|
+
it 'should raise an error if a field is already an output field' do
|
49
|
+
expect {
|
50
|
+
validator.validate_unique_output_mapping('output_field2')
|
51
|
+
}.to raise_error
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '#validate_unique_column_mapping_type' do
|
56
|
+
it 'should not raise an error if a field is not already mapped' do
|
57
|
+
expect {
|
58
|
+
validator.validate_unique_column_mapping_type('input_column3')
|
59
|
+
}.not_to raise_error
|
60
|
+
end
|
61
|
+
it 'should not raise an error if a field is already mapped but is not ignored' do
|
62
|
+
expect {
|
63
|
+
validator.validate_unique_column_mapping_type('input_column2')
|
64
|
+
}.not_to raise_error
|
65
|
+
end
|
66
|
+
it 'should raise an error if a field is already ignored' do
|
67
|
+
expect {
|
68
|
+
validator.validate_unique_column_mapping_type('ignored_column1')
|
69
|
+
}.to raise_error
|
70
|
+
end
|
71
|
+
it 'should raise an error when validating an ignored column that has already been mapped' do
|
72
|
+
expect {
|
73
|
+
validator.validate_unique_column_mapping_type('input_column1', ignored: true)
|
74
|
+
}.to raise_error
|
75
|
+
expect {
|
76
|
+
validator.validate_unique_column_mapping_type('ignored_column1', ignored: true)
|
77
|
+
}.to raise_error
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe '#validate_unique_mapping' do
|
82
|
+
it 'should not raise an error if a new mapping has a unique output field' do
|
83
|
+
expect {
|
84
|
+
validator.validate_unique_mapping('input_column2', 'output_field3')
|
85
|
+
}.not_to raise_error
|
86
|
+
end
|
87
|
+
it 'should raise an error if a new mapping does not have a unique output field' do
|
88
|
+
expect {
|
89
|
+
validator.validate_unique_mapping('input_column2', 'output_field2')
|
90
|
+
}.to raise_error
|
91
|
+
end
|
92
|
+
it 'should raise an error if a new mapping includes an already ignored column' do
|
93
|
+
expect {
|
94
|
+
validator.validate_unique_mapping('ignored_column2', 'output_field2')
|
95
|
+
}.to raise_error
|
96
|
+
end
|
97
|
+
end
|
98
|
+
it 'should raise an error if a new mapping includes many output fields' do
|
99
|
+
expect {
|
100
|
+
validator.validate_unique_mapping('ignored_column2', %w(output_field2 output_field3))
|
101
|
+
}.to raise_error
|
102
|
+
end
|
103
|
+
|
104
|
+
describe '#validate_key_field' do
|
105
|
+
it 'should raise an error if a new key field includes multiple fields' do
|
106
|
+
expect {
|
107
|
+
validator.validate_key_field(%w(output_field2 output_field3))
|
108
|
+
}.to raise_error
|
109
|
+
end
|
110
|
+
it 'should raise an error if a key field is duplicated' do
|
111
|
+
expect {
|
112
|
+
validator.validate_key_field('key_field1')
|
113
|
+
}.to raise_error
|
114
|
+
end
|
115
|
+
it 'should not raise an error if a key field is not already mapped' do
|
116
|
+
expect {
|
117
|
+
validator.validate_key_field('key_field2')
|
118
|
+
}.not_to raise_error
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -7,14 +7,14 @@ describe Topographer::Importer::Mapper do
|
|
7
7
|
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
8
8
|
m.required_mapping 'Field1', 'field_1'
|
9
9
|
end
|
10
|
-
mapper.
|
10
|
+
mapper.required_mapping_columns.should include("Field1")
|
11
11
|
mapper.output_fields.should include('field_1')
|
12
12
|
end
|
13
13
|
it 'can require a many to one field mapping' do
|
14
14
|
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
15
15
|
m.required_mapping ['Field1', 'Field2'], 'field_1'
|
16
16
|
end
|
17
|
-
mapper.
|
17
|
+
mapper.required_mapping_columns.should include("Field1", "Field2")
|
18
18
|
mapper.output_fields.should include('field_1')
|
19
19
|
end
|
20
20
|
it 'cannot require a one to many field mapping' do
|
@@ -29,14 +29,14 @@ describe Topographer::Importer::Mapper do
|
|
29
29
|
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
30
30
|
m.optional_mapping 'Field1', 'field_1'
|
31
31
|
end
|
32
|
-
mapper.
|
32
|
+
mapper.optional_mapping_columns.should include("Field1")
|
33
33
|
mapper.output_fields.should include('field_1')
|
34
34
|
end
|
35
35
|
it 'can create an optional many to one field mapping' do
|
36
36
|
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
37
37
|
m.optional_mapping ['Field1', 'Field2'], 'field_1'
|
38
38
|
end
|
39
|
-
mapper.
|
39
|
+
mapper.optional_mapping_columns.should include("Field1", "Field2")
|
40
40
|
mapper.output_fields.should include('field_1')
|
41
41
|
end
|
42
42
|
it 'cannot create an optional one to many field mapping' do
|
@@ -51,7 +51,7 @@ describe Topographer::Importer::Mapper do
|
|
51
51
|
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
52
52
|
m.ignored_column 'Field1'
|
53
53
|
end
|
54
|
-
mapper.
|
54
|
+
mapper.ignored_mapping_columns.should include('Field1')
|
55
55
|
mapper.output_fields.should be_empty
|
56
56
|
end
|
57
57
|
|
@@ -84,7 +84,7 @@ describe Topographer::Importer::Mapper do
|
|
84
84
|
raise 'No Input' unless input
|
85
85
|
end
|
86
86
|
end
|
87
|
-
expect(mapper.
|
87
|
+
expect(mapper.validation_mapping_columns).to include('Field1')
|
88
88
|
expect(mapper.output_fields.empty?).to be_true
|
89
89
|
end
|
90
90
|
it 'can create a multicolumn validation' do
|
@@ -93,7 +93,7 @@ describe Topographer::Importer::Mapper do
|
|
93
93
|
raise 'No Input' unless input
|
94
94
|
end
|
95
95
|
end
|
96
|
-
expect(mapper.
|
96
|
+
expect(mapper.validation_mapping_columns).to eql(['Field1', 'Field2'])
|
97
97
|
expect(mapper.output_fields.empty?).to be_true
|
98
98
|
end
|
99
99
|
it 'raises an error if a validation name is repeated' do
|
@@ -133,6 +133,32 @@ describe Topographer::Importer::Mapper do
|
|
133
133
|
end
|
134
134
|
end }.to raise_error(Topographer::InvalidMappingError)
|
135
135
|
end
|
136
|
+
|
137
|
+
|
138
|
+
end
|
139
|
+
|
140
|
+
describe 'key field mappings' do
|
141
|
+
it 'should add a key field to the list of key fields' do
|
142
|
+
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
143
|
+
m.key_field 'Field1'
|
144
|
+
end
|
145
|
+
expect(mapper.key_fields).to eql(['Field1'])
|
146
|
+
end
|
147
|
+
it 'should not allow multiple key fields at one time' do
|
148
|
+
expect {
|
149
|
+
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
150
|
+
m.key_field ['Field1', 'Field2']
|
151
|
+
end
|
152
|
+
}.to raise_error(Topographer::InvalidMappingError)
|
153
|
+
end
|
154
|
+
it 'should not allow the same key field more than once' do
|
155
|
+
expect {
|
156
|
+
mapper = Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
157
|
+
m.key_field 'Field1'
|
158
|
+
m.key_field 'Field1'
|
159
|
+
end
|
160
|
+
}.to raise_error(Topographer::InvalidMappingError)
|
161
|
+
end
|
136
162
|
end
|
137
163
|
|
138
164
|
it 'associates the model class with the mapper instance' do
|
@@ -144,25 +170,6 @@ describe Topographer::Importer::Mapper do
|
|
144
170
|
end
|
145
171
|
end
|
146
172
|
|
147
|
-
describe '#key_field' do
|
148
|
-
let(:mapper) { Topographer::Importer::Mapper.new(Object) }
|
149
|
-
it 'should add a key field to the list of key fields' do
|
150
|
-
mapper.key_field 'Field1'
|
151
|
-
expect(mapper.key_fields).to eql(['Field1'])
|
152
|
-
end
|
153
|
-
it 'should not allow multiple key fields at one time' do
|
154
|
-
expect {
|
155
|
-
mapper.key_field ['Field1', 'Field2']
|
156
|
-
}.to raise_error(Topographer::InvalidMappingError)
|
157
|
-
end
|
158
|
-
it 'should not allow the same key field more than once' do
|
159
|
-
expect {
|
160
|
-
mapper.key_field 'Field1'
|
161
|
-
mapper.key_field 'Field1'
|
162
|
-
}.to raise_error(Topographer::InvalidMappingError)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
173
|
describe '#input_structure_valid?' do
|
167
174
|
let(:mapper) do
|
168
175
|
Topographer::Importer::Mapper.build_mapper(Object) do |m|
|
@@ -178,7 +185,8 @@ describe Topographer::Importer::Mapper do
|
|
178
185
|
let(:valid_input_structure_without_options) { ['Field1', 'Field2', 'Field4'] }
|
179
186
|
let(:missing_required_column_structure) { ['Field1', 'Field3'] }
|
180
187
|
let(:missing_validation_column_structure) { ['Field1', 'Field2', 'Field3'] }
|
181
|
-
let(:bad_column_structure) { ['Field1', 'UnknownField', '
|
188
|
+
let(:bad_column_structure) { ['Field1', 'UnknownField', 'Field4'] }
|
189
|
+
let(:unmapped_column_structure) {['Field1', 'Field2', 'Field3', 'Field4', 'UnmappedField'] }
|
182
190
|
|
183
191
|
it 'returns false if required fields are missing' do
|
184
192
|
expect(mapper.input_structure_valid?(missing_required_column_structure)).to be_false
|
@@ -196,8 +204,18 @@ describe Topographer::Importer::Mapper do
|
|
196
204
|
expect(mapper.input_structure_valid?(valid_input_structure_with_options)).to be_true
|
197
205
|
expect(mapper.input_structure_valid?(valid_input_structure_with_options+['Field5'])).to be_true
|
198
206
|
end
|
199
|
-
|
200
|
-
|
207
|
+
context 'not ignoring unmapped columns' do
|
208
|
+
it 'returns false if there are any extra fields that have not been ignored' do
|
209
|
+
expect(mapper.input_structure_valid?(bad_column_structure)).to be_false
|
210
|
+
end
|
211
|
+
end
|
212
|
+
context 'ignoring unmapped columns' do
|
213
|
+
it 'returns false if there are any extra fields that have not been ignored and required fields are missing' do
|
214
|
+
expect(mapper.input_structure_valid?(bad_column_structure, ignore_unmapped_columns: true)).to be_false
|
215
|
+
end
|
216
|
+
it 'returns true if there are any extra fields that have not been ignored but all required fields are present' do
|
217
|
+
expect(mapper.input_structure_valid?(unmapped_column_structure, ignore_unmapped_columns: true)).to be_true
|
218
|
+
end
|
201
219
|
end
|
202
220
|
end
|
203
221
|
|
@@ -255,7 +273,8 @@ describe Topographer::Importer::Mapper do
|
|
255
273
|
source_identifier: 'row1',
|
256
274
|
data: {'Field1' => 'datum1',
|
257
275
|
'Field2' => 'datum2',
|
258
|
-
'Field3' => 6}
|
276
|
+
'Field3' => 6},
|
277
|
+
empty?: false
|
259
278
|
end
|
260
279
|
|
261
280
|
let(:invalid_data_input) do
|
@@ -263,14 +282,23 @@ describe Topographer::Importer::Mapper do
|
|
263
282
|
source_identifier: 'row1',
|
264
283
|
data: {'Field1' => 'datum1',
|
265
284
|
'Field2' => 'bad_field2_data',
|
266
|
-
'Field3' => 6}
|
285
|
+
'Field3' => 6},
|
286
|
+
empty?: false
|
267
287
|
end
|
268
288
|
|
269
289
|
let(:missing_field_input) do
|
270
290
|
double 'SourceData',
|
271
291
|
source_identifier: 'row1',
|
272
292
|
data: {'Field1' => 'datum1',
|
273
|
-
'Field2' => 'datum2'}
|
293
|
+
'Field2' => 'datum2'},
|
294
|
+
empty?: false
|
295
|
+
end
|
296
|
+
|
297
|
+
let(:empty_input) do
|
298
|
+
double 'SourceData',
|
299
|
+
source_identifier: 'row1',
|
300
|
+
data: {},
|
301
|
+
empty?: true
|
274
302
|
end
|
275
303
|
|
276
304
|
let(:result) do
|
@@ -314,5 +342,11 @@ describe Topographer::Importer::Mapper do
|
|
314
342
|
it 'does not return an error if an optional field is missing in the input data' do
|
315
343
|
expect(result.errors?).to be_false
|
316
344
|
end
|
345
|
+
|
346
|
+
it 'returns a `blank row` error if an entire row is blank' do
|
347
|
+
empty_result = mapper.map_input(empty_input)
|
348
|
+
expect(empty_result.errors?).to be_true
|
349
|
+
expect(empty_result.errors['EmptyRow']).to include('empty row')
|
350
|
+
end
|
317
351
|
end
|
318
352
|
end
|
@@ -9,12 +9,14 @@ describe Topographer::Importer::Strategy::CreateOrUpdateRecord do
|
|
9
9
|
double 'Data',
|
10
10
|
source_identifier: 'record',
|
11
11
|
data: {'Field1' => 'datum1',
|
12
|
-
'Field2' => 'datum2'}
|
12
|
+
'Field2' => 'datum2'},
|
13
|
+
empty?: false
|
13
14
|
end
|
14
15
|
let(:invalid_input) do
|
15
16
|
double 'Data',
|
16
17
|
source_identifier: 'bad record',
|
17
|
-
data: {'Field1' => 'datum1'}
|
18
|
+
data: {'Field1' => 'datum1'},
|
19
|
+
empty?: false
|
18
20
|
end
|
19
21
|
|
20
22
|
|
@@ -9,12 +9,14 @@ describe Topographer::Importer::Strategy::UpdateRecord do
|
|
9
9
|
double 'Data',
|
10
10
|
source_identifier: 'record',
|
11
11
|
data: {'Field1' => 'datum1',
|
12
|
-
'Field2' => 'datum2'}
|
12
|
+
'Field2' => 'datum2'},
|
13
|
+
empty?: false
|
13
14
|
end
|
14
15
|
let(:invalid_input) do
|
15
16
|
double 'Data',
|
16
17
|
source_identifier: 'bad record',
|
17
|
-
data: {'Field1' => 'datum1'}
|
18
|
+
data: {'Field1' => 'datum1'},
|
19
|
+
empty?: false
|
18
20
|
end
|
19
21
|
|
20
22
|
describe '#import_record' do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: topographer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Simpson
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-02-
|
12
|
+
date: 2014-02-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -76,6 +76,7 @@ extensions: []
|
|
76
76
|
extra_rdoc_files: []
|
77
77
|
files:
|
78
78
|
- .gitignore
|
79
|
+
- .travis.yml
|
79
80
|
- Gemfile
|
80
81
|
- LICENSE
|
81
82
|
- LICENSE.txt
|
@@ -100,6 +101,9 @@ files:
|
|
100
101
|
- lib/Topographer/importer/mapper/default_field_mapping.rb
|
101
102
|
- lib/Topographer/importer/mapper/field_mapping.rb
|
102
103
|
- lib/Topographer/importer/mapper/ignored_field_mapping.rb
|
104
|
+
- lib/Topographer/importer/mapper/mapper_builder.rb
|
105
|
+
- lib/Topographer/importer/mapper/mapping_columns.rb
|
106
|
+
- lib/Topographer/importer/mapper/mapping_validator.rb
|
103
107
|
- lib/Topographer/importer/mapper/result.rb
|
104
108
|
- lib/Topographer/importer/mapper/validation_field_mapping.rb
|
105
109
|
- lib/Topographer/importer/strategy.rb
|
@@ -110,23 +114,27 @@ files:
|
|
110
114
|
- lib/Topographer/importer/strategy/update_record.rb
|
111
115
|
- lib/Topographer/version.rb
|
112
116
|
- lib/topographer.rb
|
113
|
-
- spec/
|
114
|
-
- spec/
|
115
|
-
- spec/
|
116
|
-
- spec/
|
117
|
-
- spec/
|
118
|
-
- spec/
|
119
|
-
- spec/
|
120
|
-
- spec/
|
121
|
-
- spec/
|
122
|
-
- spec/
|
123
|
-
- spec/
|
124
|
-
- spec/
|
125
|
-
- spec/
|
126
|
-
- spec/
|
127
|
-
- spec/
|
128
|
-
- spec/
|
129
|
-
- spec/
|
117
|
+
- spec/Topographer/importer/helpers/write_log_to_csv_spec.rb
|
118
|
+
- spec/Topographer/importer/helpers_spec.rb
|
119
|
+
- spec/Topographer/importer/importable_spec.rb
|
120
|
+
- spec/Topographer/importer/importer_spec.rb
|
121
|
+
- spec/Topographer/importer/input/source_data_spec.rb
|
122
|
+
- spec/Topographer/importer/logger/base_spec.rb
|
123
|
+
- spec/Topographer/importer/logger/fatal_error_entry_spec.rb
|
124
|
+
- spec/Topographer/importer/logger/simple_spec.rb
|
125
|
+
- spec/Topographer/importer/mapper/default_field_mapping_spec.rb
|
126
|
+
- spec/Topographer/importer/mapper/field_mapping_spec.rb
|
127
|
+
- spec/Topographer/importer/mapper/mapper_builder_spec.rb
|
128
|
+
- spec/Topographer/importer/mapper/mapping_columns_spec.rb
|
129
|
+
- spec/Topographer/importer/mapper/mapping_validator_spec.rb
|
130
|
+
- spec/Topographer/importer/mapper/validation_field_mapping_spec.rb
|
131
|
+
- spec/Topographer/importer/mapper_spec.rb
|
132
|
+
- spec/Topographer/importer/strategy/base_spec.rb
|
133
|
+
- spec/Topographer/importer/strategy/create_or_update_record_spec.rb
|
134
|
+
- spec/Topographer/importer/strategy/import_new_records_spec.rb
|
135
|
+
- spec/Topographer/importer/strategy/import_status_spec.rb
|
136
|
+
- spec/Topographer/importer/strategy/mapped_model.rb
|
137
|
+
- spec/Topographer/importer/strategy/update_record_spec.rb
|
130
138
|
- spec/spec_helper.rb
|
131
139
|
- topographer.gemspec
|
132
140
|
homepage: ''
|
@@ -155,22 +163,26 @@ specification_version: 4
|
|
155
163
|
summary: Topographer allows the mapping of columnar input data to fields for active
|
156
164
|
record models. This facilitates importing from a variety of sources.
|
157
165
|
test_files:
|
158
|
-
- spec/
|
159
|
-
- spec/
|
160
|
-
- spec/
|
161
|
-
- spec/
|
162
|
-
- spec/
|
163
|
-
- spec/
|
164
|
-
- spec/
|
165
|
-
- spec/
|
166
|
-
- spec/
|
167
|
-
- spec/
|
168
|
-
- spec/
|
169
|
-
- spec/
|
170
|
-
- spec/
|
171
|
-
- spec/
|
172
|
-
- spec/
|
173
|
-
- spec/
|
174
|
-
- spec/
|
166
|
+
- spec/Topographer/importer/helpers/write_log_to_csv_spec.rb
|
167
|
+
- spec/Topographer/importer/helpers_spec.rb
|
168
|
+
- spec/Topographer/importer/importable_spec.rb
|
169
|
+
- spec/Topographer/importer/importer_spec.rb
|
170
|
+
- spec/Topographer/importer/input/source_data_spec.rb
|
171
|
+
- spec/Topographer/importer/logger/base_spec.rb
|
172
|
+
- spec/Topographer/importer/logger/fatal_error_entry_spec.rb
|
173
|
+
- spec/Topographer/importer/logger/simple_spec.rb
|
174
|
+
- spec/Topographer/importer/mapper/default_field_mapping_spec.rb
|
175
|
+
- spec/Topographer/importer/mapper/field_mapping_spec.rb
|
176
|
+
- spec/Topographer/importer/mapper/mapper_builder_spec.rb
|
177
|
+
- spec/Topographer/importer/mapper/mapping_columns_spec.rb
|
178
|
+
- spec/Topographer/importer/mapper/mapping_validator_spec.rb
|
179
|
+
- spec/Topographer/importer/mapper/validation_field_mapping_spec.rb
|
180
|
+
- spec/Topographer/importer/mapper_spec.rb
|
181
|
+
- spec/Topographer/importer/strategy/base_spec.rb
|
182
|
+
- spec/Topographer/importer/strategy/create_or_update_record_spec.rb
|
183
|
+
- spec/Topographer/importer/strategy/import_new_records_spec.rb
|
184
|
+
- spec/Topographer/importer/strategy/import_status_spec.rb
|
185
|
+
- spec/Topographer/importer/strategy/mapped_model.rb
|
186
|
+
- spec/Topographer/importer/strategy/update_record_spec.rb
|
175
187
|
- spec/spec_helper.rb
|
176
188
|
has_rdoc:
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|