topographer 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,22 +1,28 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
def initialize(output_column, &output_block)
|
7
|
+
unless block_given?
|
8
|
+
raise Topographer::InvalidMappingError, 'Static fields must have an output block'
|
9
|
+
end
|
10
|
+
@output_field = output_column
|
11
|
+
@output_block = output_block
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
def process_input(_, result)
|
15
|
+
@output_data = @output_block.()
|
16
|
+
result.add_data(@output_field, @output_data)
|
17
|
+
rescue => exception
|
18
|
+
result.add_error(@output_field, exception.message)
|
19
|
+
end
|
17
20
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
def required?
|
22
|
+
true
|
23
|
+
end
|
21
24
|
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
22
28
|
end
|
@@ -1,55 +1,62 @@
|
|
1
1
|
require 'active_support/core_ext/hash'
|
2
2
|
require 'active_support/core_ext/object/blank'
|
3
|
-
class Topographer::Importer::Mapper::FieldMapping
|
4
|
-
attr_reader :input_columns, :output_field
|
5
|
-
|
6
|
-
def initialize(required, input_columns, output_field, &mapping_behavior)
|
7
|
-
@required = required
|
8
|
-
@input_columns = Array(input_columns)
|
9
|
-
@output_field = output_field
|
10
|
-
@mapping_behavior = mapping_behavior
|
11
|
-
@invalid_keys = []
|
12
|
-
end
|
13
|
-
|
14
|
-
def process_input(input, result)
|
15
|
-
mapping_input = input.slice(*input_columns)
|
16
|
-
@invalid_keys = get_invalid_keys(mapping_input)
|
17
|
-
data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
|
18
|
-
if !data.nil?
|
19
|
-
result.add_data(output_field, data)
|
20
|
-
elsif required?
|
21
|
-
result.add_error(output_field, invalid_input_error)
|
22
|
-
end
|
23
|
-
|
24
|
-
rescue => exception
|
25
|
-
result.add_error(output_field, exception.message)
|
26
|
-
|
27
|
-
end
|
28
|
-
|
29
|
-
def required?
|
30
|
-
@required
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
3
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
4
|
+
module Topographer
|
5
|
+
class Importer
|
6
|
+
class Mapper
|
7
|
+
class FieldMapping
|
8
|
+
attr_reader :input_columns, :output_field
|
9
|
+
|
10
|
+
def initialize(required, input_columns, output_field, &mapping_behavior)
|
11
|
+
@required = required
|
12
|
+
@input_columns = Array(input_columns)
|
13
|
+
@output_field = output_field
|
14
|
+
@mapping_behavior = mapping_behavior
|
15
|
+
@invalid_keys = []
|
16
|
+
end
|
17
|
+
|
18
|
+
def process_input(input, result)
|
19
|
+
mapping_input = input.slice(*input_columns)
|
20
|
+
@invalid_keys = get_invalid_keys(mapping_input)
|
21
|
+
data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
|
22
|
+
if !data.nil?
|
23
|
+
result.add_data(output_field, data)
|
24
|
+
elsif required?
|
25
|
+
result.add_error(output_field, invalid_input_error)
|
26
|
+
end
|
27
|
+
|
28
|
+
rescue => exception
|
29
|
+
result.add_error(output_field, exception.message)
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
def required?
|
34
|
+
@required
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def apply_mapping(mapping_input)
|
40
|
+
if @mapping_behavior
|
41
|
+
@mapping_behavior.(mapping_input)
|
42
|
+
else
|
43
|
+
(mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def invalid_input_error
|
48
|
+
"Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
|
49
|
+
end
|
50
|
+
|
51
|
+
def get_invalid_keys(input)
|
52
|
+
missing_columns = @input_columns - input.keys
|
53
|
+
#reject input that is not blank or the value `false`
|
54
|
+
#this allows boolean inputs for required fields
|
55
|
+
missing_data = @required ? input.reject { |k, v| !v.blank? || v == false }.keys : []
|
56
|
+
missing_columns + missing_data
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
40
60
|
end
|
41
61
|
end
|
42
|
-
|
43
|
-
def invalid_input_error
|
44
|
-
"Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
|
45
|
-
end
|
46
|
-
|
47
|
-
def get_invalid_keys(input)
|
48
|
-
missing_columns = @input_columns - input.keys
|
49
|
-
#reject input that is not blank or the value `false`
|
50
|
-
#this allows boolean inputs for required fields
|
51
|
-
missing_data = @required ? input.reject{|k,v| !v.blank? || v == false }.keys : []
|
52
|
-
missing_columns + missing_data
|
53
|
-
end
|
54
|
-
|
55
62
|
end
|
@@ -1,10 +1,17 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
5
|
+
def initialize(input_columns)
|
6
|
+
@input_columns = input_columns
|
7
|
+
@output_field = nil
|
8
|
+
end
|
6
9
|
|
7
|
-
|
8
|
-
|
10
|
+
def required?
|
11
|
+
false
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
9
15
|
end
|
10
16
|
end
|
17
|
+
|
@@ -1,46 +1,59 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class MapperBuilder
|
5
|
+
include Topographer::Importer::Mapper::MappingColumns
|
6
|
+
include Topographer::Importer::Mapper::MappingValidator
|
7
|
+
|
8
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
|
9
|
+
:validation_mappings, :default_values, :key_fields, :field_mappings
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@required_mappings = {}
|
13
|
+
@optional_mappings = {}
|
14
|
+
@ignored_mappings = {}
|
15
|
+
@validation_mappings = {}
|
16
|
+
@default_values = {}
|
17
|
+
@key_fields = []
|
18
|
+
@field_mappings = {}
|
19
|
+
end
|
20
|
+
|
21
|
+
def required_mapping(input_columns, output_field, &mapping_behavior)
|
22
|
+
validate_unique_mapping(input_columns, output_field)
|
23
|
+
mapping = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
|
24
|
+
@required_mappings[output_field] = mapping
|
25
|
+
@field_mappings[output_field] = mapping
|
26
|
+
end
|
27
|
+
|
28
|
+
def optional_mapping(input_columns, output_field, &mapping_behavior)
|
29
|
+
validate_unique_mapping(input_columns, output_field)
|
30
|
+
mapping = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
|
31
|
+
@optional_mappings[output_field] = mapping
|
32
|
+
@field_mappings[output_field] = mapping
|
33
|
+
end
|
34
|
+
|
35
|
+
def validation_field(name, input_columns, &mapping_behavior)
|
36
|
+
validate_unique_validation_name(name)
|
37
|
+
@validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
|
38
|
+
end
|
39
|
+
|
40
|
+
def default_value(output_field, &mapping_behavior)
|
41
|
+
validate_unique_mapping([], output_field)
|
42
|
+
mapping = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
|
43
|
+
@default_values[output_field] = mapping
|
44
|
+
@field_mappings[output_field] = mapping
|
45
|
+
end
|
46
|
+
|
47
|
+
def key_field(output_field)
|
48
|
+
validate_key_field(output_field)
|
49
|
+
@key_fields << output_field
|
50
|
+
end
|
51
|
+
|
52
|
+
def ignored_column(input_column)
|
53
|
+
validate_unique_column_mapping_type(input_column, ignored: true)
|
54
|
+
@ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
45
58
|
end
|
46
59
|
end
|
@@ -1,46 +1,53 @@
|
|
1
|
-
module Topographer
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
module MappingColumns
|
5
|
+
def output_fields
|
6
|
+
(required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
|
7
|
+
end
|
8
|
+
|
9
|
+
def required_input_columns
|
10
|
+
(required_mapping_columns + validation_mapping_columns).uniq
|
11
|
+
end
|
12
|
+
|
13
|
+
def input_columns
|
14
|
+
(required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
|
15
|
+
end
|
16
|
+
|
17
|
+
def default_fields
|
18
|
+
default_values.keys
|
19
|
+
end
|
20
|
+
|
21
|
+
def validation_mapping_columns
|
22
|
+
validation_mappings.values.flat_map(&:input_columns)
|
23
|
+
end
|
24
|
+
|
25
|
+
def ignored_mapping_columns
|
26
|
+
ignored_mappings.values.flat_map(&:input_columns)
|
27
|
+
end
|
28
|
+
|
29
|
+
def optional_mapping_columns
|
30
|
+
optional_mappings.values.flat_map(&:input_columns)
|
31
|
+
end
|
32
|
+
|
33
|
+
def required_mapping_columns
|
34
|
+
required_mappings.values.flat_map(&:input_columns)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
def mapped_input_columns
|
39
|
+
required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
|
40
|
+
end
|
41
|
+
|
42
|
+
def mappings
|
43
|
+
@field_mappings
|
44
|
+
end
|
45
|
+
|
46
|
+
def non_ignored_columns
|
47
|
+
required_mappings.merge(optional_mappings)
|
48
|
+
end
|
49
|
+
end
|
45
50
|
end
|
51
|
+
end
|
46
52
|
end
|
53
|
+
|
@@ -1,40 +1,47 @@
|
|
1
|
-
module Topographer
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
module MappingValidator
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
+
def validate_unique_validation_name(name)
|
7
|
+
raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
|
8
|
+
end
|
6
9
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
def validate_unique_output_mapping(output_field)
|
11
|
+
if output_fields.include?(output_field)
|
12
|
+
raise Topographer::InvalidMappingError, 'Output column already mapped.'
|
13
|
+
end
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
16
|
+
def validate_unique_column_mapping_type(mapping_input_columns, options = {})
|
17
|
+
ignored = options.fetch(:ignored, false)
|
18
|
+
mapping_input_columns = Array(mapping_input_columns)
|
19
|
+
mapping_input_columns.each do |col|
|
20
|
+
if ignored && ((input_columns + ignored_mapping_columns).include?(col))
|
21
|
+
raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
|
22
|
+
elsif (ignored_mapping_columns.include?(col))
|
23
|
+
raise Topographer::InvalidMappingError, 'Input column already ignored.'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
24
27
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
def validate_unique_mapping(mapping_input_columns, output_field)
|
29
|
+
if (output_field.is_a?(Array))
|
30
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
31
|
+
end
|
32
|
+
validate_unique_column_mapping_type(mapping_input_columns)
|
33
|
+
validate_unique_output_mapping(output_field)
|
34
|
+
end
|
32
35
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
def validate_key_field(field)
|
37
|
+
if field.is_a?(Array)
|
38
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
39
|
+
elsif key_fields.include?(field)
|
40
|
+
raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
38
44
|
end
|
39
45
|
end
|
40
46
|
end
|
47
|
+
|