topographer 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,22 +1,28 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
def initialize(output_column, &output_block)
|
7
|
+
unless block_given?
|
8
|
+
raise Topographer::InvalidMappingError, 'Static fields must have an output block'
|
9
|
+
end
|
10
|
+
@output_field = output_column
|
11
|
+
@output_block = output_block
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
def process_input(_, result)
|
15
|
+
@output_data = @output_block.()
|
16
|
+
result.add_data(@output_field, @output_data)
|
17
|
+
rescue => exception
|
18
|
+
result.add_error(@output_field, exception.message)
|
19
|
+
end
|
17
20
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
def required?
|
22
|
+
true
|
23
|
+
end
|
21
24
|
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
22
28
|
end
|
@@ -1,55 +1,62 @@
|
|
1
1
|
require 'active_support/core_ext/hash'
|
2
2
|
require 'active_support/core_ext/object/blank'
|
3
|
-
class Topographer::Importer::Mapper::FieldMapping
|
4
|
-
attr_reader :input_columns, :output_field
|
5
|
-
|
6
|
-
def initialize(required, input_columns, output_field, &mapping_behavior)
|
7
|
-
@required = required
|
8
|
-
@input_columns = Array(input_columns)
|
9
|
-
@output_field = output_field
|
10
|
-
@mapping_behavior = mapping_behavior
|
11
|
-
@invalid_keys = []
|
12
|
-
end
|
13
|
-
|
14
|
-
def process_input(input, result)
|
15
|
-
mapping_input = input.slice(*input_columns)
|
16
|
-
@invalid_keys = get_invalid_keys(mapping_input)
|
17
|
-
data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
|
18
|
-
if !data.nil?
|
19
|
-
result.add_data(output_field, data)
|
20
|
-
elsif required?
|
21
|
-
result.add_error(output_field, invalid_input_error)
|
22
|
-
end
|
23
|
-
|
24
|
-
rescue => exception
|
25
|
-
result.add_error(output_field, exception.message)
|
26
|
-
|
27
|
-
end
|
28
|
-
|
29
|
-
def required?
|
30
|
-
@required
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
3
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
4
|
+
module Topographer
|
5
|
+
class Importer
|
6
|
+
class Mapper
|
7
|
+
class FieldMapping
|
8
|
+
attr_reader :input_columns, :output_field
|
9
|
+
|
10
|
+
def initialize(required, input_columns, output_field, &mapping_behavior)
|
11
|
+
@required = required
|
12
|
+
@input_columns = Array(input_columns)
|
13
|
+
@output_field = output_field
|
14
|
+
@mapping_behavior = mapping_behavior
|
15
|
+
@invalid_keys = []
|
16
|
+
end
|
17
|
+
|
18
|
+
def process_input(input, result)
|
19
|
+
mapping_input = input.slice(*input_columns)
|
20
|
+
@invalid_keys = get_invalid_keys(mapping_input)
|
21
|
+
data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
|
22
|
+
if !data.nil?
|
23
|
+
result.add_data(output_field, data)
|
24
|
+
elsif required?
|
25
|
+
result.add_error(output_field, invalid_input_error)
|
26
|
+
end
|
27
|
+
|
28
|
+
rescue => exception
|
29
|
+
result.add_error(output_field, exception.message)
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
def required?
|
34
|
+
@required
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def apply_mapping(mapping_input)
|
40
|
+
if @mapping_behavior
|
41
|
+
@mapping_behavior.(mapping_input)
|
42
|
+
else
|
43
|
+
(mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def invalid_input_error
|
48
|
+
"Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
|
49
|
+
end
|
50
|
+
|
51
|
+
def get_invalid_keys(input)
|
52
|
+
missing_columns = @input_columns - input.keys
|
53
|
+
#reject input that is not blank or the value `false`
|
54
|
+
#this allows boolean inputs for required fields
|
55
|
+
missing_data = @required ? input.reject { |k, v| !v.blank? || v == false }.keys : []
|
56
|
+
missing_columns + missing_data
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
40
60
|
end
|
41
61
|
end
|
42
|
-
|
43
|
-
def invalid_input_error
|
44
|
-
"Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
|
45
|
-
end
|
46
|
-
|
47
|
-
def get_invalid_keys(input)
|
48
|
-
missing_columns = @input_columns - input.keys
|
49
|
-
#reject input that is not blank or the value `false`
|
50
|
-
#this allows boolean inputs for required fields
|
51
|
-
missing_data = @required ? input.reject{|k,v| !v.blank? || v == false }.keys : []
|
52
|
-
missing_columns + missing_data
|
53
|
-
end
|
54
|
-
|
55
62
|
end
|
@@ -1,10 +1,17 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
5
|
+
def initialize(input_columns)
|
6
|
+
@input_columns = input_columns
|
7
|
+
@output_field = nil
|
8
|
+
end
|
6
9
|
|
7
|
-
|
8
|
-
|
10
|
+
def required?
|
11
|
+
false
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
9
15
|
end
|
10
16
|
end
|
17
|
+
|
@@ -1,46 +1,59 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class MapperBuilder
|
5
|
+
include Topographer::Importer::Mapper::MappingColumns
|
6
|
+
include Topographer::Importer::Mapper::MappingValidator
|
7
|
+
|
8
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
|
9
|
+
:validation_mappings, :default_values, :key_fields, :field_mappings
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@required_mappings = {}
|
13
|
+
@optional_mappings = {}
|
14
|
+
@ignored_mappings = {}
|
15
|
+
@validation_mappings = {}
|
16
|
+
@default_values = {}
|
17
|
+
@key_fields = []
|
18
|
+
@field_mappings = {}
|
19
|
+
end
|
20
|
+
|
21
|
+
def required_mapping(input_columns, output_field, &mapping_behavior)
|
22
|
+
validate_unique_mapping(input_columns, output_field)
|
23
|
+
mapping = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
|
24
|
+
@required_mappings[output_field] = mapping
|
25
|
+
@field_mappings[output_field] = mapping
|
26
|
+
end
|
27
|
+
|
28
|
+
def optional_mapping(input_columns, output_field, &mapping_behavior)
|
29
|
+
validate_unique_mapping(input_columns, output_field)
|
30
|
+
mapping = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
|
31
|
+
@optional_mappings[output_field] = mapping
|
32
|
+
@field_mappings[output_field] = mapping
|
33
|
+
end
|
34
|
+
|
35
|
+
def validation_field(name, input_columns, &mapping_behavior)
|
36
|
+
validate_unique_validation_name(name)
|
37
|
+
@validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
|
38
|
+
end
|
39
|
+
|
40
|
+
def default_value(output_field, &mapping_behavior)
|
41
|
+
validate_unique_mapping([], output_field)
|
42
|
+
mapping = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
|
43
|
+
@default_values[output_field] = mapping
|
44
|
+
@field_mappings[output_field] = mapping
|
45
|
+
end
|
46
|
+
|
47
|
+
def key_field(output_field)
|
48
|
+
validate_key_field(output_field)
|
49
|
+
@key_fields << output_field
|
50
|
+
end
|
51
|
+
|
52
|
+
def ignored_column(input_column)
|
53
|
+
validate_unique_column_mapping_type(input_column, ignored: true)
|
54
|
+
@ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
45
58
|
end
|
46
59
|
end
|
@@ -1,46 +1,53 @@
|
|
1
|
-
module Topographer
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
module MappingColumns
|
5
|
+
def output_fields
|
6
|
+
(required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
|
7
|
+
end
|
8
|
+
|
9
|
+
def required_input_columns
|
10
|
+
(required_mapping_columns + validation_mapping_columns).uniq
|
11
|
+
end
|
12
|
+
|
13
|
+
def input_columns
|
14
|
+
(required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
|
15
|
+
end
|
16
|
+
|
17
|
+
def default_fields
|
18
|
+
default_values.keys
|
19
|
+
end
|
20
|
+
|
21
|
+
def validation_mapping_columns
|
22
|
+
validation_mappings.values.flat_map(&:input_columns)
|
23
|
+
end
|
24
|
+
|
25
|
+
def ignored_mapping_columns
|
26
|
+
ignored_mappings.values.flat_map(&:input_columns)
|
27
|
+
end
|
28
|
+
|
29
|
+
def optional_mapping_columns
|
30
|
+
optional_mappings.values.flat_map(&:input_columns)
|
31
|
+
end
|
32
|
+
|
33
|
+
def required_mapping_columns
|
34
|
+
required_mappings.values.flat_map(&:input_columns)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
def mapped_input_columns
|
39
|
+
required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
|
40
|
+
end
|
41
|
+
|
42
|
+
def mappings
|
43
|
+
@field_mappings
|
44
|
+
end
|
45
|
+
|
46
|
+
def non_ignored_columns
|
47
|
+
required_mappings.merge(optional_mappings)
|
48
|
+
end
|
49
|
+
end
|
45
50
|
end
|
51
|
+
end
|
46
52
|
end
|
53
|
+
|
@@ -1,40 +1,47 @@
|
|
1
|
-
module Topographer
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
module MappingValidator
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
+
def validate_unique_validation_name(name)
|
7
|
+
raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
|
8
|
+
end
|
6
9
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
def validate_unique_output_mapping(output_field)
|
11
|
+
if output_fields.include?(output_field)
|
12
|
+
raise Topographer::InvalidMappingError, 'Output column already mapped.'
|
13
|
+
end
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
16
|
+
def validate_unique_column_mapping_type(mapping_input_columns, options = {})
|
17
|
+
ignored = options.fetch(:ignored, false)
|
18
|
+
mapping_input_columns = Array(mapping_input_columns)
|
19
|
+
mapping_input_columns.each do |col|
|
20
|
+
if ignored && ((input_columns + ignored_mapping_columns).include?(col))
|
21
|
+
raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
|
22
|
+
elsif (ignored_mapping_columns.include?(col))
|
23
|
+
raise Topographer::InvalidMappingError, 'Input column already ignored.'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
24
27
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
def validate_unique_mapping(mapping_input_columns, output_field)
|
29
|
+
if (output_field.is_a?(Array))
|
30
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
31
|
+
end
|
32
|
+
validate_unique_column_mapping_type(mapping_input_columns)
|
33
|
+
validate_unique_output_mapping(output_field)
|
34
|
+
end
|
32
35
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
def validate_key_field(field)
|
37
|
+
if field.is_a?(Array)
|
38
|
+
raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
|
39
|
+
elsif key_fields.include?(field)
|
40
|
+
raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
38
44
|
end
|
39
45
|
end
|
40
46
|
end
|
47
|
+
|