topographer 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,21 +1,27 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class Result
|
5
|
+
attr_reader :data, :errors, :source_identifier
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
7
|
+
def initialize(source_identifier)
|
8
|
+
@source_identifier = source_identifier
|
9
|
+
@data = {}
|
10
|
+
@errors = {}
|
11
|
+
end
|
9
12
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
+
def add_data (key, value)
|
14
|
+
@data[key] = value
|
15
|
+
end
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
+
def add_error (key, value)
|
18
|
+
@errors[key] = value
|
19
|
+
end
|
17
20
|
|
18
|
-
|
19
|
-
|
21
|
+
def errors?
|
22
|
+
errors.any?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
20
26
|
end
|
21
27
|
end
|
@@ -1,36 +1,43 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class ValidationFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
5
|
+
attr_reader :name
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
7
|
+
def initialize(name, input_columns, &validation_block)
|
8
|
+
unless block_given?
|
9
|
+
raise Topographer::InvalidMappingError, 'Validation fields must have a behavior block'
|
10
|
+
end
|
11
|
+
@name = name
|
12
|
+
@input_columns = Array(input_columns)
|
13
|
+
@validation_block = validation_block
|
14
|
+
@output_field = nil
|
15
|
+
end
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
def process_input(input, result)
|
18
|
+
mapping_input = input.slice(*input_columns)
|
19
|
+
@invalid_keys = get_invalid_keys(mapping_input)
|
20
|
+
if @invalid_keys.blank?
|
21
|
+
@validation_block.(mapping_input)
|
22
|
+
else
|
23
|
+
result.add_error(name, invalid_input_error)
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
26
|
+
rescue => exception
|
27
|
+
result.add_error(name, exception.message)
|
25
28
|
|
26
|
-
|
29
|
+
end
|
27
30
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
+
def required?
|
32
|
+
true
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
31
36
|
|
32
|
-
|
33
|
-
|
34
|
-
|
37
|
+
def get_invalid_keys(input)
|
38
|
+
@input_columns - input.keys
|
39
|
+
end
|
40
|
+
end
|
35
41
|
end
|
42
|
+
end
|
36
43
|
end
|
@@ -1,7 +1,13 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
1
|
+
require_relative 'strategy/base'
|
2
|
+
require_relative 'strategy/import_new_record'
|
3
|
+
require_relative 'strategy/update_record'
|
4
|
+
require_relative 'strategy/create_or_update_record'
|
5
|
+
require_relative 'strategy/import_status'
|
6
|
+
|
7
|
+
module Topographer
|
8
|
+
class Importer
|
9
|
+
module Strategy
|
10
|
+
|
11
|
+
end
|
12
|
+
end
|
7
13
|
end
|
@@ -1,42 +1,47 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class Base
|
5
|
+
|
6
|
+
attr_accessor :dry_run, :mapper
|
7
|
+
|
8
|
+
def initialize(mapper)
|
9
|
+
@mapper = mapper
|
10
|
+
@dry_run = false
|
11
|
+
end
|
12
|
+
|
13
|
+
def import_record (record_input)
|
14
|
+
raise NotImplementedError
|
15
|
+
end
|
16
|
+
|
17
|
+
def success_message
|
18
|
+
'Imported'
|
19
|
+
end
|
20
|
+
|
21
|
+
def failure_message
|
22
|
+
'Unable to import'
|
23
|
+
end
|
24
|
+
|
25
|
+
def should_persist_import?(status)
|
26
|
+
(@dry_run || status.errors?) ? false : true
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def get_import_status(mapping_result, new_model_errors)
|
32
|
+
status = Topographer::Importer::Strategy::ImportStatus.new(mapping_result.source_identifier)
|
33
|
+
mapping_result.errors.values.each do |error|
|
34
|
+
status.add_error(:mapping, error)
|
35
|
+
end
|
36
|
+
new_model_errors.each do |error|
|
37
|
+
status.add_error(:validation, error)
|
38
|
+
end
|
39
|
+
status.message = (status.errors?) ? failure_message : success_message
|
40
|
+
status.set_timestamp
|
41
|
+
status
|
42
|
+
end
|
2
43
|
|
3
|
-
attr_reader :mapper
|
4
|
-
attr_accessor :dry_run
|
5
|
-
|
6
|
-
def initialize(mapper)
|
7
|
-
@mapper = mapper
|
8
|
-
@dry_run = false
|
9
|
-
end
|
10
|
-
|
11
|
-
def import_record (record_input)
|
12
|
-
raise NotImplementedError
|
13
|
-
end
|
14
|
-
|
15
|
-
def success_message
|
16
|
-
'Imported'
|
17
|
-
end
|
18
|
-
|
19
|
-
def failure_message
|
20
|
-
'Unable to import'
|
21
|
-
end
|
22
|
-
|
23
|
-
def should_persist_import?(status)
|
24
|
-
(@dry_run || status.errors?) ? false : true
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
def get_import_status(mapping_result, new_model_errors)
|
30
|
-
status = Topographer::Importer::Strategy::ImportStatus.new(mapping_result.source_identifier)
|
31
|
-
mapping_result.errors.values.each do |error|
|
32
|
-
status.add_error(:mapping, error)
|
33
44
|
end
|
34
|
-
new_model_errors.each do |error|
|
35
|
-
status.add_error(:validation, error)
|
36
|
-
end
|
37
|
-
status.message = (status.errors?) ? failure_message : success_message
|
38
|
-
status.set_timestamp
|
39
|
-
status
|
40
45
|
end
|
41
|
-
|
46
|
+
end
|
42
47
|
end
|
@@ -1,50 +1,55 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class CreateOrUpdateRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
5
8
|
|
6
|
-
|
7
|
-
|
9
|
+
search_params = mapping_result.data.slice(*mapper.key_fields)
|
10
|
+
model_instances = mapper.model_class.where(search_params)
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
if model_instances.any?
|
13
|
+
model_instance = model_instances.first
|
14
|
+
else
|
15
|
+
model_instance = mapper.model_class.new(search_params)
|
16
|
+
end
|
14
17
|
|
15
|
-
|
18
|
+
generate_messages(model_instance, search_params)
|
16
19
|
|
17
|
-
|
18
|
-
|
20
|
+
model_instance.attributes = mapping_result.data
|
21
|
+
model_instance.valid?
|
19
22
|
|
20
|
-
|
21
|
-
|
23
|
+
model_errors = model_instance.errors.full_messages
|
24
|
+
status = get_import_status(mapping_result, model_errors)
|
22
25
|
|
23
|
-
|
26
|
+
model_instance.save if should_persist_import?(status)
|
24
27
|
|
25
|
-
|
26
|
-
|
28
|
+
status
|
29
|
+
end
|
27
30
|
|
31
|
+
def success_message
|
32
|
+
@success_message
|
33
|
+
end
|
28
34
|
|
29
|
-
|
30
|
-
|
31
|
-
|
35
|
+
def failure_message
|
36
|
+
@failure_message
|
37
|
+
end
|
32
38
|
|
33
|
-
|
34
|
-
@failure_message
|
35
|
-
end
|
39
|
+
private
|
36
40
|
|
37
|
-
|
41
|
+
def generate_messages(model_instance, search_params)
|
42
|
+
if model_instance.new_record?
|
43
|
+
@success_message = 'Imported record'
|
44
|
+
@failure_message = 'Import failed'
|
45
|
+
else
|
46
|
+
params_string = search_params.map { |k, v| "#{k}: #{v}" }.join(', ')
|
47
|
+
@success_message = "Updated record matching `#{params_string}`"
|
48
|
+
@failure_message = "Update failed for record matching `#{params_string}`"
|
49
|
+
end
|
50
|
+
end
|
38
51
|
|
39
|
-
def generate_messages(model_instance, search_params)
|
40
|
-
if model_instance.new_record?
|
41
|
-
@success_message = 'Imported record'
|
42
|
-
@failure_message = 'Import failed'
|
43
|
-
else
|
44
|
-
params_string = search_params.map{|k, v| "#{k}: #{v}"}.join(', ')
|
45
|
-
@success_message = "Updated record matching `#{params_string}`"
|
46
|
-
@failure_message = "Update failed for record matching `#{params_string}`"
|
47
52
|
end
|
48
53
|
end
|
49
|
-
|
54
|
+
end
|
50
55
|
end
|
@@ -1,17 +1,23 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class ImportNewRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
8
|
+
new_model = mapper.model_class.new(mapping_result.data)
|
9
|
+
new_model.valid?
|
10
|
+
model_errors = new_model.errors.full_messages
|
11
|
+
status = get_import_status(mapping_result, model_errors)
|
9
12
|
|
10
|
-
|
13
|
+
new_model.save if should_persist_import?(status)
|
11
14
|
|
12
|
-
|
13
|
-
|
15
|
+
status
|
16
|
+
end
|
14
17
|
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
15
21
|
end
|
16
22
|
|
17
23
|
|
@@ -1,28 +1,35 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class ImportStatus
|
5
|
+
attr_reader :errors, :input_identifier, :timestamp
|
6
|
+
attr_accessor :message
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
8
|
+
def initialize(input_identifier)
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@errors = {mapping: [],
|
11
|
+
validation: []}
|
9
12
|
|
10
|
-
|
13
|
+
end
|
11
14
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
+
def set_timestamp
|
16
|
+
@timestamp ||= DateTime.now
|
17
|
+
end
|
15
18
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
+
def add_error(error_source, error)
|
20
|
+
errors[error_source] << error
|
21
|
+
end
|
19
22
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
+
def error_count
|
24
|
+
errors.values.flatten.length
|
25
|
+
end
|
23
26
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
+
def errors?
|
28
|
+
errors.values.flatten.any?
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
27
32
|
|
33
|
+
end
|
34
|
+
end
|
28
35
|
end
|
@@ -1,33 +1,37 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class UpdateRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
5
8
|
|
6
|
-
|
7
|
-
|
9
|
+
search_params = mapping_result.data.slice(*mapper.key_fields)
|
10
|
+
model_instance = mapper.model_class.where(search_params).first
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
if model_instance
|
13
|
+
model_instance.attributes = mapping_result.data
|
14
|
+
model_instance.valid?
|
15
|
+
model_errors = model_instance.errors.full_messages
|
16
|
+
status = get_import_status(mapping_result, model_errors)
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
18
|
+
model_instance.save if should_persist_import?(status)
|
19
|
+
else
|
20
|
+
status = get_import_status(mapping_result, ["Record not found with params: #{search_params.to_yaml}"])
|
21
|
+
end
|
19
22
|
|
20
|
-
|
21
|
-
|
23
|
+
status
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def success_message
|
27
|
+
'Updated'
|
28
|
+
end
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
+
def failure_message
|
31
|
+
'Unable to update from import'
|
32
|
+
end
|
30
33
|
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
31
37
|
end
|
32
|
-
|
33
|
-
|