topographer 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,21 +1,27 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class Result
|
5
|
+
attr_reader :data, :errors, :source_identifier
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
7
|
+
def initialize(source_identifier)
|
8
|
+
@source_identifier = source_identifier
|
9
|
+
@data = {}
|
10
|
+
@errors = {}
|
11
|
+
end
|
9
12
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
+
def add_data (key, value)
|
14
|
+
@data[key] = value
|
15
|
+
end
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
+
def add_error (key, value)
|
18
|
+
@errors[key] = value
|
19
|
+
end
|
17
20
|
|
18
|
-
|
19
|
-
|
21
|
+
def errors?
|
22
|
+
errors.any?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
20
26
|
end
|
21
27
|
end
|
@@ -1,36 +1,43 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
class Mapper
|
4
|
+
class ValidationFieldMapping < Topographer::Importer::Mapper::FieldMapping
|
5
|
+
attr_reader :name
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
7
|
+
def initialize(name, input_columns, &validation_block)
|
8
|
+
unless block_given?
|
9
|
+
raise Topographer::InvalidMappingError, 'Validation fields must have a behavior block'
|
10
|
+
end
|
11
|
+
@name = name
|
12
|
+
@input_columns = Array(input_columns)
|
13
|
+
@validation_block = validation_block
|
14
|
+
@output_field = nil
|
15
|
+
end
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
def process_input(input, result)
|
18
|
+
mapping_input = input.slice(*input_columns)
|
19
|
+
@invalid_keys = get_invalid_keys(mapping_input)
|
20
|
+
if @invalid_keys.blank?
|
21
|
+
@validation_block.(mapping_input)
|
22
|
+
else
|
23
|
+
result.add_error(name, invalid_input_error)
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
26
|
+
rescue => exception
|
27
|
+
result.add_error(name, exception.message)
|
25
28
|
|
26
|
-
|
29
|
+
end
|
27
30
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
+
def required?
|
32
|
+
true
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
31
36
|
|
32
|
-
|
33
|
-
|
34
|
-
|
37
|
+
def get_invalid_keys(input)
|
38
|
+
@input_columns - input.keys
|
39
|
+
end
|
40
|
+
end
|
35
41
|
end
|
42
|
+
end
|
36
43
|
end
|
@@ -1,7 +1,13 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
1
|
+
require_relative 'strategy/base'
|
2
|
+
require_relative 'strategy/import_new_record'
|
3
|
+
require_relative 'strategy/update_record'
|
4
|
+
require_relative 'strategy/create_or_update_record'
|
5
|
+
require_relative 'strategy/import_status'
|
6
|
+
|
7
|
+
module Topographer
|
8
|
+
class Importer
|
9
|
+
module Strategy
|
10
|
+
|
11
|
+
end
|
12
|
+
end
|
7
13
|
end
|
@@ -1,42 +1,47 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class Base
|
5
|
+
|
6
|
+
attr_accessor :dry_run, :mapper
|
7
|
+
|
8
|
+
def initialize(mapper)
|
9
|
+
@mapper = mapper
|
10
|
+
@dry_run = false
|
11
|
+
end
|
12
|
+
|
13
|
+
def import_record (record_input)
|
14
|
+
raise NotImplementedError
|
15
|
+
end
|
16
|
+
|
17
|
+
def success_message
|
18
|
+
'Imported'
|
19
|
+
end
|
20
|
+
|
21
|
+
def failure_message
|
22
|
+
'Unable to import'
|
23
|
+
end
|
24
|
+
|
25
|
+
def should_persist_import?(status)
|
26
|
+
(@dry_run || status.errors?) ? false : true
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def get_import_status(mapping_result, new_model_errors)
|
32
|
+
status = Topographer::Importer::Strategy::ImportStatus.new(mapping_result.source_identifier)
|
33
|
+
mapping_result.errors.values.each do |error|
|
34
|
+
status.add_error(:mapping, error)
|
35
|
+
end
|
36
|
+
new_model_errors.each do |error|
|
37
|
+
status.add_error(:validation, error)
|
38
|
+
end
|
39
|
+
status.message = (status.errors?) ? failure_message : success_message
|
40
|
+
status.set_timestamp
|
41
|
+
status
|
42
|
+
end
|
2
43
|
|
3
|
-
attr_reader :mapper
|
4
|
-
attr_accessor :dry_run
|
5
|
-
|
6
|
-
def initialize(mapper)
|
7
|
-
@mapper = mapper
|
8
|
-
@dry_run = false
|
9
|
-
end
|
10
|
-
|
11
|
-
def import_record (record_input)
|
12
|
-
raise NotImplementedError
|
13
|
-
end
|
14
|
-
|
15
|
-
def success_message
|
16
|
-
'Imported'
|
17
|
-
end
|
18
|
-
|
19
|
-
def failure_message
|
20
|
-
'Unable to import'
|
21
|
-
end
|
22
|
-
|
23
|
-
def should_persist_import?(status)
|
24
|
-
(@dry_run || status.errors?) ? false : true
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
def get_import_status(mapping_result, new_model_errors)
|
30
|
-
status = Topographer::Importer::Strategy::ImportStatus.new(mapping_result.source_identifier)
|
31
|
-
mapping_result.errors.values.each do |error|
|
32
|
-
status.add_error(:mapping, error)
|
33
44
|
end
|
34
|
-
new_model_errors.each do |error|
|
35
|
-
status.add_error(:validation, error)
|
36
|
-
end
|
37
|
-
status.message = (status.errors?) ? failure_message : success_message
|
38
|
-
status.set_timestamp
|
39
|
-
status
|
40
45
|
end
|
41
|
-
|
46
|
+
end
|
42
47
|
end
|
@@ -1,50 +1,55 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class CreateOrUpdateRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
5
8
|
|
6
|
-
|
7
|
-
|
9
|
+
search_params = mapping_result.data.slice(*mapper.key_fields)
|
10
|
+
model_instances = mapper.model_class.where(search_params)
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
if model_instances.any?
|
13
|
+
model_instance = model_instances.first
|
14
|
+
else
|
15
|
+
model_instance = mapper.model_class.new(search_params)
|
16
|
+
end
|
14
17
|
|
15
|
-
|
18
|
+
generate_messages(model_instance, search_params)
|
16
19
|
|
17
|
-
|
18
|
-
|
20
|
+
model_instance.attributes = mapping_result.data
|
21
|
+
model_instance.valid?
|
19
22
|
|
20
|
-
|
21
|
-
|
23
|
+
model_errors = model_instance.errors.full_messages
|
24
|
+
status = get_import_status(mapping_result, model_errors)
|
22
25
|
|
23
|
-
|
26
|
+
model_instance.save if should_persist_import?(status)
|
24
27
|
|
25
|
-
|
26
|
-
|
28
|
+
status
|
29
|
+
end
|
27
30
|
|
31
|
+
def success_message
|
32
|
+
@success_message
|
33
|
+
end
|
28
34
|
|
29
|
-
|
30
|
-
|
31
|
-
|
35
|
+
def failure_message
|
36
|
+
@failure_message
|
37
|
+
end
|
32
38
|
|
33
|
-
|
34
|
-
@failure_message
|
35
|
-
end
|
39
|
+
private
|
36
40
|
|
37
|
-
|
41
|
+
def generate_messages(model_instance, search_params)
|
42
|
+
if model_instance.new_record?
|
43
|
+
@success_message = 'Imported record'
|
44
|
+
@failure_message = 'Import failed'
|
45
|
+
else
|
46
|
+
params_string = search_params.map { |k, v| "#{k}: #{v}" }.join(', ')
|
47
|
+
@success_message = "Updated record matching `#{params_string}`"
|
48
|
+
@failure_message = "Update failed for record matching `#{params_string}`"
|
49
|
+
end
|
50
|
+
end
|
38
51
|
|
39
|
-
def generate_messages(model_instance, search_params)
|
40
|
-
if model_instance.new_record?
|
41
|
-
@success_message = 'Imported record'
|
42
|
-
@failure_message = 'Import failed'
|
43
|
-
else
|
44
|
-
params_string = search_params.map{|k, v| "#{k}: #{v}"}.join(', ')
|
45
|
-
@success_message = "Updated record matching `#{params_string}`"
|
46
|
-
@failure_message = "Update failed for record matching `#{params_string}`"
|
47
52
|
end
|
48
53
|
end
|
49
|
-
|
54
|
+
end
|
50
55
|
end
|
@@ -1,17 +1,23 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class ImportNewRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
8
|
+
new_model = mapper.model_class.new(mapping_result.data)
|
9
|
+
new_model.valid?
|
10
|
+
model_errors = new_model.errors.full_messages
|
11
|
+
status = get_import_status(mapping_result, model_errors)
|
9
12
|
|
10
|
-
|
13
|
+
new_model.save if should_persist_import?(status)
|
11
14
|
|
12
|
-
|
13
|
-
|
15
|
+
status
|
16
|
+
end
|
14
17
|
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
15
21
|
end
|
16
22
|
|
17
23
|
|
@@ -1,28 +1,35 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class ImportStatus
|
5
|
+
attr_reader :errors, :input_identifier, :timestamp
|
6
|
+
attr_accessor :message
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
8
|
+
def initialize(input_identifier)
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@errors = {mapping: [],
|
11
|
+
validation: []}
|
9
12
|
|
10
|
-
|
13
|
+
end
|
11
14
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
+
def set_timestamp
|
16
|
+
@timestamp ||= DateTime.now
|
17
|
+
end
|
15
18
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
+
def add_error(error_source, error)
|
20
|
+
errors[error_source] << error
|
21
|
+
end
|
19
22
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
+
def error_count
|
24
|
+
errors.values.flatten.length
|
25
|
+
end
|
23
26
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
+
def errors?
|
28
|
+
errors.values.flatten.any?
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
27
32
|
|
33
|
+
end
|
34
|
+
end
|
28
35
|
end
|
@@ -1,33 +1,37 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Strategy
|
4
|
+
class UpdateRecord < Topographer::Importer::Strategy::Base
|
2
5
|
|
3
|
-
|
4
|
-
|
6
|
+
def import_record (source_data)
|
7
|
+
mapping_result = mapper.map_input(source_data)
|
5
8
|
|
6
|
-
|
7
|
-
|
9
|
+
search_params = mapping_result.data.slice(*mapper.key_fields)
|
10
|
+
model_instance = mapper.model_class.where(search_params).first
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
if model_instance
|
13
|
+
model_instance.attributes = mapping_result.data
|
14
|
+
model_instance.valid?
|
15
|
+
model_errors = model_instance.errors.full_messages
|
16
|
+
status = get_import_status(mapping_result, model_errors)
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
18
|
+
model_instance.save if should_persist_import?(status)
|
19
|
+
else
|
20
|
+
status = get_import_status(mapping_result, ["Record not found with params: #{search_params.to_yaml}"])
|
21
|
+
end
|
19
22
|
|
20
|
-
|
21
|
-
|
23
|
+
status
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def success_message
|
27
|
+
'Updated'
|
28
|
+
end
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
+
def failure_message
|
31
|
+
'Unable to update from import'
|
32
|
+
end
|
30
33
|
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
31
37
|
end
|
32
|
-
|
33
|
-
|