topographer 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,12 +1,18 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Input
|
4
|
+
class SourceData
|
5
|
+
attr_reader :source_identifier, :data
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
7
|
+
def initialize(source_identifier, data)
|
8
|
+
@source_identifier = source_identifier
|
9
|
+
@data = data
|
10
|
+
end
|
8
11
|
|
9
|
-
|
10
|
-
|
12
|
+
def empty?
|
13
|
+
@data.empty?
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
11
17
|
end
|
12
18
|
end
|
@@ -1,8 +1,12 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
require_relative 'logger/fatal_error_entry'
|
1
|
+
require_relative 'logger/base'
|
2
|
+
require_relative 'logger/simple'
|
3
|
+
require_relative 'logger/file'
|
4
|
+
require_relative 'logger/log_entry'
|
5
|
+
require_relative 'logger/fatal_error_entry'
|
7
6
|
|
7
|
+
module Topographer
|
8
|
+
class Importer
|
9
|
+
module Logger
|
10
|
+
end
|
11
|
+
end
|
8
12
|
end
|
@@ -1,69 +1,78 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class Base
|
5
|
+
|
6
|
+
attr_reader :fatal_errors
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@fatal_errors = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def successes
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
def failures
|
17
|
+
raise NotImplementedError
|
18
|
+
end
|
19
|
+
|
20
|
+
def log_import(log_entry)
|
21
|
+
if log_entry.success?
|
22
|
+
log_success(log_entry)
|
23
|
+
else
|
24
|
+
log_failure(log_entry)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def log_success(log_entry)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
def log_failure(log_entry)
|
33
|
+
raise NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_fatal(source, message)
|
37
|
+
@fatal_errors << Topographer::Importer::Logger::FatalErrorEntry.new(source, message)
|
38
|
+
end
|
39
|
+
|
40
|
+
def successful_imports
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
|
44
|
+
def failed_imports
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
def entries?
|
49
|
+
total_imports > 0
|
50
|
+
end
|
51
|
+
|
52
|
+
def total_imports
|
53
|
+
(successful_imports + failed_imports)
|
54
|
+
end
|
55
|
+
|
56
|
+
def all_entries
|
57
|
+
(successes + failures + fatal_errors).sort { |a, b| a.timestamp <=> b.timestamp }
|
58
|
+
end
|
59
|
+
|
60
|
+
def errors?
|
61
|
+
fatal_error? || failed_imports > 0
|
62
|
+
end
|
63
|
+
|
64
|
+
def success?
|
65
|
+
!errors?
|
66
|
+
end
|
67
|
+
|
68
|
+
def fatal_error?
|
69
|
+
@fatal_errors.any?
|
70
|
+
end
|
71
|
+
|
72
|
+
def save
|
73
|
+
raise NotImplementedError
|
74
|
+
end
|
75
|
+
end
|
22
76
|
end
|
23
77
|
end
|
24
|
-
|
25
|
-
def log_success(log_entry)
|
26
|
-
raise NotImplementedError
|
27
|
-
end
|
28
|
-
|
29
|
-
def log_failure(log_entry)
|
30
|
-
raise NotImplementedError
|
31
|
-
end
|
32
|
-
|
33
|
-
def log_fatal(source, message)
|
34
|
-
@fatal_errors << Topographer::Importer::Logger::FatalErrorEntry.new(source, message)
|
35
|
-
end
|
36
|
-
|
37
|
-
def successful_imports
|
38
|
-
raise NotImplementedError
|
39
|
-
end
|
40
|
-
|
41
|
-
def failed_imports
|
42
|
-
raise NotImplementedError
|
43
|
-
end
|
44
|
-
|
45
|
-
def entries?
|
46
|
-
total_imports > 0
|
47
|
-
end
|
48
|
-
|
49
|
-
def total_imports
|
50
|
-
(successful_imports + failed_imports)
|
51
|
-
end
|
52
|
-
|
53
|
-
def all_entries
|
54
|
-
(successes + failures + fatal_errors).sort {|a, b| a.timestamp <=> b.timestamp}
|
55
|
-
end
|
56
|
-
|
57
|
-
def errors?
|
58
|
-
fatal_error? || failed_imports > 0
|
59
|
-
end
|
60
|
-
|
61
|
-
def fatal_error?
|
62
|
-
@fatal_errors.any?
|
63
|
-
end
|
64
|
-
|
65
|
-
def save
|
66
|
-
raise NotImplementedError
|
67
|
-
end
|
68
|
-
|
69
78
|
end
|
@@ -1,19 +1,25 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class FatalErrorEntry < Topographer::Importer::Logger::LogEntry
|
5
|
+
attr_reader :message, :timestamp, :model_name
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
7
|
+
def initialize(input_identifier, message)
|
8
|
+
@timestamp = DateTime.now
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@model_name = 'N/A'
|
11
|
+
@message = message
|
12
|
+
end
|
13
|
+
def source_identifier
|
14
|
+
'import failure'
|
15
|
+
end
|
16
|
+
def details
|
17
|
+
{}
|
18
|
+
end
|
19
|
+
def failure?
|
20
|
+
true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
18
24
|
end
|
19
25
|
end
|
@@ -1,34 +1,40 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class LogEntry
|
5
|
+
attr_reader :input_identifier,
|
6
|
+
:model_name
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
8
|
+
def initialize(input_identifier, model_name, import_status)
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@model_name = model_name
|
11
|
+
@import_status = import_status
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
+
def source_identifier
|
15
|
+
@import_status.input_identifier
|
16
|
+
end
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
+
def message
|
19
|
+
@import_status.message
|
20
|
+
end
|
18
21
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
def timestamp
|
23
|
+
@import_status.timestamp
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def details
|
27
|
+
@import_status.errors
|
28
|
+
end
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
+
def success?
|
31
|
+
!failure?
|
32
|
+
end
|
30
33
|
|
31
|
-
|
32
|
-
|
34
|
+
def failure?
|
35
|
+
@import_status.errors?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
33
39
|
end
|
34
40
|
end
|
@@ -1,27 +1,33 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class Simple < Topographer::Importer::Logger::Base
|
2
5
|
|
3
|
-
|
6
|
+
attr_reader :successes, :failures
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
8
|
+
def initialize
|
9
|
+
@successes = []
|
10
|
+
@failures = []
|
11
|
+
super
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
+
def log_success(message)
|
15
|
+
@successes << message
|
16
|
+
end
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
+
def log_failure(message)
|
19
|
+
@failures << message
|
20
|
+
end
|
18
21
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
def successful_imports
|
23
|
+
@successes.size
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def failed_imports
|
27
|
+
@failures.size
|
28
|
+
end
|
26
29
|
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
27
33
|
end
|
@@ -1,68 +1,73 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
require_relative 'mapper/result'
|
1
|
+
require_relative 'mapper/mapping_validator'
|
2
|
+
require_relative 'mapper/mapping_columns'
|
3
|
+
require_relative 'mapper/mapper_builder'
|
4
|
+
require_relative 'mapper/field_mapping'
|
5
|
+
require_relative 'mapper/ignored_field_mapping'
|
6
|
+
require_relative 'mapper/validation_field_mapping'
|
7
|
+
require_relative 'mapper/default_field_mapping'
|
8
|
+
require_relative 'mapper/result'
|
10
9
|
|
11
|
-
|
10
|
+
module Topographer
|
11
|
+
class Importer
|
12
|
+
class Mapper
|
12
13
|
|
13
|
-
|
14
|
-
:default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
|
14
|
+
include Topographer::Importer::Mapper::MappingColumns
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
yield mapper_builder
|
16
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings, :validation_mappings,
|
17
|
+
:default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
|
19
18
|
|
20
|
-
|
21
|
-
|
19
|
+
def self.build_mapper(model_class)
|
20
|
+
mapper_builder = MapperBuilder.new()
|
21
|
+
yield mapper_builder
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
@optional_mappings = mapper_builder.optional_mappings
|
26
|
-
@ignored_mappings = mapper_builder.ignored_mappings
|
27
|
-
@validation_mappings = mapper_builder.validation_mappings
|
28
|
-
@default_values = mapper_builder.default_values
|
29
|
-
@key_fields = mapper_builder.key_fields
|
30
|
-
@model_class = model_class
|
31
|
-
end
|
23
|
+
new(mapper_builder, model_class)
|
24
|
+
end
|
32
25
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
26
|
+
def initialize(mapper_builder, model_class)
|
27
|
+
@required_mappings = mapper_builder.required_mappings
|
28
|
+
@optional_mappings = mapper_builder.optional_mappings
|
29
|
+
@ignored_mappings = mapper_builder.ignored_mappings
|
30
|
+
@validation_mappings = mapper_builder.validation_mappings
|
31
|
+
@default_values = mapper_builder.default_values
|
32
|
+
@key_fields = mapper_builder.key_fields
|
33
|
+
@field_mappings = mapper_builder.field_mappings
|
34
|
+
@model_class = model_class
|
35
|
+
end
|
37
36
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
37
|
+
def input_structure_valid?(input_columns, options={})
|
38
|
+
ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
|
39
|
+
@bad_columns ||= input_columns - mapped_input_columns
|
40
|
+
@missing_columns ||= required_input_columns - input_columns
|
44
41
|
|
45
|
-
|
46
|
-
|
42
|
+
if ignore_unmapped_columns
|
43
|
+
@missing_columns.empty?
|
44
|
+
else
|
45
|
+
@bad_columns.empty? && @missing_columns.empty?
|
46
|
+
end
|
47
|
+
end
|
47
48
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
49
|
+
def map_input(source_data)
|
50
|
+
mapping_result = Result.new(source_data.source_identifier)
|
51
|
+
|
52
|
+
if source_data.empty?
|
53
|
+
handle_no_data(mapping_result)
|
54
|
+
else
|
55
|
+
@validation_mappings.values.each do |validation_field_mapping|
|
56
|
+
validation_field_mapping.process_input(source_data.data, mapping_result)
|
57
|
+
end
|
58
|
+
|
59
|
+
mappings.each do |_output_field, field_mapping|
|
60
|
+
field_mapping.process_input(source_data.data, mapping_result)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
mapping_result
|
53
65
|
end
|
54
66
|
|
55
|
-
|
56
|
-
|
57
|
-
|
67
|
+
private
|
68
|
+
def handle_no_data(mapping_result)
|
69
|
+
mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
|
58
70
|
end
|
59
71
|
end
|
60
|
-
|
61
|
-
mapping_result
|
62
72
|
end
|
63
|
-
|
64
|
-
private
|
65
|
-
def handle_no_data(mapping_result)
|
66
|
-
mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
|
67
|
-
end
|
68
73
|
end
|