topographer 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/topographer/exceptions.rb +5 -3
- data/lib/topographer/importer.rb +25 -9
- data/lib/topographer/importer/helpers.rb +19 -13
- data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
- data/lib/topographer/importer/importable.rb +7 -3
- data/lib/topographer/importer/input.rb +11 -4
- data/lib/topographer/importer/input/base.rb +21 -15
- data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
- data/lib/topographer/importer/input/roo.rb +31 -24
- data/lib/topographer/importer/input/source_data.rb +14 -8
- data/lib/topographer/importer/logger.rb +10 -6
- data/lib/topographer/importer/logger/base.rb +75 -66
- data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
- data/lib/topographer/importer/logger/log_entry.rb +31 -25
- data/lib/topographer/importer/logger/simple.rb +25 -19
- data/lib/topographer/importer/mapper.rb +58 -53
- data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
- data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
- data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
- data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
- data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
- data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
- data/lib/topographer/importer/mapper/result.rb +21 -15
- data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
- data/lib/topographer/importer/strategy.rb +12 -6
- data/lib/topographer/importer/strategy/base.rb +43 -38
- data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
- data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
- data/lib/topographer/importer/strategy/import_status.rb +27 -20
- data/lib/topographer/importer/strategy/update_record.rb +28 -24
- data/lib/topographer/version.rb +1 -1
- data/spec/assets/test_files/a_csv.csv +3 -0
- data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
- data/spec/topographer/importer/importer_spec.rb +21 -5
- data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
- data/spec/topographer/importer/input/source_data_spec.rb +2 -2
- data/spec/topographer/importer/logger/base_spec.rb +19 -0
- data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
- data/spec/topographer/importer/logger/simple_spec.rb +3 -3
- data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
- data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
- data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
- data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
- data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
- data/spec/topographer/importer/mapper_spec.rb +25 -25
- data/spec/topographer/importer/strategy/base_spec.rb +16 -5
- data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
- data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
- data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
- data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
- data/topographer.gemspec +3 -2
- metadata +101 -102
@@ -1,12 +1,18 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Input
|
4
|
+
class SourceData
|
5
|
+
attr_reader :source_identifier, :data
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
7
|
+
def initialize(source_identifier, data)
|
8
|
+
@source_identifier = source_identifier
|
9
|
+
@data = data
|
10
|
+
end
|
8
11
|
|
9
|
-
|
10
|
-
|
12
|
+
def empty?
|
13
|
+
@data.empty?
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
11
17
|
end
|
12
18
|
end
|
@@ -1,8 +1,12 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
require_relative 'logger/fatal_error_entry'
|
1
|
+
require_relative 'logger/base'
|
2
|
+
require_relative 'logger/simple'
|
3
|
+
require_relative 'logger/file'
|
4
|
+
require_relative 'logger/log_entry'
|
5
|
+
require_relative 'logger/fatal_error_entry'
|
7
6
|
|
7
|
+
module Topographer
|
8
|
+
class Importer
|
9
|
+
module Logger
|
10
|
+
end
|
11
|
+
end
|
8
12
|
end
|
@@ -1,69 +1,78 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class Base
|
5
|
+
|
6
|
+
attr_reader :fatal_errors
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@fatal_errors = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def successes
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
def failures
|
17
|
+
raise NotImplementedError
|
18
|
+
end
|
19
|
+
|
20
|
+
def log_import(log_entry)
|
21
|
+
if log_entry.success?
|
22
|
+
log_success(log_entry)
|
23
|
+
else
|
24
|
+
log_failure(log_entry)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def log_success(log_entry)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
def log_failure(log_entry)
|
33
|
+
raise NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_fatal(source, message)
|
37
|
+
@fatal_errors << Topographer::Importer::Logger::FatalErrorEntry.new(source, message)
|
38
|
+
end
|
39
|
+
|
40
|
+
def successful_imports
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
|
44
|
+
def failed_imports
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
def entries?
|
49
|
+
total_imports > 0
|
50
|
+
end
|
51
|
+
|
52
|
+
def total_imports
|
53
|
+
(successful_imports + failed_imports)
|
54
|
+
end
|
55
|
+
|
56
|
+
def all_entries
|
57
|
+
(successes + failures + fatal_errors).sort { |a, b| a.timestamp <=> b.timestamp }
|
58
|
+
end
|
59
|
+
|
60
|
+
def errors?
|
61
|
+
fatal_error? || failed_imports > 0
|
62
|
+
end
|
63
|
+
|
64
|
+
def success?
|
65
|
+
!errors?
|
66
|
+
end
|
67
|
+
|
68
|
+
def fatal_error?
|
69
|
+
@fatal_errors.any?
|
70
|
+
end
|
71
|
+
|
72
|
+
def save
|
73
|
+
raise NotImplementedError
|
74
|
+
end
|
75
|
+
end
|
22
76
|
end
|
23
77
|
end
|
24
|
-
|
25
|
-
def log_success(log_entry)
|
26
|
-
raise NotImplementedError
|
27
|
-
end
|
28
|
-
|
29
|
-
def log_failure(log_entry)
|
30
|
-
raise NotImplementedError
|
31
|
-
end
|
32
|
-
|
33
|
-
def log_fatal(source, message)
|
34
|
-
@fatal_errors << Topographer::Importer::Logger::FatalErrorEntry.new(source, message)
|
35
|
-
end
|
36
|
-
|
37
|
-
def successful_imports
|
38
|
-
raise NotImplementedError
|
39
|
-
end
|
40
|
-
|
41
|
-
def failed_imports
|
42
|
-
raise NotImplementedError
|
43
|
-
end
|
44
|
-
|
45
|
-
def entries?
|
46
|
-
total_imports > 0
|
47
|
-
end
|
48
|
-
|
49
|
-
def total_imports
|
50
|
-
(successful_imports + failed_imports)
|
51
|
-
end
|
52
|
-
|
53
|
-
def all_entries
|
54
|
-
(successes + failures + fatal_errors).sort {|a, b| a.timestamp <=> b.timestamp}
|
55
|
-
end
|
56
|
-
|
57
|
-
def errors?
|
58
|
-
fatal_error? || failed_imports > 0
|
59
|
-
end
|
60
|
-
|
61
|
-
def fatal_error?
|
62
|
-
@fatal_errors.any?
|
63
|
-
end
|
64
|
-
|
65
|
-
def save
|
66
|
-
raise NotImplementedError
|
67
|
-
end
|
68
|
-
|
69
78
|
end
|
@@ -1,19 +1,25 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class FatalErrorEntry < Topographer::Importer::Logger::LogEntry
|
5
|
+
attr_reader :message, :timestamp, :model_name
|
3
6
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
7
|
+
def initialize(input_identifier, message)
|
8
|
+
@timestamp = DateTime.now
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@model_name = 'N/A'
|
11
|
+
@message = message
|
12
|
+
end
|
13
|
+
def source_identifier
|
14
|
+
'import failure'
|
15
|
+
end
|
16
|
+
def details
|
17
|
+
{}
|
18
|
+
end
|
19
|
+
def failure?
|
20
|
+
true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
18
24
|
end
|
19
25
|
end
|
@@ -1,34 +1,40 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class LogEntry
|
5
|
+
attr_reader :input_identifier,
|
6
|
+
:model_name
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
8
|
+
def initialize(input_identifier, model_name, import_status)
|
9
|
+
@input_identifier = input_identifier
|
10
|
+
@model_name = model_name
|
11
|
+
@import_status = import_status
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
+
def source_identifier
|
15
|
+
@import_status.input_identifier
|
16
|
+
end
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
+
def message
|
19
|
+
@import_status.message
|
20
|
+
end
|
18
21
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
def timestamp
|
23
|
+
@import_status.timestamp
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def details
|
27
|
+
@import_status.errors
|
28
|
+
end
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
+
def success?
|
31
|
+
!failure?
|
32
|
+
end
|
30
33
|
|
31
|
-
|
32
|
-
|
34
|
+
def failure?
|
35
|
+
@import_status.errors?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
33
39
|
end
|
34
40
|
end
|
@@ -1,27 +1,33 @@
|
|
1
|
-
|
1
|
+
module Topographer
|
2
|
+
class Importer
|
3
|
+
module Logger
|
4
|
+
class Simple < Topographer::Importer::Logger::Base
|
2
5
|
|
3
|
-
|
6
|
+
attr_reader :successes, :failures
|
4
7
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
8
|
+
def initialize
|
9
|
+
@successes = []
|
10
|
+
@failures = []
|
11
|
+
super
|
12
|
+
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
+
def log_success(message)
|
15
|
+
@successes << message
|
16
|
+
end
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
+
def log_failure(message)
|
19
|
+
@failures << message
|
20
|
+
end
|
18
21
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
def successful_imports
|
23
|
+
@successes.size
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
def failed_imports
|
27
|
+
@failures.size
|
28
|
+
end
|
26
29
|
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
27
33
|
end
|
@@ -1,68 +1,73 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
require_relative 'mapper/result'
|
1
|
+
require_relative 'mapper/mapping_validator'
|
2
|
+
require_relative 'mapper/mapping_columns'
|
3
|
+
require_relative 'mapper/mapper_builder'
|
4
|
+
require_relative 'mapper/field_mapping'
|
5
|
+
require_relative 'mapper/ignored_field_mapping'
|
6
|
+
require_relative 'mapper/validation_field_mapping'
|
7
|
+
require_relative 'mapper/default_field_mapping'
|
8
|
+
require_relative 'mapper/result'
|
10
9
|
|
11
|
-
|
10
|
+
module Topographer
|
11
|
+
class Importer
|
12
|
+
class Mapper
|
12
13
|
|
13
|
-
|
14
|
-
:default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
|
14
|
+
include Topographer::Importer::Mapper::MappingColumns
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
yield mapper_builder
|
16
|
+
attr_reader :required_mappings, :optional_mappings, :ignored_mappings, :validation_mappings,
|
17
|
+
:default_values, :key_fields, :bad_columns, :missing_columns, :model_class, :key_fields
|
19
18
|
|
20
|
-
|
21
|
-
|
19
|
+
def self.build_mapper(model_class)
|
20
|
+
mapper_builder = MapperBuilder.new()
|
21
|
+
yield mapper_builder
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
@optional_mappings = mapper_builder.optional_mappings
|
26
|
-
@ignored_mappings = mapper_builder.ignored_mappings
|
27
|
-
@validation_mappings = mapper_builder.validation_mappings
|
28
|
-
@default_values = mapper_builder.default_values
|
29
|
-
@key_fields = mapper_builder.key_fields
|
30
|
-
@model_class = model_class
|
31
|
-
end
|
23
|
+
new(mapper_builder, model_class)
|
24
|
+
end
|
32
25
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
26
|
+
def initialize(mapper_builder, model_class)
|
27
|
+
@required_mappings = mapper_builder.required_mappings
|
28
|
+
@optional_mappings = mapper_builder.optional_mappings
|
29
|
+
@ignored_mappings = mapper_builder.ignored_mappings
|
30
|
+
@validation_mappings = mapper_builder.validation_mappings
|
31
|
+
@default_values = mapper_builder.default_values
|
32
|
+
@key_fields = mapper_builder.key_fields
|
33
|
+
@field_mappings = mapper_builder.field_mappings
|
34
|
+
@model_class = model_class
|
35
|
+
end
|
37
36
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
37
|
+
def input_structure_valid?(input_columns, options={})
|
38
|
+
ignore_unmapped_columns = options.fetch(:ignore_unmapped_columns, false)
|
39
|
+
@bad_columns ||= input_columns - mapped_input_columns
|
40
|
+
@missing_columns ||= required_input_columns - input_columns
|
44
41
|
|
45
|
-
|
46
|
-
|
42
|
+
if ignore_unmapped_columns
|
43
|
+
@missing_columns.empty?
|
44
|
+
else
|
45
|
+
@bad_columns.empty? && @missing_columns.empty?
|
46
|
+
end
|
47
|
+
end
|
47
48
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
49
|
+
def map_input(source_data)
|
50
|
+
mapping_result = Result.new(source_data.source_identifier)
|
51
|
+
|
52
|
+
if source_data.empty?
|
53
|
+
handle_no_data(mapping_result)
|
54
|
+
else
|
55
|
+
@validation_mappings.values.each do |validation_field_mapping|
|
56
|
+
validation_field_mapping.process_input(source_data.data, mapping_result)
|
57
|
+
end
|
58
|
+
|
59
|
+
mappings.each do |_output_field, field_mapping|
|
60
|
+
field_mapping.process_input(source_data.data, mapping_result)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
mapping_result
|
53
65
|
end
|
54
66
|
|
55
|
-
|
56
|
-
|
57
|
-
|
67
|
+
private
|
68
|
+
def handle_no_data(mapping_result)
|
69
|
+
mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
|
58
70
|
end
|
59
71
|
end
|
60
|
-
|
61
|
-
mapping_result
|
62
72
|
end
|
63
|
-
|
64
|
-
private
|
65
|
-
def handle_no_data(mapping_result)
|
66
|
-
mapping_result.add_error('EmptyRow', 'Unable to import empty row.')
|
67
|
-
end
|
68
73
|
end
|