topographer 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +20 -0
- data/LICENSE.txt +22 -0
- data/README.md +38 -0
- data/Rakefile +1 -0
- data/lib/Topographer/exceptions.rb +4 -0
- data/lib/Topographer/importer.rb +57 -0
- data/lib/Topographer/importer/helpers.rb +18 -0
- data/lib/Topographer/importer/helpers/write_log_to_csv.rb +85 -0
- data/lib/Topographer/importer/importable.rb +5 -0
- data/lib/Topographer/importer/input.rb +5 -0
- data/lib/Topographer/importer/input/base.rb +13 -0
- data/lib/Topographer/importer/input/roo.rb +28 -0
- data/lib/Topographer/importer/input/source_data.rb +8 -0
- data/lib/Topographer/importer/logger.rb +8 -0
- data/lib/Topographer/importer/logger/base.rb +69 -0
- data/lib/Topographer/importer/logger/fatal_error_entry.rb +19 -0
- data/lib/Topographer/importer/logger/file.rb +0 -0
- data/lib/Topographer/importer/logger/log_entry.rb +34 -0
- data/lib/Topographer/importer/logger/simple.rb +27 -0
- data/lib/Topographer/importer/mapper.rb +161 -0
- data/lib/Topographer/importer/mapper/default_field_mapping.rb +22 -0
- data/lib/Topographer/importer/mapper/field_mapping.rb +55 -0
- data/lib/Topographer/importer/mapper/ignored_field_mapping.rb +10 -0
- data/lib/Topographer/importer/mapper/result.rb +21 -0
- data/lib/Topographer/importer/mapper/validation_field_mapping.rb +36 -0
- data/lib/Topographer/importer/strategy.rb +7 -0
- data/lib/Topographer/importer/strategy/base.rb +42 -0
- data/lib/Topographer/importer/strategy/create_or_update_record.rb +50 -0
- data/lib/Topographer/importer/strategy/import_new_record.rb +17 -0
- data/lib/Topographer/importer/strategy/import_status.rb +28 -0
- data/lib/Topographer/importer/strategy/update_record.rb +33 -0
- data/lib/Topographer/version.rb +3 -0
- data/lib/topographer.rb +6 -0
- data/spec/Cartographer/importer/helpers/write_log_to_csv_spec.rb +69 -0
- data/spec/Cartographer/importer/helpers_spec.rb +33 -0
- data/spec/Cartographer/importer/importable_spec.rb +13 -0
- data/spec/Cartographer/importer/importer_spec.rb +132 -0
- data/spec/Cartographer/importer/logger/base_spec.rb +12 -0
- data/spec/Cartographer/importer/logger/fatal_error_entry_spec.rb +31 -0
- data/spec/Cartographer/importer/logger/simple_spec.rb +53 -0
- data/spec/Cartographer/importer/mapper/default_field_mapping_spec.rb +41 -0
- data/spec/Cartographer/importer/mapper/field_mapping_spec.rb +126 -0
- data/spec/Cartographer/importer/mapper/validation_field_mapping_spec.rb +42 -0
- data/spec/Cartographer/importer/mapper_spec.rb +318 -0
- data/spec/Cartographer/importer/strategy/base_spec.rb +43 -0
- data/spec/Cartographer/importer/strategy/create_or_update_record_spec.rb +46 -0
- data/spec/Cartographer/importer/strategy/import_new_records_spec.rb +66 -0
- data/spec/Cartographer/importer/strategy/import_status_spec.rb +24 -0
- data/spec/Cartographer/importer/strategy/mapped_model.rb +41 -0
- data/spec/Cartographer/importer/strategy/update_record_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -0
- data/topographer.gemspec +26 -0
- metadata +175 -0
@@ -0,0 +1,17 @@
|
|
1
|
+
class Importer::Strategy::ImportNewRecord < Importer::Strategy::Base
|
2
|
+
|
3
|
+
def import_record (source_data)
|
4
|
+
mapping_result = mapper.map_input(source_data)
|
5
|
+
new_model = mapper.model_class.new(mapping_result.data)
|
6
|
+
new_model.valid?
|
7
|
+
model_errors = new_model.errors.full_messages
|
8
|
+
status = get_import_status(mapping_result, model_errors)
|
9
|
+
|
10
|
+
new_model.save if should_persist_import?(status)
|
11
|
+
|
12
|
+
status
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class Importer::Strategy::ImportStatus
|
2
|
+
attr_reader :errors, :input_identifier, :timestamp
|
3
|
+
attr_accessor :message
|
4
|
+
|
5
|
+
def initialize(input_identifier)
|
6
|
+
@input_identifier = input_identifier
|
7
|
+
@errors = {mapping: [],
|
8
|
+
validation: []}
|
9
|
+
|
10
|
+
end
|
11
|
+
|
12
|
+
def set_timestamp
|
13
|
+
@timestamp ||= DateTime.now
|
14
|
+
end
|
15
|
+
|
16
|
+
def add_error(error_source, error)
|
17
|
+
errors[error_source] << error
|
18
|
+
end
|
19
|
+
|
20
|
+
def error_count
|
21
|
+
errors.values.flatten.length
|
22
|
+
end
|
23
|
+
|
24
|
+
def errors?
|
25
|
+
errors.values.flatten.any?
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class Importer::Strategy::UpdateRecord < Importer::Strategy::Base
|
2
|
+
|
3
|
+
def import_record (source_data)
|
4
|
+
mapping_result = mapper.map_input(source_data)
|
5
|
+
|
6
|
+
search_params = mapping_result.data.slice(*mapper.key_fields)
|
7
|
+
model_instance = mapper.model_class.where(search_params).first
|
8
|
+
|
9
|
+
if model_instance
|
10
|
+
model_instance.attributes = mapping_result.data
|
11
|
+
model_instance.valid?
|
12
|
+
model_errors = model_instance.errors.full_messages
|
13
|
+
status = get_import_status(mapping_result, model_errors)
|
14
|
+
|
15
|
+
model_instance.save if should_persist_import?(status)
|
16
|
+
else
|
17
|
+
status = get_import_status(mapping_result, ["Record not found with params: #{search_params.to_yaml}"])
|
18
|
+
end
|
19
|
+
|
20
|
+
status
|
21
|
+
end
|
22
|
+
|
23
|
+
def success_message
|
24
|
+
'Updated'
|
25
|
+
end
|
26
|
+
|
27
|
+
def failure_message
|
28
|
+
'Unable to update from import'
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
|
data/lib/topographer.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Importer::Helpers::WriteLogToCSV do
|
4
|
+
let(:successful_entry) do
|
5
|
+
status = Importer::Strategy::ImportStatus.new('test-input')
|
6
|
+
status.set_timestamp
|
7
|
+
Importer::Logger::LogEntry.new('test-input', 'TestModel', status)
|
8
|
+
end
|
9
|
+
let(:failed_entry) do
|
10
|
+
status = Importer::Strategy::ImportStatus.new('test-input')
|
11
|
+
status.set_timestamp
|
12
|
+
(1+rand(6)).times do |n|
|
13
|
+
status.add_error(:mapping, "Test error #{n}")
|
14
|
+
end
|
15
|
+
Importer::Logger::LogEntry.new('test-input', 'TestModel', status)
|
16
|
+
end
|
17
|
+
let(:fatal_error) do
|
18
|
+
Importer::Logger::FatalErrorEntry.new('test-input', 'FATAL ERROR')
|
19
|
+
end
|
20
|
+
let(:successes) do
|
21
|
+
successes = []
|
22
|
+
2.times do
|
23
|
+
successes << successful_entry
|
24
|
+
end
|
25
|
+
successes
|
26
|
+
end
|
27
|
+
let(:failures) do
|
28
|
+
failures = []
|
29
|
+
3.times do
|
30
|
+
failures << failed_entry
|
31
|
+
end
|
32
|
+
failures
|
33
|
+
end
|
34
|
+
let(:fatal_errors) do
|
35
|
+
errors = []
|
36
|
+
4.times do
|
37
|
+
errors << fatal_error
|
38
|
+
end
|
39
|
+
errors
|
40
|
+
end
|
41
|
+
let(:logger) do
|
42
|
+
double 'Logger::Base',
|
43
|
+
successful_imports: 2,
|
44
|
+
failed_imports: 3,
|
45
|
+
total_imports: 5,
|
46
|
+
fatal_error?: false,
|
47
|
+
errors?: true,
|
48
|
+
successes: successes,
|
49
|
+
failures: failures,
|
50
|
+
fatal_errors: fatal_errors,
|
51
|
+
entries?: true,
|
52
|
+
all_entries: (successes + failures + fatal_errors)
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '#write_log_to_csv' do
|
56
|
+
it 'should write a passed logger instance to a CSV file' do
|
57
|
+
file = double('file')
|
58
|
+
CSV.should_receive(:open).with('fake_file_path', 'wb').and_yield(file)
|
59
|
+
file.should_receive(:<<).exactly(12).times
|
60
|
+
Importer::Helpers::WriteLogToCSV.instance.write_log_to_csv(logger, 'fake_file_path', write_all: true)
|
61
|
+
end
|
62
|
+
it 'should only write failures and fatal errors if write_all is false' do
|
63
|
+
file = double('file')
|
64
|
+
CSV.should_receive(:open).with('fake_file_path', 'wb').and_yield(file)
|
65
|
+
file.should_receive(:<<).exactly(10).times
|
66
|
+
Importer::Helpers::WriteLogToCSV.instance.write_log_to_csv(logger, 'fake_file_path', write_all: false)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestImportable
|
4
|
+
extend Importer::Helpers
|
5
|
+
end
|
6
|
+
|
7
|
+
describe Importer::Helpers do
|
8
|
+
describe ".boolify" do
|
9
|
+
it "returns true if given 'Yes'" do
|
10
|
+
expect(TestImportable.boolify('Yes')).to eql true
|
11
|
+
end
|
12
|
+
|
13
|
+
it "returns true if given 'True'" do
|
14
|
+
expect(TestImportable.boolify('True')).to eql true
|
15
|
+
end
|
16
|
+
|
17
|
+
it "returns false if given 'No'" do
|
18
|
+
expect(TestImportable.boolify('No')).to eql false
|
19
|
+
end
|
20
|
+
|
21
|
+
it "returns false if given 'False'" do
|
22
|
+
expect(TestImportable.boolify('False')).to eql false
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'returns nil if the type is unknown' do
|
26
|
+
expect(TestImportable.boolify('Unknown')).to be_nil
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'returns nil if it is given a nil' do
|
30
|
+
expect(TestImportable.boolify(nil)).to be_nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestImportable
|
4
|
+
extend Importer::Importable
|
5
|
+
end
|
6
|
+
|
7
|
+
describe Importer::Importable do
|
8
|
+
describe "#get_mapper" do
|
9
|
+
it 'should raise NotImplementedError' do
|
10
|
+
expect { TestImportable.get_mapper(nil) }.to raise_error(NotImplementedError)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ostruct'
|
3
|
+
|
4
|
+
class MockImportable < OpenStruct
|
5
|
+
include Importer::Importable
|
6
|
+
|
7
|
+
def self.create(params)
|
8
|
+
self.new(params)
|
9
|
+
end
|
10
|
+
|
11
|
+
def valid?
|
12
|
+
self.errors = OpenStruct.new(full_messages: [])
|
13
|
+
if field_2 == 'datum2'
|
14
|
+
true
|
15
|
+
else
|
16
|
+
self.errors = OpenStruct.new(full_messages: ['Field 2 is not datum2'])
|
17
|
+
false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.get_mapper(strategy_class)
|
22
|
+
case
|
23
|
+
when strategy_class == HashImportStrategy
|
24
|
+
Importer::Mapper.build_mapper(MockImportable) do |mapping|
|
25
|
+
mapping.required_mapping 'Field1', 'field_1'
|
26
|
+
mapping.required_mapping 'Field2', 'field_2'
|
27
|
+
mapping.optional_mapping 'Field3', 'field_3'
|
28
|
+
mapping.ignored_column 'IgnoredField'
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
class HashImportStrategy < Importer::Strategy::Base
|
36
|
+
attr_reader :imported_data
|
37
|
+
|
38
|
+
def initialize(mapper)
|
39
|
+
@imported_data = []
|
40
|
+
@mapper = mapper
|
41
|
+
end
|
42
|
+
|
43
|
+
def import_record(source_data)
|
44
|
+
mapping_result = mapper.map_input(source_data)
|
45
|
+
new_model = mapper.model_class.new(mapping_result.data)
|
46
|
+
new_model.valid?
|
47
|
+
model_errors = new_model.errors.full_messages
|
48
|
+
status = get_import_status(mapping_result, model_errors)
|
49
|
+
|
50
|
+
@imported_data << new_model unless status.errors?
|
51
|
+
|
52
|
+
status
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
class MockInput
|
58
|
+
include Enumerable
|
59
|
+
|
60
|
+
def initialize
|
61
|
+
|
62
|
+
end
|
63
|
+
def get_header
|
64
|
+
['Field1',
|
65
|
+
'Field2',
|
66
|
+
'Field3']
|
67
|
+
end
|
68
|
+
|
69
|
+
def input_identifier
|
70
|
+
'test'
|
71
|
+
end
|
72
|
+
|
73
|
+
def each
|
74
|
+
yield Importer::Input::SourceData.new('1', {'Field1' => 'datum1', 'Field2' => 'datum2'})
|
75
|
+
yield Importer::Input::SourceData.new('2', {'Field1' => 'datum2', 'Field2' => 'datum2', 'Field3' => 'datum3'})
|
76
|
+
yield Importer::Input::SourceData.new('3', {'Field1' => 'datum3', 'Field2' => 'invalid value!!!!1ONE'}) #I am INVALID!!!
|
77
|
+
yield Importer::Input::SourceData.new('4', {'Field1' => 'datum4', 'Field2' => 'datum2', 'Field3' => 'datum3', 'IgnoredField' => 'ignore me'})
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe Importer do
|
82
|
+
let(:input) { MockInput.new }
|
83
|
+
let(:model_class) { MockImportable }
|
84
|
+
let(:strategy_class) { HashImportStrategy }
|
85
|
+
let(:bad_input) do
|
86
|
+
double 'Input',
|
87
|
+
get_header: ['BadCol1', 'BadCol2', 'Field1', 'Field3'],
|
88
|
+
input_identifier: 'Test'
|
89
|
+
end
|
90
|
+
let(:simple_logger) { Importer::Logger::Simple.new }
|
91
|
+
let(:import_log) { Importer.import_data(input, model_class, strategy_class, simple_logger) }
|
92
|
+
|
93
|
+
describe '.import_data' do
|
94
|
+
it 'returns a logger instance' do
|
95
|
+
expect(import_log).to be simple_logger
|
96
|
+
end
|
97
|
+
|
98
|
+
it 'tries to import data from a valid import object' do
|
99
|
+
expect(import_log.total_imports).to be 4
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'imports valid data and does not import invalid data' do
|
103
|
+
expect(import_log.successful_imports).to be 3
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'logs invalid data' do
|
107
|
+
expect(import_log.errors?).to be_true
|
108
|
+
expect(import_log.failed_imports).to be 1
|
109
|
+
end
|
110
|
+
|
111
|
+
it 'does not import data with an invalid header' do
|
112
|
+
import_log = Importer.import_data(bad_input, model_class, strategy_class, simple_logger)
|
113
|
+
expect(import_log.errors?).to be_true
|
114
|
+
expect(import_log.fatal_error?).to be_true
|
115
|
+
expect(import_log.fatal_errors.first.message).
|
116
|
+
to match(/Invalid Input Header.+Missing Columns:\s+Field2.+Invalid Columns:\s+BadCol1.+BadCol2/)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
describe '.build_mapper' do
|
120
|
+
it 'returns a mapper with the defined mappings' do
|
121
|
+
mapper = Importer.build_mapper(MockImportable) do |mapping|
|
122
|
+
mapping.required_mapping 'Field1', 'field_1'
|
123
|
+
mapping.required_mapping 'Field2', 'field_2'
|
124
|
+
mapping.optional_mapping 'Field3', 'field_3'
|
125
|
+
mapping.ignored_column 'IgnoredField'
|
126
|
+
end
|
127
|
+
expect(mapper.required_columns).to eql(['Field1', 'Field2'])
|
128
|
+
expect(mapper.optional_columns).to eql(['Field3'])
|
129
|
+
expect(mapper.ignored_columns).to eql(['IgnoredField'])
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Importer::Logger::Base do
|
4
|
+
let(:logger){Importer::Logger::Base.new}
|
5
|
+
describe '#log_fatal' do
|
6
|
+
it 'should log a fatal error' do
|
7
|
+
logger.log_fatal('test input', 'Fatal Error')
|
8
|
+
expect(logger.fatal_errors.first).to be_a Importer::Logger::LogEntry
|
9
|
+
expect(logger.fatal_errors.first)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Importer::Logger::FatalErrorEntry do
|
4
|
+
let(:entry) { Importer::Logger::FatalErrorEntry.new('test-input', 'failure message') }
|
5
|
+
describe '#failure?' do
|
6
|
+
it 'should return true' do
|
7
|
+
expect(entry.failure?).to be_true
|
8
|
+
end
|
9
|
+
end
|
10
|
+
describe '#success?' do
|
11
|
+
it 'should return false' do
|
12
|
+
expect(entry.success?).to be_false
|
13
|
+
end
|
14
|
+
end
|
15
|
+
describe '#source_identifier' do
|
16
|
+
it 'should return `import failure`' do
|
17
|
+
expect(entry.source_identifier).to eql 'import failure'
|
18
|
+
end
|
19
|
+
end
|
20
|
+
describe '#timestamp' do
|
21
|
+
it 'should have a timestamp' do
|
22
|
+
expect(entry.timestamp).to be_a(DateTime)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
describe '#message' do
|
26
|
+
it 'should return the message it was initialized with' do
|
27
|
+
expect(entry.message).to eql('failure message')
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Importer::Logger::Simple do
|
4
|
+
|
5
|
+
let(:logger) do
|
6
|
+
Importer::Logger::Simple.new
|
7
|
+
end
|
8
|
+
|
9
|
+
describe '#log_success' do
|
10
|
+
it 'logs a success' do
|
11
|
+
logger.log_success({record_id: 1,
|
12
|
+
message: 'success'})
|
13
|
+
expect(logger.successful_imports).to eql 1
|
14
|
+
expect(logger.failed_imports).to eql 0
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe '#log_failure' do
|
19
|
+
it 'logs a failure' do
|
20
|
+
logger.log_failure({record_id: 1,
|
21
|
+
message: 'failure'})
|
22
|
+
expect(logger.successful_imports).to eql 0
|
23
|
+
expect(logger.failed_imports).to eql 1
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe '#total_imports' do
|
28
|
+
it 'returns the total number of imports' do
|
29
|
+
logger.log_success({record_id: 1,
|
30
|
+
message: 'success'})
|
31
|
+
logger.log_failure({record_id: 2,
|
32
|
+
message: 'failure'})
|
33
|
+
expect(logger.total_imports).to eql 2
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#errors?' do
|
38
|
+
it 'returns true if there are fatal errors' do
|
39
|
+
logger.log_fatal('input', 'FATAL ERROR')
|
40
|
+
expect(logger.errors?).to be_true
|
41
|
+
end
|
42
|
+
it 'returns true if there are import errors' do
|
43
|
+
logger.log_failure({record_id: 2,
|
44
|
+
message: 'failure'})
|
45
|
+
expect(logger.errors?).to be_true
|
46
|
+
end
|
47
|
+
it 'returns false if there are no errors' do
|
48
|
+
logger.log_success({record_id: 2,
|
49
|
+
message: 'failure'})
|
50
|
+
expect(logger.errors?).to be_false
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|