topographer 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/Gemfile +4 -0
  4. data/LICENSE +20 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +38 -0
  7. data/Rakefile +1 -0
  8. data/lib/Topographer/exceptions.rb +4 -0
  9. data/lib/Topographer/importer.rb +57 -0
  10. data/lib/Topographer/importer/helpers.rb +18 -0
  11. data/lib/Topographer/importer/helpers/write_log_to_csv.rb +85 -0
  12. data/lib/Topographer/importer/importable.rb +5 -0
  13. data/lib/Topographer/importer/input.rb +5 -0
  14. data/lib/Topographer/importer/input/base.rb +13 -0
  15. data/lib/Topographer/importer/input/roo.rb +28 -0
  16. data/lib/Topographer/importer/input/source_data.rb +8 -0
  17. data/lib/Topographer/importer/logger.rb +8 -0
  18. data/lib/Topographer/importer/logger/base.rb +69 -0
  19. data/lib/Topographer/importer/logger/fatal_error_entry.rb +19 -0
  20. data/lib/Topographer/importer/logger/file.rb +0 -0
  21. data/lib/Topographer/importer/logger/log_entry.rb +34 -0
  22. data/lib/Topographer/importer/logger/simple.rb +27 -0
  23. data/lib/Topographer/importer/mapper.rb +161 -0
  24. data/lib/Topographer/importer/mapper/default_field_mapping.rb +22 -0
  25. data/lib/Topographer/importer/mapper/field_mapping.rb +55 -0
  26. data/lib/Topographer/importer/mapper/ignored_field_mapping.rb +10 -0
  27. data/lib/Topographer/importer/mapper/result.rb +21 -0
  28. data/lib/Topographer/importer/mapper/validation_field_mapping.rb +36 -0
  29. data/lib/Topographer/importer/strategy.rb +7 -0
  30. data/lib/Topographer/importer/strategy/base.rb +42 -0
  31. data/lib/Topographer/importer/strategy/create_or_update_record.rb +50 -0
  32. data/lib/Topographer/importer/strategy/import_new_record.rb +17 -0
  33. data/lib/Topographer/importer/strategy/import_status.rb +28 -0
  34. data/lib/Topographer/importer/strategy/update_record.rb +33 -0
  35. data/lib/Topographer/version.rb +3 -0
  36. data/lib/topographer.rb +6 -0
  37. data/spec/Cartographer/importer/helpers/write_log_to_csv_spec.rb +69 -0
  38. data/spec/Cartographer/importer/helpers_spec.rb +33 -0
  39. data/spec/Cartographer/importer/importable_spec.rb +13 -0
  40. data/spec/Cartographer/importer/importer_spec.rb +132 -0
  41. data/spec/Cartographer/importer/logger/base_spec.rb +12 -0
  42. data/spec/Cartographer/importer/logger/fatal_error_entry_spec.rb +31 -0
  43. data/spec/Cartographer/importer/logger/simple_spec.rb +53 -0
  44. data/spec/Cartographer/importer/mapper/default_field_mapping_spec.rb +41 -0
  45. data/spec/Cartographer/importer/mapper/field_mapping_spec.rb +126 -0
  46. data/spec/Cartographer/importer/mapper/validation_field_mapping_spec.rb +42 -0
  47. data/spec/Cartographer/importer/mapper_spec.rb +318 -0
  48. data/spec/Cartographer/importer/strategy/base_spec.rb +43 -0
  49. data/spec/Cartographer/importer/strategy/create_or_update_record_spec.rb +46 -0
  50. data/spec/Cartographer/importer/strategy/import_new_records_spec.rb +66 -0
  51. data/spec/Cartographer/importer/strategy/import_status_spec.rb +24 -0
  52. data/spec/Cartographer/importer/strategy/mapped_model.rb +41 -0
  53. data/spec/Cartographer/importer/strategy/update_record_spec.rb +45 -0
  54. data/spec/spec_helper.rb +1 -0
  55. data/topographer.gemspec +26 -0
  56. metadata +175 -0
@@ -0,0 +1,17 @@
1
+ class Importer::Strategy::ImportNewRecord < Importer::Strategy::Base
2
+
3
+ def import_record (source_data)
4
+ mapping_result = mapper.map_input(source_data)
5
+ new_model = mapper.model_class.new(mapping_result.data)
6
+ new_model.valid?
7
+ model_errors = new_model.errors.full_messages
8
+ status = get_import_status(mapping_result, model_errors)
9
+
10
+ new_model.save if should_persist_import?(status)
11
+
12
+ status
13
+ end
14
+
15
+ end
16
+
17
+
@@ -0,0 +1,28 @@
1
+ class Importer::Strategy::ImportStatus
2
+ attr_reader :errors, :input_identifier, :timestamp
3
+ attr_accessor :message
4
+
5
+ def initialize(input_identifier)
6
+ @input_identifier = input_identifier
7
+ @errors = {mapping: [],
8
+ validation: []}
9
+
10
+ end
11
+
12
+ def set_timestamp
13
+ @timestamp ||= DateTime.now
14
+ end
15
+
16
+ def add_error(error_source, error)
17
+ errors[error_source] << error
18
+ end
19
+
20
+ def error_count
21
+ errors.values.flatten.length
22
+ end
23
+
24
+ def errors?
25
+ errors.values.flatten.any?
26
+ end
27
+
28
+ end
@@ -0,0 +1,33 @@
1
+ class Importer::Strategy::UpdateRecord < Importer::Strategy::Base
2
+
3
+ def import_record (source_data)
4
+ mapping_result = mapper.map_input(source_data)
5
+
6
+ search_params = mapping_result.data.slice(*mapper.key_fields)
7
+ model_instance = mapper.model_class.where(search_params).first
8
+
9
+ if model_instance
10
+ model_instance.attributes = mapping_result.data
11
+ model_instance.valid?
12
+ model_errors = model_instance.errors.full_messages
13
+ status = get_import_status(mapping_result, model_errors)
14
+
15
+ model_instance.save if should_persist_import?(status)
16
+ else
17
+ status = get_import_status(mapping_result, ["Record not found with params: #{search_params.to_yaml}"])
18
+ end
19
+
20
+ status
21
+ end
22
+
23
+ def success_message
24
+ 'Updated'
25
+ end
26
+
27
+ def failure_message
28
+ 'Unable to update from import'
29
+ end
30
+
31
+ end
32
+
33
+
@@ -0,0 +1,3 @@
1
+ module Topographer
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,6 @@
1
+ require 'Topographer/version'
2
+
3
+ module Topographer
4
+ require 'Topographer/exceptions'
5
+ require 'Topographer/importer'
6
+ end
@@ -0,0 +1,69 @@
1
+ require 'spec_helper'
2
+
3
+ describe Importer::Helpers::WriteLogToCSV do
4
+ let(:successful_entry) do
5
+ status = Importer::Strategy::ImportStatus.new('test-input')
6
+ status.set_timestamp
7
+ Importer::Logger::LogEntry.new('test-input', 'TestModel', status)
8
+ end
9
+ let(:failed_entry) do
10
+ status = Importer::Strategy::ImportStatus.new('test-input')
11
+ status.set_timestamp
12
+ (1+rand(6)).times do |n|
13
+ status.add_error(:mapping, "Test error #{n}")
14
+ end
15
+ Importer::Logger::LogEntry.new('test-input', 'TestModel', status)
16
+ end
17
+ let(:fatal_error) do
18
+ Importer::Logger::FatalErrorEntry.new('test-input', 'FATAL ERROR')
19
+ end
20
+ let(:successes) do
21
+ successes = []
22
+ 2.times do
23
+ successes << successful_entry
24
+ end
25
+ successes
26
+ end
27
+ let(:failures) do
28
+ failures = []
29
+ 3.times do
30
+ failures << failed_entry
31
+ end
32
+ failures
33
+ end
34
+ let(:fatal_errors) do
35
+ errors = []
36
+ 4.times do
37
+ errors << fatal_error
38
+ end
39
+ errors
40
+ end
41
+ let(:logger) do
42
+ double 'Logger::Base',
43
+ successful_imports: 2,
44
+ failed_imports: 3,
45
+ total_imports: 5,
46
+ fatal_error?: false,
47
+ errors?: true,
48
+ successes: successes,
49
+ failures: failures,
50
+ fatal_errors: fatal_errors,
51
+ entries?: true,
52
+ all_entries: (successes + failures + fatal_errors)
53
+ end
54
+
55
+ describe '#write_log_to_csv' do
56
+ it 'should write a passed logger instance to a CSV file' do
57
+ file = double('file')
58
+ CSV.should_receive(:open).with('fake_file_path', 'wb').and_yield(file)
59
+ file.should_receive(:<<).exactly(12).times
60
+ Importer::Helpers::WriteLogToCSV.instance.write_log_to_csv(logger, 'fake_file_path', write_all: true)
61
+ end
62
+ it 'should only write failures and fatal errors if write_all is false' do
63
+ file = double('file')
64
+ CSV.should_receive(:open).with('fake_file_path', 'wb').and_yield(file)
65
+ file.should_receive(:<<).exactly(10).times
66
+ Importer::Helpers::WriteLogToCSV.instance.write_log_to_csv(logger, 'fake_file_path', write_all: false)
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+
3
+ class TestImportable
4
+ extend Importer::Helpers
5
+ end
6
+
7
+ describe Importer::Helpers do
8
+ describe ".boolify" do
9
+ it "returns true if given 'Yes'" do
10
+ expect(TestImportable.boolify('Yes')).to eql true
11
+ end
12
+
13
+ it "returns true if given 'True'" do
14
+ expect(TestImportable.boolify('True')).to eql true
15
+ end
16
+
17
+ it "returns false if given 'No'" do
18
+ expect(TestImportable.boolify('No')).to eql false
19
+ end
20
+
21
+ it "returns false if given 'False'" do
22
+ expect(TestImportable.boolify('False')).to eql false
23
+ end
24
+
25
+ it 'returns nil if the type is unknown' do
26
+ expect(TestImportable.boolify('Unknown')).to be_nil
27
+ end
28
+
29
+ it 'returns nil if it is given a nil' do
30
+ expect(TestImportable.boolify(nil)).to be_nil
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,13 @@
1
+ require 'spec_helper'
2
+
3
+ class TestImportable
4
+ extend Importer::Importable
5
+ end
6
+
7
+ describe Importer::Importable do
8
+ describe "#get_mapper" do
9
+ it 'should raise NotImplementedError' do
10
+ expect { TestImportable.get_mapper(nil) }.to raise_error(NotImplementedError)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,132 @@
1
+ require 'spec_helper'
2
+ require 'ostruct'
3
+
4
+ class MockImportable < OpenStruct
5
+ include Importer::Importable
6
+
7
+ def self.create(params)
8
+ self.new(params)
9
+ end
10
+
11
+ def valid?
12
+ self.errors = OpenStruct.new(full_messages: [])
13
+ if field_2 == 'datum2'
14
+ true
15
+ else
16
+ self.errors = OpenStruct.new(full_messages: ['Field 2 is not datum2'])
17
+ false
18
+ end
19
+ end
20
+
21
+ def self.get_mapper(strategy_class)
22
+ case
23
+ when strategy_class == HashImportStrategy
24
+ Importer::Mapper.build_mapper(MockImportable) do |mapping|
25
+ mapping.required_mapping 'Field1', 'field_1'
26
+ mapping.required_mapping 'Field2', 'field_2'
27
+ mapping.optional_mapping 'Field3', 'field_3'
28
+ mapping.ignored_column 'IgnoredField'
29
+ end
30
+ end
31
+ end
32
+
33
+ end
34
+
35
+ class HashImportStrategy < Importer::Strategy::Base
36
+ attr_reader :imported_data
37
+
38
+ def initialize(mapper)
39
+ @imported_data = []
40
+ @mapper = mapper
41
+ end
42
+
43
+ def import_record(source_data)
44
+ mapping_result = mapper.map_input(source_data)
45
+ new_model = mapper.model_class.new(mapping_result.data)
46
+ new_model.valid?
47
+ model_errors = new_model.errors.full_messages
48
+ status = get_import_status(mapping_result, model_errors)
49
+
50
+ @imported_data << new_model unless status.errors?
51
+
52
+ status
53
+ end
54
+
55
+ end
56
+
57
+ class MockInput
58
+ include Enumerable
59
+
60
+ def initialize
61
+
62
+ end
63
+ def get_header
64
+ ['Field1',
65
+ 'Field2',
66
+ 'Field3']
67
+ end
68
+
69
+ def input_identifier
70
+ 'test'
71
+ end
72
+
73
+ def each
74
+ yield Importer::Input::SourceData.new('1', {'Field1' => 'datum1', 'Field2' => 'datum2'})
75
+ yield Importer::Input::SourceData.new('2', {'Field1' => 'datum2', 'Field2' => 'datum2', 'Field3' => 'datum3'})
76
+ yield Importer::Input::SourceData.new('3', {'Field1' => 'datum3', 'Field2' => 'invalid value!!!!1ONE'}) #I am INVALID!!!
77
+ yield Importer::Input::SourceData.new('4', {'Field1' => 'datum4', 'Field2' => 'datum2', 'Field3' => 'datum3', 'IgnoredField' => 'ignore me'})
78
+ end
79
+ end
80
+
81
+ describe Importer do
82
+ let(:input) { MockInput.new }
83
+ let(:model_class) { MockImportable }
84
+ let(:strategy_class) { HashImportStrategy }
85
+ let(:bad_input) do
86
+ double 'Input',
87
+ get_header: ['BadCol1', 'BadCol2', 'Field1', 'Field3'],
88
+ input_identifier: 'Test'
89
+ end
90
+ let(:simple_logger) { Importer::Logger::Simple.new }
91
+ let(:import_log) { Importer.import_data(input, model_class, strategy_class, simple_logger) }
92
+
93
+ describe '.import_data' do
94
+ it 'returns a logger instance' do
95
+ expect(import_log).to be simple_logger
96
+ end
97
+
98
+ it 'tries to import data from a valid import object' do
99
+ expect(import_log.total_imports).to be 4
100
+ end
101
+
102
+ it 'imports valid data and does not import invalid data' do
103
+ expect(import_log.successful_imports).to be 3
104
+ end
105
+
106
+ it 'logs invalid data' do
107
+ expect(import_log.errors?).to be_true
108
+ expect(import_log.failed_imports).to be 1
109
+ end
110
+
111
+ it 'does not import data with an invalid header' do
112
+ import_log = Importer.import_data(bad_input, model_class, strategy_class, simple_logger)
113
+ expect(import_log.errors?).to be_true
114
+ expect(import_log.fatal_error?).to be_true
115
+ expect(import_log.fatal_errors.first.message).
116
+ to match(/Invalid Input Header.+Missing Columns:\s+Field2.+Invalid Columns:\s+BadCol1.+BadCol2/)
117
+ end
118
+ end
119
+ describe '.build_mapper' do
120
+ it 'returns a mapper with the defined mappings' do
121
+ mapper = Importer.build_mapper(MockImportable) do |mapping|
122
+ mapping.required_mapping 'Field1', 'field_1'
123
+ mapping.required_mapping 'Field2', 'field_2'
124
+ mapping.optional_mapping 'Field3', 'field_3'
125
+ mapping.ignored_column 'IgnoredField'
126
+ end
127
+ expect(mapper.required_columns).to eql(['Field1', 'Field2'])
128
+ expect(mapper.optional_columns).to eql(['Field3'])
129
+ expect(mapper.ignored_columns).to eql(['IgnoredField'])
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,12 @@
1
+ require 'spec_helper'
2
+
3
+ describe Importer::Logger::Base do
4
+ let(:logger){Importer::Logger::Base.new}
5
+ describe '#log_fatal' do
6
+ it 'should log a fatal error' do
7
+ logger.log_fatal('test input', 'Fatal Error')
8
+ expect(logger.fatal_errors.first).to be_a Importer::Logger::LogEntry
9
+ expect(logger.fatal_errors.first)
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+
3
+ describe Importer::Logger::FatalErrorEntry do
4
+ let(:entry) { Importer::Logger::FatalErrorEntry.new('test-input', 'failure message') }
5
+ describe '#failure?' do
6
+ it 'should return true' do
7
+ expect(entry.failure?).to be_true
8
+ end
9
+ end
10
+ describe '#success?' do
11
+ it 'should return false' do
12
+ expect(entry.success?).to be_false
13
+ end
14
+ end
15
+ describe '#source_identifier' do
16
+ it 'should return `import failure`' do
17
+ expect(entry.source_identifier).to eql 'import failure'
18
+ end
19
+ end
20
+ describe '#timestamp' do
21
+ it 'should have a timestamp' do
22
+ expect(entry.timestamp).to be_a(DateTime)
23
+ end
24
+ end
25
+ describe '#message' do
26
+ it 'should return the message it was initialized with' do
27
+ expect(entry.message).to eql('failure message')
28
+ end
29
+ end
30
+ end
31
+
@@ -0,0 +1,53 @@
1
+ require 'spec_helper'
2
+
3
+ describe Importer::Logger::Simple do
4
+
5
+ let(:logger) do
6
+ Importer::Logger::Simple.new
7
+ end
8
+
9
+ describe '#log_success' do
10
+ it 'logs a success' do
11
+ logger.log_success({record_id: 1,
12
+ message: 'success'})
13
+ expect(logger.successful_imports).to eql 1
14
+ expect(logger.failed_imports).to eql 0
15
+ end
16
+ end
17
+
18
+ describe '#log_failure' do
19
+ it 'logs a failure' do
20
+ logger.log_failure({record_id: 1,
21
+ message: 'failure'})
22
+ expect(logger.successful_imports).to eql 0
23
+ expect(logger.failed_imports).to eql 1
24
+ end
25
+ end
26
+
27
+ describe '#total_imports' do
28
+ it 'returns the total number of imports' do
29
+ logger.log_success({record_id: 1,
30
+ message: 'success'})
31
+ logger.log_failure({record_id: 2,
32
+ message: 'failure'})
33
+ expect(logger.total_imports).to eql 2
34
+ end
35
+ end
36
+
37
+ describe '#errors?' do
38
+ it 'returns true if there are fatal errors' do
39
+ logger.log_fatal('input', 'FATAL ERROR')
40
+ expect(logger.errors?).to be_true
41
+ end
42
+ it 'returns true if there are import errors' do
43
+ logger.log_failure({record_id: 2,
44
+ message: 'failure'})
45
+ expect(logger.errors?).to be_true
46
+ end
47
+ it 'returns false if there are no errors' do
48
+ logger.log_success({record_id: 2,
49
+ message: 'failure'})
50
+ expect(logger.errors?).to be_false
51
+ end
52
+ end
53
+ end