topographer 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +20 -0
- data/LICENSE.txt +22 -0
- data/README.md +38 -0
- data/Rakefile +1 -0
- data/lib/Topographer/exceptions.rb +4 -0
- data/lib/Topographer/importer.rb +57 -0
- data/lib/Topographer/importer/helpers.rb +18 -0
- data/lib/Topographer/importer/helpers/write_log_to_csv.rb +85 -0
- data/lib/Topographer/importer/importable.rb +5 -0
- data/lib/Topographer/importer/input.rb +5 -0
- data/lib/Topographer/importer/input/base.rb +13 -0
- data/lib/Topographer/importer/input/roo.rb +28 -0
- data/lib/Topographer/importer/input/source_data.rb +8 -0
- data/lib/Topographer/importer/logger.rb +8 -0
- data/lib/Topographer/importer/logger/base.rb +69 -0
- data/lib/Topographer/importer/logger/fatal_error_entry.rb +19 -0
- data/lib/Topographer/importer/logger/file.rb +0 -0
- data/lib/Topographer/importer/logger/log_entry.rb +34 -0
- data/lib/Topographer/importer/logger/simple.rb +27 -0
- data/lib/Topographer/importer/mapper.rb +161 -0
- data/lib/Topographer/importer/mapper/default_field_mapping.rb +22 -0
- data/lib/Topographer/importer/mapper/field_mapping.rb +55 -0
- data/lib/Topographer/importer/mapper/ignored_field_mapping.rb +10 -0
- data/lib/Topographer/importer/mapper/result.rb +21 -0
- data/lib/Topographer/importer/mapper/validation_field_mapping.rb +36 -0
- data/lib/Topographer/importer/strategy.rb +7 -0
- data/lib/Topographer/importer/strategy/base.rb +42 -0
- data/lib/Topographer/importer/strategy/create_or_update_record.rb +50 -0
- data/lib/Topographer/importer/strategy/import_new_record.rb +17 -0
- data/lib/Topographer/importer/strategy/import_status.rb +28 -0
- data/lib/Topographer/importer/strategy/update_record.rb +33 -0
- data/lib/Topographer/version.rb +3 -0
- data/lib/topographer.rb +6 -0
- data/spec/Cartographer/importer/helpers/write_log_to_csv_spec.rb +69 -0
- data/spec/Cartographer/importer/helpers_spec.rb +33 -0
- data/spec/Cartographer/importer/importable_spec.rb +13 -0
- data/spec/Cartographer/importer/importer_spec.rb +132 -0
- data/spec/Cartographer/importer/logger/base_spec.rb +12 -0
- data/spec/Cartographer/importer/logger/fatal_error_entry_spec.rb +31 -0
- data/spec/Cartographer/importer/logger/simple_spec.rb +53 -0
- data/spec/Cartographer/importer/mapper/default_field_mapping_spec.rb +41 -0
- data/spec/Cartographer/importer/mapper/field_mapping_spec.rb +126 -0
- data/spec/Cartographer/importer/mapper/validation_field_mapping_spec.rb +42 -0
- data/spec/Cartographer/importer/mapper_spec.rb +318 -0
- data/spec/Cartographer/importer/strategy/base_spec.rb +43 -0
- data/spec/Cartographer/importer/strategy/create_or_update_record_spec.rb +46 -0
- data/spec/Cartographer/importer/strategy/import_new_records_spec.rb +66 -0
- data/spec/Cartographer/importer/strategy/import_status_spec.rb +24 -0
- data/spec/Cartographer/importer/strategy/mapped_model.rb +41 -0
- data/spec/Cartographer/importer/strategy/update_record_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -0
- data/topographer.gemspec +26 -0
- metadata +175 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 0f240a307703861f83b040a054ad1f89ae397823
|
4
|
+
data.tar.gz: 3fab2dc9130cd3507b2f7fccb169ab1082847b38
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f5a4c37297d5ee4c31f125b39e371067b2b78bff094a4052a3e48c690a6969ade6cc752542652a7bcc37ca4eca76d115bc1eeab41cecec4b6e154bfc794c174e
|
7
|
+
data.tar.gz: 6f18ca76a190026fc57a216ede46e51d2354df44de0b8c6270c1b224e230b712a2151af4aec806b0b5c54a33d86b2fd26be6e2dd8fcb2186eaf56830e58764bb
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2014 SciMed Solutions
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 SciMed Solutions
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# Topographer
|
2
|
+
|
3
|
+
Topographer is a gem that provides functionality to conveniently import from various data sources into
|
4
|
+
ActiveRecord or other ORM systems. This accomplished by defining an input wrapper, a mapping from input data to
|
5
|
+
models, and a strategy for use in persisting data.
|
6
|
+
|
7
|
+
Check back soon for more documentation.
|
8
|
+
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
Add this line to your application's Gemfile:
|
13
|
+
|
14
|
+
gem 'Topographer'
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install Topographer
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Check back soon for detailed usage instructions
|
27
|
+
|
28
|
+
## Contributing
|
29
|
+
|
30
|
+
1. Fork it
|
31
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
32
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
33
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
34
|
+
5. Create new Pull Request
|
35
|
+
=======
|
36
|
+
Topographer
|
37
|
+
===========
|
38
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class Importer
|
2
|
+
require_relative 'importer/mapper'
|
3
|
+
require_relative 'importer/strategy'
|
4
|
+
require_relative 'importer/importable'
|
5
|
+
require_relative 'importer/logger'
|
6
|
+
require_relative 'importer/input'
|
7
|
+
require_relative 'importer/helpers'
|
8
|
+
|
9
|
+
attr_reader :logger
|
10
|
+
|
11
|
+
def self.build_mapper(model_class, &mapper_definition)
|
12
|
+
Mapper.build_mapper(model_class, &mapper_definition)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.import_data(input, import_class, strategy_class, logger, options = {})
|
16
|
+
dry_run = options.fetch(:dry_run, false)
|
17
|
+
importer = new(input, import_class, strategy_class, logger, dry_run)
|
18
|
+
importer.logger
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(input, import_class, strategy_class, logger, dry_run)
|
22
|
+
@logger = logger
|
23
|
+
mapper = import_class.get_mapper(strategy_class)
|
24
|
+
valid_header = mapper.input_structure_valid?(input.get_header)
|
25
|
+
|
26
|
+
if valid_header
|
27
|
+
strategy = strategy_class.new(mapper)
|
28
|
+
strategy.dry_run = dry_run
|
29
|
+
import_data(strategy, input, mapper.model_class.name)
|
30
|
+
else
|
31
|
+
log_invalid_header(input, mapper)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def import_data(strategy, input, import_class)
|
36
|
+
input.each do |data|
|
37
|
+
status = strategy.import_record(data)
|
38
|
+
log_entry = Logger::LogEntry.new(input.input_identifier, import_class, status)
|
39
|
+
@logger.log_import(log_entry)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def log_invalid_header(input, mapper)
|
46
|
+
@logger.log_fatal input.input_identifier,
|
47
|
+
invalid_header_message(mapper)
|
48
|
+
end
|
49
|
+
|
50
|
+
def invalid_header_message(mapper)
|
51
|
+
'Invalid Input Header - Missing Columns: ' +
|
52
|
+
mapper.missing_columns.join(', ') +
|
53
|
+
' Invalid Columns: ' +
|
54
|
+
mapper.bad_columns.join(', ')
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Importer::Helpers
|
2
|
+
require_relative 'helpers/write_log_to_csv'
|
3
|
+
|
4
|
+
def boolify(word)
|
5
|
+
return nil if word.nil?
|
6
|
+
|
7
|
+
case word.downcase
|
8
|
+
when 'yes'
|
9
|
+
true
|
10
|
+
when 'no'
|
11
|
+
false
|
12
|
+
when 'true'
|
13
|
+
true
|
14
|
+
when 'false'
|
15
|
+
false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
require 'csv'
|
3
|
+
class Importer::Helpers::WriteLogToCSV
|
4
|
+
include Singleton
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
|
8
|
+
end
|
9
|
+
|
10
|
+
def write_log_to_csv(log, output_file_path, options = {})
|
11
|
+
@log = log
|
12
|
+
@write_all = options.fetch(:write_all, true)
|
13
|
+
CSV.open(output_file_path, 'wb') do |csv_file|
|
14
|
+
csv_file << get_detail_header
|
15
|
+
csv_file << get_details
|
16
|
+
csv_file << get_log_header if @log.entries?
|
17
|
+
@log.all_entries.each do |entry|
|
18
|
+
if entry.failure? || @write_all
|
19
|
+
|
20
|
+
csv_file << format_log_entry(entry)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def get_detail_header
|
29
|
+
if @write_all
|
30
|
+
['Fatal Errors', 'Total Imports', 'Successful Imports', 'Failed Imports']
|
31
|
+
else
|
32
|
+
['Fatal Errors', 'Failed Imports']
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_details
|
37
|
+
details = []
|
38
|
+
details << ((@log.fatal_error?) ? @log.fatal_errors.size : 'None')
|
39
|
+
if @write_all
|
40
|
+
details << @log.total_imports
|
41
|
+
details << @log.successful_imports
|
42
|
+
end
|
43
|
+
details << @log.failed_imports
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_log_header
|
47
|
+
header = []
|
48
|
+
header << 'Input Identifier'
|
49
|
+
header << 'Source Identifier'
|
50
|
+
header << 'Model Class'
|
51
|
+
header << 'Timestamp'
|
52
|
+
header << 'Status'
|
53
|
+
header << 'Message'
|
54
|
+
header << 'Details'
|
55
|
+
end
|
56
|
+
|
57
|
+
def format_log_entry(log_entry)
|
58
|
+
entry = []
|
59
|
+
entry << log_entry.input_identifier
|
60
|
+
entry << log_entry.source_identifier
|
61
|
+
entry << log_entry.model_name
|
62
|
+
entry << log_entry.timestamp.strftime('%F - %T:%L')
|
63
|
+
entry << ((log_entry.success?) ? 'Success' : 'Failure')
|
64
|
+
entry << log_entry.message
|
65
|
+
entry << format_log_entry_details(log_entry.details)
|
66
|
+
end
|
67
|
+
|
68
|
+
def format_log_entry_details(details)
|
69
|
+
if details
|
70
|
+
formatted_details = []
|
71
|
+
details.keys.each do |key|
|
72
|
+
detail_string = ''
|
73
|
+
detail_messages = Array(details[key])
|
74
|
+
if detail_messages.any?
|
75
|
+
detail_string << key.to_s.capitalize << ': '
|
76
|
+
detail_string << detail_messages.join(', ')
|
77
|
+
end
|
78
|
+
formatted_details << detail_string unless detail_string.empty?
|
79
|
+
end
|
80
|
+
formatted_details.join("; ")
|
81
|
+
else
|
82
|
+
''
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class Importer::Input::Roo < Importer::Input::Base
|
2
|
+
include Enumerable
|
3
|
+
|
4
|
+
def initialize(roo_sheet, header_row=1, data_row=2)
|
5
|
+
@sheet = roo_sheet
|
6
|
+
@header = @sheet.row(header_row).map(&:strip)
|
7
|
+
@start_data_row = data_row
|
8
|
+
@end_data_row = @sheet.last_row
|
9
|
+
end
|
10
|
+
|
11
|
+
def get_header
|
12
|
+
@header
|
13
|
+
end
|
14
|
+
|
15
|
+
def input_identifier
|
16
|
+
#This is apparently how you get the name of the sheet...this makes me sad
|
17
|
+
@sheet.default_sheet
|
18
|
+
end
|
19
|
+
|
20
|
+
def each
|
21
|
+
@start_data_row.upto @end_data_row do |row_number|
|
22
|
+
data = @sheet.row(row_number)
|
23
|
+
source_identifier = "Row: #{row_number}"
|
24
|
+
yield Importer::Input::SourceData.new(source_identifier,
|
25
|
+
Hash[@header.zip(data)])
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
class Importer::Logger::Base
|
2
|
+
|
3
|
+
attr_reader :fatal_errors
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@fatal_errors = []
|
7
|
+
end
|
8
|
+
|
9
|
+
def successes
|
10
|
+
raise NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
def failures
|
14
|
+
raise NotImplementedError
|
15
|
+
end
|
16
|
+
|
17
|
+
def log_import(log_entry)
|
18
|
+
if log_entry.success?
|
19
|
+
log_success(log_entry)
|
20
|
+
else
|
21
|
+
log_failure(log_entry)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def log_success(log_entry)
|
26
|
+
raise NotImplementedError
|
27
|
+
end
|
28
|
+
|
29
|
+
def log_failure(log_entry)
|
30
|
+
raise NotImplementedError
|
31
|
+
end
|
32
|
+
|
33
|
+
def log_fatal(source, message)
|
34
|
+
@fatal_errors << Importer::Logger::FatalErrorEntry.new(source, message)
|
35
|
+
end
|
36
|
+
|
37
|
+
def successful_imports
|
38
|
+
raise NotImplementedError
|
39
|
+
end
|
40
|
+
|
41
|
+
def failed_imports
|
42
|
+
raise NotImplementedError
|
43
|
+
end
|
44
|
+
|
45
|
+
def entries?
|
46
|
+
total_imports > 0
|
47
|
+
end
|
48
|
+
|
49
|
+
def total_imports
|
50
|
+
(successful_imports + failed_imports)
|
51
|
+
end
|
52
|
+
|
53
|
+
def all_entries
|
54
|
+
(successes + failures + fatal_errors).sort {|a, b| a.timestamp <=> b.timestamp}
|
55
|
+
end
|
56
|
+
|
57
|
+
def errors?
|
58
|
+
fatal_error? || failed_imports > 0
|
59
|
+
end
|
60
|
+
|
61
|
+
def fatal_error?
|
62
|
+
@fatal_errors.any?
|
63
|
+
end
|
64
|
+
|
65
|
+
def save
|
66
|
+
raise NotImplementedError
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class Importer::Logger::FatalErrorEntry < Importer::Logger::LogEntry
|
2
|
+
attr_reader :message, :timestamp, :model_name
|
3
|
+
|
4
|
+
def initialize(input_identifier, message)
|
5
|
+
@timestamp = DateTime.now
|
6
|
+
@input_identifier = input_identifier
|
7
|
+
@model_name = 'N/A'
|
8
|
+
@message = message
|
9
|
+
end
|
10
|
+
def source_identifier
|
11
|
+
'import failure'
|
12
|
+
end
|
13
|
+
def details
|
14
|
+
{}
|
15
|
+
end
|
16
|
+
def failure?
|
17
|
+
true
|
18
|
+
end
|
19
|
+
end
|