abstract_importer 1.2.0.rc1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -2
- data/abstract_importer.gemspec +1 -0
- data/lib/abstract_importer/base.rb +26 -3
- data/lib/abstract_importer/collection.rb +15 -0
- data/lib/abstract_importer/collection_importer.rb +8 -69
- data/lib/abstract_importer/import_options.rb +21 -35
- data/lib/abstract_importer/reporters.rb +4 -0
- data/lib/abstract_importer/reporters/base_reporter.rb +72 -0
- data/lib/abstract_importer/reporters/debug_reporter.rb +131 -0
- data/lib/abstract_importer/reporters/null_reporter.rb +19 -0
- data/lib/abstract_importer/reporters/performance_reporter.rb +103 -0
- data/lib/abstract_importer/strategies.rb +2 -0
- data/lib/abstract_importer/strategies/base.rb +30 -0
- data/lib/abstract_importer/strategies/default_strategy.rb +83 -0
- data/lib/abstract_importer/strategies/replace_strategy.rb +67 -0
- data/lib/abstract_importer/version.rb +1 -1
- data/test/importer_test.rb +79 -6
- data/test/support/mock_data_source.rb +3 -3
- data/test/test_helper.rb +3 -2
- metadata +53 -31
- data/lib/abstract_importer/reporter.rb +0 -150
metadata
CHANGED
@@ -1,167 +1,181 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: abstract_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bob Lail
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '1.3'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '4.7'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '4.7'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: rake
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
45
|
-
- -
|
59
|
+
- - ">="
|
46
60
|
- !ruby/object:Gem::Version
|
47
61
|
version: '0'
|
48
62
|
type: :development
|
49
63
|
prerelease: false
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
|
-
- -
|
66
|
+
- - ">="
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: rails
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
|
-
- -
|
73
|
+
- - ">="
|
60
74
|
- !ruby/object:Gem::Version
|
61
75
|
version: '0'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
|
-
- -
|
80
|
+
- - ">="
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: sqlite3
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- -
|
87
|
+
- - ">="
|
74
88
|
- !ruby/object:Gem::Version
|
75
89
|
version: '0'
|
76
90
|
type: :development
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- -
|
94
|
+
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: turn
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
|
-
- -
|
101
|
+
- - ">="
|
88
102
|
- !ruby/object:Gem::Version
|
89
103
|
version: '0'
|
90
104
|
type: :development
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
|
-
- -
|
108
|
+
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: pry
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
|
-
- -
|
115
|
+
- - ">="
|
102
116
|
- !ruby/object:Gem::Version
|
103
117
|
version: '0'
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
|
-
- -
|
122
|
+
- - ">="
|
109
123
|
- !ruby/object:Gem::Version
|
110
124
|
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: rr
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
114
128
|
requirements:
|
115
|
-
- -
|
129
|
+
- - ">="
|
116
130
|
- !ruby/object:Gem::Version
|
117
131
|
version: '0'
|
118
132
|
type: :development
|
119
133
|
prerelease: false
|
120
134
|
version_requirements: !ruby/object:Gem::Requirement
|
121
135
|
requirements:
|
122
|
-
- -
|
136
|
+
- - ">="
|
123
137
|
- !ruby/object:Gem::Version
|
124
138
|
version: '0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: database_cleaner
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
128
142
|
requirements:
|
129
|
-
- -
|
143
|
+
- - ">="
|
130
144
|
- !ruby/object:Gem::Version
|
131
145
|
version: '0'
|
132
146
|
type: :development
|
133
147
|
prerelease: false
|
134
148
|
version_requirements: !ruby/object:Gem::Requirement
|
135
149
|
requirements:
|
136
|
-
- -
|
150
|
+
- - ">="
|
137
151
|
- !ruby/object:Gem::Version
|
138
152
|
version: '0'
|
139
153
|
- !ruby/object:Gem::Dependency
|
140
154
|
name: simplecov
|
141
155
|
requirement: !ruby/object:Gem::Requirement
|
142
156
|
requirements:
|
143
|
-
- -
|
157
|
+
- - ">="
|
144
158
|
- !ruby/object:Gem::Version
|
145
159
|
version: '0'
|
146
160
|
type: :development
|
147
161
|
prerelease: false
|
148
162
|
version_requirements: !ruby/object:Gem::Requirement
|
149
163
|
requirements:
|
150
|
-
- -
|
164
|
+
- - ">="
|
151
165
|
- !ruby/object:Gem::Version
|
152
166
|
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: shoulda-context
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|
156
170
|
requirements:
|
157
|
-
- -
|
171
|
+
- - ">="
|
158
172
|
- !ruby/object:Gem::Version
|
159
173
|
version: '0'
|
160
174
|
type: :development
|
161
175
|
prerelease: false
|
162
176
|
version_requirements: !ruby/object:Gem::Requirement
|
163
177
|
requirements:
|
164
|
-
- -
|
178
|
+
- - ">="
|
165
179
|
- !ruby/object:Gem::Version
|
166
180
|
version: '0'
|
167
181
|
description:
|
@@ -171,8 +185,8 @@ executables: []
|
|
171
185
|
extensions: []
|
172
186
|
extra_rdoc_files: []
|
173
187
|
files:
|
174
|
-
- .gitignore
|
175
|
-
- .travis.yml
|
188
|
+
- ".gitignore"
|
189
|
+
- ".travis.yml"
|
176
190
|
- Gemfile
|
177
191
|
- LICENSE.txt
|
178
192
|
- README.md
|
@@ -185,7 +199,15 @@ files:
|
|
185
199
|
- lib/abstract_importer/id_map.rb
|
186
200
|
- lib/abstract_importer/import_options.rb
|
187
201
|
- lib/abstract_importer/import_plan.rb
|
188
|
-
- lib/abstract_importer/
|
202
|
+
- lib/abstract_importer/reporters.rb
|
203
|
+
- lib/abstract_importer/reporters/base_reporter.rb
|
204
|
+
- lib/abstract_importer/reporters/debug_reporter.rb
|
205
|
+
- lib/abstract_importer/reporters/null_reporter.rb
|
206
|
+
- lib/abstract_importer/reporters/performance_reporter.rb
|
207
|
+
- lib/abstract_importer/strategies.rb
|
208
|
+
- lib/abstract_importer/strategies/base.rb
|
209
|
+
- lib/abstract_importer/strategies/default_strategy.rb
|
210
|
+
- lib/abstract_importer/strategies/replace_strategy.rb
|
189
211
|
- lib/abstract_importer/summary.rb
|
190
212
|
- lib/abstract_importer/version.rb
|
191
213
|
- test/callback_test.rb
|
@@ -204,17 +226,17 @@ require_paths:
|
|
204
226
|
- lib
|
205
227
|
required_ruby_version: !ruby/object:Gem::Requirement
|
206
228
|
requirements:
|
207
|
-
- -
|
229
|
+
- - ">="
|
208
230
|
- !ruby/object:Gem::Version
|
209
231
|
version: '0'
|
210
232
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
211
233
|
requirements:
|
212
|
-
- -
|
234
|
+
- - ">="
|
213
235
|
- !ruby/object:Gem::Version
|
214
|
-
version:
|
236
|
+
version: '0'
|
215
237
|
requirements: []
|
216
238
|
rubyforge_project:
|
217
|
-
rubygems_version: 2.2.
|
239
|
+
rubygems_version: 2.2.2
|
218
240
|
signing_key:
|
219
241
|
specification_version: 4
|
220
242
|
summary: Provides services for the mass-import of complex relational data
|
@@ -1,150 +0,0 @@
|
|
1
|
-
module AbstractImporter
|
2
|
-
class Reporter
|
3
|
-
|
4
|
-
def initialize(io, production)
|
5
|
-
@io = io
|
6
|
-
@notices = {}
|
7
|
-
@errors = {}
|
8
|
-
@production = production
|
9
|
-
@invalid_params = {}
|
10
|
-
end
|
11
|
-
|
12
|
-
attr_reader :io, :invalid_params
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
def production?
|
17
|
-
@production
|
18
|
-
end
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
def start_all(importer)
|
23
|
-
status "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
|
24
|
-
end
|
25
|
-
|
26
|
-
def finish_all(importer, ms)
|
27
|
-
print_invalid_params
|
28
|
-
status "\n\nFinished in #{distance_of_time(ms)}"
|
29
|
-
end
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
def finish_setup(ms)
|
34
|
-
status "Setup took #{distance_of_time(ms)}\n"
|
35
|
-
end
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
def start_collection(collection)
|
40
|
-
status "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
|
41
|
-
@notices = {}
|
42
|
-
@errors = {}
|
43
|
-
end
|
44
|
-
|
45
|
-
def finish_collection(collection, summary)
|
46
|
-
print_summary summary, collection.name
|
47
|
-
print_messages @notices, "Notices"
|
48
|
-
print_messages @errors, "Errors"
|
49
|
-
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
def record_created(record)
|
54
|
-
io.print "." unless production?
|
55
|
-
end
|
56
|
-
|
57
|
-
def record_failed(record, hash)
|
58
|
-
io.print "×" unless production?
|
59
|
-
|
60
|
-
error_messages = invalid_params[record.class.name] ||= {}
|
61
|
-
record.errors.full_messages.each do |error_message|
|
62
|
-
error_messages[error_message] = hash unless error_messages.key?(error_message)
|
63
|
-
count_error(error_message)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
def status(s)
|
70
|
-
io.puts s
|
71
|
-
end
|
72
|
-
|
73
|
-
def stat(s)
|
74
|
-
io.puts " #{s}"
|
75
|
-
end
|
76
|
-
alias :info :stat
|
77
|
-
|
78
|
-
def file(s)
|
79
|
-
io.puts s.inspect
|
80
|
-
end
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
def count_notice(message)
|
85
|
-
return if production?
|
86
|
-
@notices[message] = (@notices[message] || 0) + 1
|
87
|
-
end
|
88
|
-
|
89
|
-
def count_error(message)
|
90
|
-
@errors[message] = (@errors[message] || 0) + 1
|
91
|
-
end
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
private
|
96
|
-
|
97
|
-
def print_invalid_params
|
98
|
-
return if invalid_params.empty?
|
99
|
-
status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
|
100
|
-
invalid_params.each do |model_name, errors|
|
101
|
-
status "\n\n--#{model_name}#{("-"*(78 - model_name.length))}"
|
102
|
-
errors.each do |error_message, hash|
|
103
|
-
status "\n #{error_message}:\n #{hash.inspect}"
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def print_summary(summary, plural)
|
109
|
-
stat "\n #{summary.total} #{plural} were found"
|
110
|
-
if summary.total > 0
|
111
|
-
stat "#{summary.already_imported} #{plural} were imported previously"
|
112
|
-
stat "#{summary.redundant} #{plural} would create duplicates and will not be imported"
|
113
|
-
stat "#{summary.invalid} #{plural} were invalid"
|
114
|
-
stat "#{summary.skipped} #{plural} were skipped"
|
115
|
-
stat "#{summary.created} #{plural} were imported"
|
116
|
-
stat "#{distance_of_time(summary.ms)} elapsed (#{summary.average_ms.to_i}ms each)"
|
117
|
-
else
|
118
|
-
stat "#{distance_of_time(summary.ms)} elapsed"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
def print_messages(array, caption)
|
123
|
-
return if array.empty?
|
124
|
-
status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
|
125
|
-
array.each do |message, count|
|
126
|
-
stat "#{count} × #{message}"
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
def distance_of_time(milliseconds)
|
131
|
-
milliseconds = milliseconds.to_i
|
132
|
-
seconds = milliseconds / 1000
|
133
|
-
milliseconds %= 1000
|
134
|
-
minutes = seconds / 60
|
135
|
-
seconds %= 60
|
136
|
-
hours = minutes / 60
|
137
|
-
minutes %= 60
|
138
|
-
days = hours / 24
|
139
|
-
hours %= 24
|
140
|
-
|
141
|
-
time = []
|
142
|
-
time << "#{days} days" unless days.zero?
|
143
|
-
time << "#{hours} hours" unless hours.zero?
|
144
|
-
time << "#{minutes} minutes" unless minutes.zero?
|
145
|
-
time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
|
146
|
-
time.join(", ")
|
147
|
-
end
|
148
|
-
|
149
|
-
end
|
150
|
-
end
|