abstract_importer 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/.travis.yml +5 -3
- data/abstract_importer.gemspec +2 -1
- data/lib/abstract_importer/base.rb +63 -75
- data/lib/abstract_importer/collection_importer.rb +34 -33
- data/lib/abstract_importer/id_map.rb +22 -14
- data/lib/abstract_importer/import_options.rb +2 -0
- data/lib/abstract_importer/import_plan.rb +4 -4
- data/lib/abstract_importer/reporters/base_reporter.rb +22 -22
- data/lib/abstract_importer/reporters/debug_reporter.rb +36 -36
- data/lib/abstract_importer/reporters/null_reporter.rb +5 -5
- data/lib/abstract_importer/reporters/performance_reporter.rb +17 -17
- data/lib/abstract_importer/strategies.rb +1 -0
- data/lib/abstract_importer/strategies/base.rb +12 -0
- data/lib/abstract_importer/strategies/default_strategy.rb +1 -7
- data/lib/abstract_importer/strategies/insert_strategy.rb +58 -0
- data/lib/abstract_importer/summary.rb +3 -3
- data/lib/abstract_importer/version.rb +1 -1
- data/test/callback_test.rb +29 -29
- data/test/{importer_test.rb → default_strategy_test.rb} +49 -77
- data/test/insert_strategy_test.rb +82 -0
- data/test/replace_strategy_test.rb +29 -0
- data/test/support/mock_data_source.rb +10 -10
- data/test/support/mock_objects.rb +1 -1
- data/test/support/schema.rb +7 -6
- data/test/test_helper.rb +10 -10
- metadata +27 -7
@@ -1,32 +1,40 @@
|
|
1
1
|
module AbstractImporter
|
2
2
|
class IdMap
|
3
|
-
|
3
|
+
|
4
4
|
class IdNotMappedError < StandardError; end
|
5
|
-
|
5
|
+
|
6
6
|
def initialize
|
7
7
|
@id_map = Hash.new { |hash, key| hash[key] = {} }
|
8
8
|
end
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
def init(table_name,
|
9
|
+
|
10
|
+
|
11
|
+
|
12
|
+
def init(table_name, query)
|
13
13
|
table_name = table_name.to_sym
|
14
|
-
@id_map[table_name] =
|
14
|
+
@id_map[table_name] = {}
|
15
|
+
merge! table_name, query
|
15
16
|
end
|
16
|
-
|
17
|
+
|
18
|
+
def merge!(table_name, query)
|
19
|
+
table_name = table_name.to_sym
|
20
|
+
@id_map[table_name].merge!(query
|
21
|
+
.pluck(:id, :legacy_id)
|
22
|
+
.each_with_object({}) { |(id, legacy_id), map| map[legacy_id] = id })
|
23
|
+
end
|
24
|
+
|
17
25
|
def get(table_name)
|
18
26
|
@id_map[table_name.to_sym].dup
|
19
27
|
end
|
20
28
|
alias :[] :get
|
21
|
-
|
29
|
+
|
22
30
|
def contains?(table_name, id)
|
23
31
|
@id_map[table_name.to_sym].key?(id)
|
24
32
|
end
|
25
|
-
|
33
|
+
|
26
34
|
def <<(record)
|
27
35
|
register(record: record)
|
28
36
|
end
|
29
|
-
|
37
|
+
|
30
38
|
def register(options={})
|
31
39
|
if options.key?(:record)
|
32
40
|
record = options[:record]
|
@@ -35,17 +43,17 @@ module AbstractImporter
|
|
35
43
|
table_name = options[:table_name] if options.key?(:table_name)
|
36
44
|
legacy_id = options[:legacy_id] if options.key?(:legacy_id)
|
37
45
|
record_id = options[:record_id] if options.key?(:record_id)
|
38
|
-
|
46
|
+
|
39
47
|
table_name = table_name.to_sym
|
40
48
|
@id_map[table_name][legacy_id] = record_id
|
41
49
|
end
|
42
|
-
|
50
|
+
|
43
51
|
def apply!(legacy_id, depends_on)
|
44
52
|
return nil if legacy_id.blank?
|
45
53
|
id_map = @id_map[depends_on]
|
46
54
|
raise IdNotMappedError.new unless id_map.key?(legacy_id)
|
47
55
|
id_map[legacy_id]
|
48
56
|
end
|
49
|
-
|
57
|
+
|
50
58
|
end
|
51
59
|
end
|
@@ -1,17 +1,17 @@
|
|
1
1
|
module AbstractImporter
|
2
2
|
class ImportPlan
|
3
|
-
|
3
|
+
|
4
4
|
def initialize
|
5
5
|
@plan = {} # <-- requires Ruby 1.9's ordered hash
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
def to_h
|
9
9
|
@plan.dup
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
def method_missing(plural, &block)
|
13
13
|
@plan[plural] = block
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
end
|
17
17
|
end
|
@@ -2,52 +2,52 @@ module AbstractImporter
|
|
2
2
|
module Reporters
|
3
3
|
class BaseReporter
|
4
4
|
attr_reader :io
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(io)
|
7
7
|
@io = io
|
8
8
|
end
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
|
10
|
+
|
11
|
+
|
12
12
|
def start_all(importer)
|
13
13
|
io.puts "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
def finish_all(importer, ms)
|
17
17
|
io.puts "\n\nFinished in #{distance_of_time(ms)}"
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
def finish_setup(ms)
|
21
21
|
io.puts "Setup took #{distance_of_time(ms)}\n"
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def start_collection(collection)
|
25
25
|
io.puts "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def finish_collection(collection, summary)
|
29
29
|
end
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
|
31
|
+
|
32
|
+
|
33
33
|
def record_created(record)
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
def record_failed(record, hash)
|
37
37
|
end
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
|
39
|
+
|
40
|
+
|
41
41
|
def count_notice(message)
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
def count_error(message)
|
45
45
|
end
|
46
|
-
|
47
|
-
|
48
|
-
|
46
|
+
|
47
|
+
|
48
|
+
|
49
49
|
protected
|
50
|
-
|
50
|
+
|
51
51
|
def distance_of_time(milliseconds)
|
52
52
|
milliseconds = milliseconds.to_i
|
53
53
|
seconds = milliseconds / 1000
|
@@ -58,7 +58,7 @@ module AbstractImporter
|
|
58
58
|
minutes %= 60
|
59
59
|
days = hours / 24
|
60
60
|
hours %= 24
|
61
|
-
|
61
|
+
|
62
62
|
time = []
|
63
63
|
time << "#{days} days" unless days.zero?
|
64
64
|
time << "#{hours} hours" unless hours.zero?
|
@@ -66,7 +66,7 @@ module AbstractImporter
|
|
66
66
|
time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
|
67
67
|
time.join(", ")
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
end
|
71
71
|
end
|
72
72
|
end
|
@@ -2,97 +2,97 @@ module AbstractImporter
|
|
2
2
|
module Reporters
|
3
3
|
class DebugReporter < BaseReporter
|
4
4
|
attr_reader :invalid_params
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(io)
|
7
7
|
super
|
8
8
|
@notices = {}
|
9
9
|
@errors = {}
|
10
10
|
@invalid_params = {}
|
11
11
|
end
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
|
13
|
+
|
14
|
+
|
15
15
|
def production?
|
16
16
|
Rails.env.production?
|
17
17
|
end
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
|
19
|
+
|
20
|
+
|
21
21
|
def start_all(importer)
|
22
22
|
super
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def finish_all(importer, ms)
|
26
26
|
print_invalid_params
|
27
27
|
super
|
28
28
|
end
|
29
|
-
|
30
|
-
|
31
|
-
|
29
|
+
|
30
|
+
|
31
|
+
|
32
32
|
def finish_setup(ms)
|
33
33
|
super
|
34
34
|
end
|
35
|
-
|
36
|
-
|
37
|
-
|
35
|
+
|
36
|
+
|
37
|
+
|
38
38
|
def start_collection(collection)
|
39
39
|
super
|
40
40
|
@notices = {}
|
41
41
|
@errors = {}
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
def finish_collection(collection, summary)
|
45
45
|
print_summary summary, collection.name
|
46
46
|
print_messages @notices, "Notices"
|
47
47
|
print_messages @errors, "Errors"
|
48
48
|
end
|
49
|
-
|
50
|
-
|
51
|
-
|
49
|
+
|
50
|
+
|
51
|
+
|
52
52
|
def record_created(record)
|
53
53
|
io.print "." unless production?
|
54
54
|
end
|
55
|
-
|
55
|
+
|
56
56
|
def record_failed(record, hash)
|
57
57
|
io.print "×" unless production?
|
58
|
-
|
58
|
+
|
59
59
|
error_messages = invalid_params[record.class.name] ||= {}
|
60
60
|
record.errors.full_messages.each do |error_message|
|
61
61
|
error_messages[error_message] = hash unless error_messages.key?(error_message)
|
62
62
|
count_error(error_message)
|
63
63
|
end
|
64
64
|
end
|
65
|
-
|
66
|
-
|
67
|
-
|
65
|
+
|
66
|
+
|
67
|
+
|
68
68
|
def status(s)
|
69
69
|
io.puts s
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def stat(s)
|
73
73
|
io.puts " #{s}"
|
74
74
|
end
|
75
75
|
alias :info :stat
|
76
|
-
|
76
|
+
|
77
77
|
def file(s)
|
78
78
|
io.puts s.inspect
|
79
79
|
end
|
80
|
-
|
81
|
-
|
82
|
-
|
80
|
+
|
81
|
+
|
82
|
+
|
83
83
|
def count_notice(message)
|
84
84
|
return if production?
|
85
85
|
@notices[message] = (@notices[message] || 0) + 1
|
86
86
|
end
|
87
|
-
|
87
|
+
|
88
88
|
def count_error(message)
|
89
89
|
@errors[message] = (@errors[message] || 0) + 1
|
90
90
|
end
|
91
|
-
|
92
|
-
|
93
|
-
|
91
|
+
|
92
|
+
|
93
|
+
|
94
94
|
private
|
95
|
-
|
95
|
+
|
96
96
|
def print_invalid_params
|
97
97
|
return if invalid_params.empty?
|
98
98
|
status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
|
@@ -103,7 +103,7 @@ module AbstractImporter
|
|
103
103
|
end
|
104
104
|
end
|
105
105
|
end
|
106
|
-
|
106
|
+
|
107
107
|
def print_summary(summary, plural)
|
108
108
|
stat "\n #{summary.total} #{plural} were found"
|
109
109
|
if summary.total > 0
|
@@ -117,7 +117,7 @@ module AbstractImporter
|
|
117
117
|
stat "#{distance_of_time(summary.ms)} elapsed"
|
118
118
|
end
|
119
119
|
end
|
120
|
-
|
120
|
+
|
121
121
|
def print_messages(array, caption)
|
122
122
|
return if array.empty?
|
123
123
|
status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
|
@@ -125,7 +125,7 @@ module AbstractImporter
|
|
125
125
|
stat "#{count} × #{message}"
|
126
126
|
end
|
127
127
|
end
|
128
|
-
|
128
|
+
|
129
129
|
end
|
130
130
|
end
|
131
131
|
end
|
@@ -1,19 +1,19 @@
|
|
1
1
|
module AbstractImporter
|
2
2
|
module Reporters
|
3
3
|
class NullReporter < BaseReporter
|
4
|
-
|
4
|
+
|
5
5
|
def start_all(importer)
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
def finish_all(importer, ms)
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
def finish_setup(ms)
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
def start_collection(collection)
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
@@ -4,50 +4,50 @@ module AbstractImporter
|
|
4
4
|
module Reporters
|
5
5
|
class PerformanceReporter < BaseReporter
|
6
6
|
attr_reader :sample_size
|
7
|
-
|
7
|
+
|
8
8
|
def initialize(io, options={})
|
9
9
|
super io
|
10
10
|
@sample_size = options.fetch(:sample_size, 50)
|
11
11
|
ObjectSpace.trace_object_allocations_start
|
12
12
|
end
|
13
|
-
|
14
|
-
|
13
|
+
|
14
|
+
|
15
15
|
def start_collection(collection)
|
16
16
|
super
|
17
17
|
@collection = collection
|
18
18
|
@major_gc_runs = GC.stat[:major_gc_count]
|
19
19
|
@i = 0
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
def finish_collection(collection, summary)
|
23
23
|
@collection = nil
|
24
24
|
return if @i.zero?
|
25
25
|
find_objects_holding_onto_references_to_a collection.model
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def record_created(record)
|
29
29
|
print_stats if @i % sample_size == 0
|
30
30
|
@i += 1
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
def record_failed(record, hash)
|
34
34
|
print_stats if @i % sample_size == 0
|
35
35
|
@i += 1
|
36
36
|
end
|
37
|
-
|
38
|
-
|
37
|
+
|
38
|
+
|
39
39
|
def print_stats
|
40
40
|
stats = GC.stat
|
41
41
|
objects = ObjectSpace.count_objects
|
42
42
|
puts "gc[minor]: #{stats[:minor_gc_count]}, gc[major]: #{stats[:major_gc_count]}, objects: #{objects[:TOTAL] - objects[:FREE]}, memsize: #{(ObjectSpace.memsize_of_all / 1048576.0).round(3)}MB, #{collection.name}: #{ObjectSpace.each_object(collection.model).count}"
|
43
43
|
end
|
44
|
-
|
44
|
+
|
45
45
|
private
|
46
46
|
attr_reader :collection
|
47
|
-
|
47
|
+
|
48
48
|
def find_objects_holding_onto_references_to_a(model)
|
49
49
|
GC.start
|
50
|
-
|
50
|
+
|
51
51
|
# After GC.start, all models in this collection should be
|
52
52
|
# garbage-collected unless there is a memory leak. Find one
|
53
53
|
# of the uncollected objects and figure out what is holding
|
@@ -58,11 +58,11 @@ module AbstractImporter
|
|
58
58
|
return
|
59
59
|
end
|
60
60
|
puts "\e[33mThere are #{ObjectSpace.each_object(model).count} #{model.name.tableize.gsub("_", " ")} still in memory\e[0m"
|
61
|
-
|
61
|
+
|
62
62
|
example_klass = example.class.name
|
63
63
|
example_id = example.object_id
|
64
64
|
example = nil
|
65
|
-
|
65
|
+
|
66
66
|
# Search through all objects to find ones that hold a reference
|
67
67
|
# to the model that hasn't been garbage-collected.
|
68
68
|
print "\e[90m"
|
@@ -72,7 +72,7 @@ module AbstractImporter
|
|
72
72
|
ObjectSpace.each_object do |o|
|
73
73
|
pbar.inc
|
74
74
|
next if ObjectSpace.reachable_objects_from(o).none? { |oo| oo.object_id == example_id }
|
75
|
-
|
75
|
+
|
76
76
|
message = "#{o.class.name}"
|
77
77
|
case o
|
78
78
|
when Array
|
@@ -84,12 +84,12 @@ module AbstractImporter
|
|
84
84
|
end
|
85
85
|
message << " [#{ObjectSpace.allocation_sourcefile(o)}" <<
|
86
86
|
":#{ObjectSpace.allocation_sourceline(o)}]"
|
87
|
-
|
87
|
+
|
88
88
|
objects_of_holding.push(message)
|
89
89
|
end
|
90
90
|
pbar.finish
|
91
91
|
print "\e[0m"
|
92
|
-
|
92
|
+
|
93
93
|
if objects_of_holding.none?
|
94
94
|
puts "\e[95mNo objects are holding a reference to the first one\e[0m"
|
95
95
|
else
|
@@ -97,7 +97,7 @@ module AbstractImporter
|
|
97
97
|
"\e[35m#{objects_of_holding.join("\n")}\e[0m"
|
98
98
|
end
|
99
99
|
end
|
100
|
-
|
100
|
+
|
101
101
|
end
|
102
102
|
end
|
103
103
|
end
|