abstract_importer 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,32 +1,40 @@
1
1
  module AbstractImporter
2
2
  class IdMap
3
-
3
+
4
4
  class IdNotMappedError < StandardError; end
5
-
5
+
6
6
  def initialize
7
7
  @id_map = Hash.new { |hash, key| hash[key] = {} }
8
8
  end
9
-
10
-
11
-
12
- def init(table_name, map)
9
+
10
+
11
+
12
+ def init(table_name, query)
13
13
  table_name = table_name.to_sym
14
- @id_map[table_name] = map
14
+ @id_map[table_name] = {}
15
+ merge! table_name, query
15
16
  end
16
-
17
+
18
+ def merge!(table_name, query)
19
+ table_name = table_name.to_sym
20
+ @id_map[table_name].merge!(query
21
+ .pluck(:id, :legacy_id)
22
+ .each_with_object({}) { |(id, legacy_id), map| map[legacy_id] = id })
23
+ end
24
+
17
25
  def get(table_name)
18
26
  @id_map[table_name.to_sym].dup
19
27
  end
20
28
  alias :[] :get
21
-
29
+
22
30
  def contains?(table_name, id)
23
31
  @id_map[table_name.to_sym].key?(id)
24
32
  end
25
-
33
+
26
34
  def <<(record)
27
35
  register(record: record)
28
36
  end
29
-
37
+
30
38
  def register(options={})
31
39
  if options.key?(:record)
32
40
  record = options[:record]
@@ -35,17 +43,17 @@ module AbstractImporter
35
43
  table_name = options[:table_name] if options.key?(:table_name)
36
44
  legacy_id = options[:legacy_id] if options.key?(:legacy_id)
37
45
  record_id = options[:record_id] if options.key?(:record_id)
38
-
46
+
39
47
  table_name = table_name.to_sym
40
48
  @id_map[table_name][legacy_id] = record_id
41
49
  end
42
-
50
+
43
51
  def apply!(legacy_id, depends_on)
44
52
  return nil if legacy_id.blank?
45
53
  id_map = @id_map[depends_on]
46
54
  raise IdNotMappedError.new unless id_map.key?(legacy_id)
47
55
  id_map[legacy_id]
48
56
  end
49
-
57
+
50
58
  end
51
59
  end
@@ -2,7 +2,9 @@ module AbstractImporter
2
2
  class ImportOptions
3
3
  CALLBACKS = [ :finder,
4
4
  :rescue,
5
+ :rescue_batch,
5
6
  :before_build,
7
+ :before_batch,
6
8
  :before_create,
7
9
  :before_update,
8
10
  :before_save,
@@ -1,17 +1,17 @@
1
1
  module AbstractImporter
2
2
  class ImportPlan
3
-
3
+
4
4
  def initialize
5
5
  @plan = {} # <-- requires Ruby 1.9's ordered hash
6
6
  end
7
-
7
+
8
8
  def to_h
9
9
  @plan.dup
10
10
  end
11
-
11
+
12
12
  def method_missing(plural, &block)
13
13
  @plan[plural] = block
14
14
  end
15
-
15
+
16
16
  end
17
17
  end
@@ -2,52 +2,52 @@ module AbstractImporter
2
2
  module Reporters
3
3
  class BaseReporter
4
4
  attr_reader :io
5
-
5
+
6
6
  def initialize(io)
7
7
  @io = io
8
8
  end
9
-
10
-
11
-
9
+
10
+
11
+
12
12
  def start_all(importer)
13
13
  io.puts "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
14
14
  end
15
-
15
+
16
16
  def finish_all(importer, ms)
17
17
  io.puts "\n\nFinished in #{distance_of_time(ms)}"
18
18
  end
19
-
19
+
20
20
  def finish_setup(ms)
21
21
  io.puts "Setup took #{distance_of_time(ms)}\n"
22
22
  end
23
-
23
+
24
24
  def start_collection(collection)
25
25
  io.puts "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
26
26
  end
27
-
27
+
28
28
  def finish_collection(collection, summary)
29
29
  end
30
-
31
-
32
-
30
+
31
+
32
+
33
33
  def record_created(record)
34
34
  end
35
-
35
+
36
36
  def record_failed(record, hash)
37
37
  end
38
-
39
-
40
-
38
+
39
+
40
+
41
41
  def count_notice(message)
42
42
  end
43
-
43
+
44
44
  def count_error(message)
45
45
  end
46
-
47
-
48
-
46
+
47
+
48
+
49
49
  protected
50
-
50
+
51
51
  def distance_of_time(milliseconds)
52
52
  milliseconds = milliseconds.to_i
53
53
  seconds = milliseconds / 1000
@@ -58,7 +58,7 @@ module AbstractImporter
58
58
  minutes %= 60
59
59
  days = hours / 24
60
60
  hours %= 24
61
-
61
+
62
62
  time = []
63
63
  time << "#{days} days" unless days.zero?
64
64
  time << "#{hours} hours" unless hours.zero?
@@ -66,7 +66,7 @@ module AbstractImporter
66
66
  time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
67
67
  time.join(", ")
68
68
  end
69
-
69
+
70
70
  end
71
71
  end
72
72
  end
@@ -2,97 +2,97 @@ module AbstractImporter
2
2
  module Reporters
3
3
  class DebugReporter < BaseReporter
4
4
  attr_reader :invalid_params
5
-
5
+
6
6
  def initialize(io)
7
7
  super
8
8
  @notices = {}
9
9
  @errors = {}
10
10
  @invalid_params = {}
11
11
  end
12
-
13
-
14
-
12
+
13
+
14
+
15
15
  def production?
16
16
  Rails.env.production?
17
17
  end
18
-
19
-
20
-
18
+
19
+
20
+
21
21
  def start_all(importer)
22
22
  super
23
23
  end
24
-
24
+
25
25
  def finish_all(importer, ms)
26
26
  print_invalid_params
27
27
  super
28
28
  end
29
-
30
-
31
-
29
+
30
+
31
+
32
32
  def finish_setup(ms)
33
33
  super
34
34
  end
35
-
36
-
37
-
35
+
36
+
37
+
38
38
  def start_collection(collection)
39
39
  super
40
40
  @notices = {}
41
41
  @errors = {}
42
42
  end
43
-
43
+
44
44
  def finish_collection(collection, summary)
45
45
  print_summary summary, collection.name
46
46
  print_messages @notices, "Notices"
47
47
  print_messages @errors, "Errors"
48
48
  end
49
-
50
-
51
-
49
+
50
+
51
+
52
52
  def record_created(record)
53
53
  io.print "." unless production?
54
54
  end
55
-
55
+
56
56
  def record_failed(record, hash)
57
57
  io.print "×" unless production?
58
-
58
+
59
59
  error_messages = invalid_params[record.class.name] ||= {}
60
60
  record.errors.full_messages.each do |error_message|
61
61
  error_messages[error_message] = hash unless error_messages.key?(error_message)
62
62
  count_error(error_message)
63
63
  end
64
64
  end
65
-
66
-
67
-
65
+
66
+
67
+
68
68
  def status(s)
69
69
  io.puts s
70
70
  end
71
-
71
+
72
72
  def stat(s)
73
73
  io.puts " #{s}"
74
74
  end
75
75
  alias :info :stat
76
-
76
+
77
77
  def file(s)
78
78
  io.puts s.inspect
79
79
  end
80
-
81
-
82
-
80
+
81
+
82
+
83
83
  def count_notice(message)
84
84
  return if production?
85
85
  @notices[message] = (@notices[message] || 0) + 1
86
86
  end
87
-
87
+
88
88
  def count_error(message)
89
89
  @errors[message] = (@errors[message] || 0) + 1
90
90
  end
91
-
92
-
93
-
91
+
92
+
93
+
94
94
  private
95
-
95
+
96
96
  def print_invalid_params
97
97
  return if invalid_params.empty?
98
98
  status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
@@ -103,7 +103,7 @@ module AbstractImporter
103
103
  end
104
104
  end
105
105
  end
106
-
106
+
107
107
  def print_summary(summary, plural)
108
108
  stat "\n #{summary.total} #{plural} were found"
109
109
  if summary.total > 0
@@ -117,7 +117,7 @@ module AbstractImporter
117
117
  stat "#{distance_of_time(summary.ms)} elapsed"
118
118
  end
119
119
  end
120
-
120
+
121
121
  def print_messages(array, caption)
122
122
  return if array.empty?
123
123
  status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
@@ -125,7 +125,7 @@ module AbstractImporter
125
125
  stat "#{count} × #{message}"
126
126
  end
127
127
  end
128
-
128
+
129
129
  end
130
130
  end
131
131
  end
@@ -1,19 +1,19 @@
1
1
  module AbstractImporter
2
2
  module Reporters
3
3
  class NullReporter < BaseReporter
4
-
4
+
5
5
  def start_all(importer)
6
6
  end
7
-
7
+
8
8
  def finish_all(importer, ms)
9
9
  end
10
-
10
+
11
11
  def finish_setup(ms)
12
12
  end
13
-
13
+
14
14
  def start_collection(collection)
15
15
  end
16
-
16
+
17
17
  end
18
18
  end
19
19
  end
@@ -4,50 +4,50 @@ module AbstractImporter
4
4
  module Reporters
5
5
  class PerformanceReporter < BaseReporter
6
6
  attr_reader :sample_size
7
-
7
+
8
8
  def initialize(io, options={})
9
9
  super io
10
10
  @sample_size = options.fetch(:sample_size, 50)
11
11
  ObjectSpace.trace_object_allocations_start
12
12
  end
13
-
14
-
13
+
14
+
15
15
  def start_collection(collection)
16
16
  super
17
17
  @collection = collection
18
18
  @major_gc_runs = GC.stat[:major_gc_count]
19
19
  @i = 0
20
20
  end
21
-
21
+
22
22
  def finish_collection(collection, summary)
23
23
  @collection = nil
24
24
  return if @i.zero?
25
25
  find_objects_holding_onto_references_to_a collection.model
26
26
  end
27
-
27
+
28
28
  def record_created(record)
29
29
  print_stats if @i % sample_size == 0
30
30
  @i += 1
31
31
  end
32
-
32
+
33
33
  def record_failed(record, hash)
34
34
  print_stats if @i % sample_size == 0
35
35
  @i += 1
36
36
  end
37
-
38
-
37
+
38
+
39
39
  def print_stats
40
40
  stats = GC.stat
41
41
  objects = ObjectSpace.count_objects
42
42
  puts "gc[minor]: #{stats[:minor_gc_count]}, gc[major]: #{stats[:major_gc_count]}, objects: #{objects[:TOTAL] - objects[:FREE]}, memsize: #{(ObjectSpace.memsize_of_all / 1048576.0).round(3)}MB, #{collection.name}: #{ObjectSpace.each_object(collection.model).count}"
43
43
  end
44
-
44
+
45
45
  private
46
46
  attr_reader :collection
47
-
47
+
48
48
  def find_objects_holding_onto_references_to_a(model)
49
49
  GC.start
50
-
50
+
51
51
  # After GC.start, all models in this collection should be
52
52
  # garbage-collected unless there is a memory leak. Find one
53
53
  # of the uncollected objects and figure out what is holding
@@ -58,11 +58,11 @@ module AbstractImporter
58
58
  return
59
59
  end
60
60
  puts "\e[33mThere are #{ObjectSpace.each_object(model).count} #{model.name.tableize.gsub("_", " ")} still in memory\e[0m"
61
-
61
+
62
62
  example_klass = example.class.name
63
63
  example_id = example.object_id
64
64
  example = nil
65
-
65
+
66
66
  # Search through all objects to find ones that hold a reference
67
67
  # to the model that hasn't been garbage-collected.
68
68
  print "\e[90m"
@@ -72,7 +72,7 @@ module AbstractImporter
72
72
  ObjectSpace.each_object do |o|
73
73
  pbar.inc
74
74
  next if ObjectSpace.reachable_objects_from(o).none? { |oo| oo.object_id == example_id }
75
-
75
+
76
76
  message = "#{o.class.name}"
77
77
  case o
78
78
  when Array
@@ -84,12 +84,12 @@ module AbstractImporter
84
84
  end
85
85
  message << " [#{ObjectSpace.allocation_sourcefile(o)}" <<
86
86
  ":#{ObjectSpace.allocation_sourceline(o)}]"
87
-
87
+
88
88
  objects_of_holding.push(message)
89
89
  end
90
90
  pbar.finish
91
91
  print "\e[0m"
92
-
92
+
93
93
  if objects_of_holding.none?
94
94
  puts "\e[95mNo objects are holding a reference to the first one\e[0m"
95
95
  else
@@ -97,7 +97,7 @@ module AbstractImporter
97
97
  "\e[35m#{objects_of_holding.join("\n")}\e[0m"
98
98
  end
99
99
  end
100
-
100
+
101
101
  end
102
102
  end
103
103
  end